From 1fabc7e0ab133f27a0453a7d2d52a6339705d603 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 23 Jan 2019 09:32:32 +1100 Subject: [PATCH 001/107] Add Hash to beacon_block_store errors --- beacon_node/db/src/stores/beacon_block_store.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/beacon_node/db/src/stores/beacon_block_store.rs b/beacon_node/db/src/stores/beacon_block_store.rs index 6477573e8..3ec558c93 100644 --- a/beacon_node/db/src/stores/beacon_block_store.rs +++ b/beacon_node/db/src/stores/beacon_block_store.rs @@ -6,8 +6,8 @@ use types::{readers::BeaconBlockReader, BeaconBlock, Hash256}; #[derive(Clone, Debug, PartialEq)] pub enum BeaconBlockAtSlotError { - UnknownBeaconBlock, - InvalidBeaconBlock, + UnknownBeaconBlock(Hash256), + InvalidBeaconBlock(Hash256), DBError(String), } @@ -73,7 +73,7 @@ impl BeaconBlockStore { current_hash = block_reader.parent_root(); } } else { - break Err(BeaconBlockAtSlotError::UnknownBeaconBlock); + break Err(BeaconBlockAtSlotError::UnknownBeaconBlock(current_hash)); } } } @@ -145,7 +145,7 @@ mod tests { db.put(DB_COLUMN, hash, ssz).unwrap(); assert_eq!( store.block_at_slot(other_hash, 42), - Err(BeaconBlockAtSlotError::UnknownBeaconBlock) + Err(BeaconBlockAtSlotError::UnknownBeaconBlock(*other_hash)) ); } @@ -243,7 +243,11 @@ mod tests { let ssz = bs.block_at_slot(&hashes[4], 6).unwrap(); assert_eq!(ssz, None); - let ssz = bs.block_at_slot(&Hash256::from("unknown".as_bytes()), 2); - assert_eq!(ssz, Err(BeaconBlockAtSlotError::UnknownBeaconBlock)); + let bad_hash = &Hash256::from("unknown".as_bytes()); + let ssz = bs.block_at_slot(bad_hash, 2); + assert_eq!( + ssz, + Err(BeaconBlockAtSlotError::UnknownBeaconBlock(*bad_hash)) + ); } } From b55591680813ab2f4ee19a5b9c11e91e3a6581ec Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 23 Jan 2019 09:33:04 +1100 Subject: [PATCH 002/107] Add slow LMD ghost implementation --- beacon_node/Cargo.toml | 4 + .../beacon_chain/{tests => }/chain_test.rs | 2 +- beacon_node/src/beacon_chain/lmd_ghost.rs | 170 ++++++++++++++++++ beacon_node/src/beacon_chain/mod.rs | 24 ++- beacon_node/src/main.rs | 26 +++ eth2/types/src/readers/block_reader.rs | 1 + 6 files changed, 224 insertions(+), 3 deletions(-) rename beacon_node/src/beacon_chain/{tests => }/chain_test.rs (96%) create mode 100644 beacon_node/src/beacon_chain/lmd_ghost.rs diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 44e6cda01..1db739d7c 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -13,8 +13,12 @@ clap = "2.32.0" db = { path = "db" } dirs = "1.0.3" futures = "0.1.23" +genesis = { path = "../eth2/genesis" } slog = "^2.2.3" +slot_clock = { path = "../eth2/utils/slot_clock" } slog-term = "^2.4.0" slog-async = "^2.3.0" +spec = { path = "../eth2/spec" } +types = { path = "../eth2/types" } ssz = { path = "../eth2/utils/ssz" } tokio = "0.1" diff --git a/beacon_node/src/beacon_chain/tests/chain_test.rs b/beacon_node/src/beacon_chain/chain_test.rs similarity index 96% rename from beacon_node/src/beacon_chain/tests/chain_test.rs rename to beacon_node/src/beacon_chain/chain_test.rs index 8d2cfa501..3d7d54d36 100644 --- a/beacon_node/src/beacon_chain/tests/chain_test.rs +++ b/beacon_node/src/beacon_chain/chain_test.rs @@ -1,4 +1,4 @@ -use chain::{BeaconChain, BlockProcessingOutcome}; +use super::{BeaconChain, BlockProcessingOutcome}; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, MemoryDB, diff --git a/beacon_node/src/beacon_chain/lmd_ghost.rs b/beacon_node/src/beacon_chain/lmd_ghost.rs new file mode 100644 index 000000000..7f00a9276 --- /dev/null +++ b/beacon_node/src/beacon_chain/lmd_ghost.rs @@ -0,0 +1,170 @@ +use super::{BeaconChain, SlotClock}; +use db::{ + stores::{BeaconBlockAtSlotError, BeaconBlockStore}, + ClientDB, DBError, +}; +use slot_clock::TestingSlotClockError; +use ssz::{ssz_encode, Encodable}; +use std::collections::HashSet; +use std::sync::Arc; +use types::{ + readers::{BeaconBlockReader, BeaconStateReader}, + validator_registry::get_active_validator_indices, + BeaconBlock, Hash256, +}; + +#[derive(Debug, PartialEq)] +pub enum Outcome { + Something, +} + +#[derive(Debug, PartialEq)] +pub enum Error { + DBError(String), + MissingBeaconState(Hash256), + InvalidBeaconState(Hash256), + MissingBeaconBlock(Hash256), + InvalidBeaconBlock(Hash256), +} + +impl BeaconChain +where + T: ClientDB, + U: SlotClock, + Error: From<::Error>, +{ + pub fn slow_lmd_ghost(&mut self, start_hash: &Hash256) -> Result { + let start = self + .block_store + .get_reader(&start_hash)? + .ok_or(Error::MissingBeaconBlock(*start_hash))?; + + let start_state_root = start.state_root(); + + let state = self + .state_store + .get_reader(&start_state_root)? + .ok_or(Error::MissingBeaconState(start_state_root))? + .into_beacon_state() + .ok_or(Error::InvalidBeaconState(start_state_root))?; + + let active_validator_indices = + get_active_validator_indices(&state.validator_registry, start.slot()); + + let mut attestation_targets = Vec::with_capacity(active_validator_indices.len()); + for i in active_validator_indices { + if let Some(target) = self.latest_attestation_targets.get(&i) { + attestation_targets.push(target); + } + } + + let mut head_hash = Hash256::zero(); + let mut head_vote_count = 0; + + loop { + let child_hashes_and_slots = + get_child_hashes_and_slots(&self.block_store, &head_hash, &self.leaf_blocks)?; + + if child_hashes_and_slots.len() == 0 { + break; + } + + for (child_hash, child_slot) in child_hashes_and_slots { + let vote_count = get_vote_count( + &self.block_store, + &attestation_targets[..], + &child_hash, + child_slot, + )?; + + if vote_count > head_vote_count { + head_hash = child_hash; + head_vote_count = vote_count; + } + } + } + + Ok(head_hash) + } +} + +fn get_vote_count( + block_store: &Arc>, + attestation_targets: &[&Hash256], + block_root: &Hash256, + slot: u64, +) -> Result { + let mut count = 0; + for target in attestation_targets { + let (root_at_slot, _) = block_store + .block_at_slot(&block_root, slot)? + .ok_or(Error::MissingBeaconBlock(*block_root))?; + if root_at_slot == *block_root { + count += 1; + } + } + Ok(count) +} + +/// Starting from some `leaf_hashes`, recurse back down each branch until the `root_hash`, adding +/// each `block_root` and `slot` to a HashSet. +fn get_child_hashes_and_slots( + block_store: &Arc>, + root_hash: &Hash256, + leaf_hashes: &HashSet, +) -> Result, Error> { + let mut hash_set = HashSet::new(); + + for leaf_hash in leaf_hashes { + let mut current_hash = *leaf_hash; + + loop { + if let Some(block_reader) = block_store.get_reader(¤t_hash)? { + let parent_root = block_reader.parent_root(); + + let new_hash = hash_set.insert((current_hash, block_reader.slot())); + + // If the hash just added was already in the set, break the loop. + // + // In such a case, the present branch has merged with a branch that is already in + // the set. + if !new_hash { + break; + } + + // The branch is exhausted if the parent of this block is the root_hash. + if parent_root == *root_hash { + break; + } + + current_hash = parent_root.clone(); + } else { + return Err(Error::MissingBeaconBlock(current_hash)); + } + } + } + + Ok(hash_set) +} + +impl From for Error { + fn from(e: DBError) -> Error { + Error::DBError(e.message) + } +} + +impl From for Error { + fn from(e: BeaconBlockAtSlotError) -> Error { + match e { + BeaconBlockAtSlotError::UnknownBeaconBlock(h) => Error::MissingBeaconBlock(h), + BeaconBlockAtSlotError::InvalidBeaconBlock(h) => Error::InvalidBeaconBlock(h), + BeaconBlockAtSlotError::DBError(msg) => Error::DBError(msg), + } + } +} + +impl From for Error { + fn from(_: TestingSlotClockError) -> Error { + unreachable!(); // Testing clock never throws an error. + } +} diff --git a/beacon_node/src/beacon_chain/mod.rs b/beacon_node/src/beacon_chain/mod.rs index 2a3d41538..be1d6ce9d 100644 --- a/beacon_node/src/beacon_chain/mod.rs +++ b/beacon_node/src/beacon_chain/mod.rs @@ -1,5 +1,8 @@ mod block_processing; mod block_production; +#[cfg(test)] +mod chain_test; +mod lmd_ghost; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, @@ -9,11 +12,18 @@ use genesis::{genesis_beacon_block, genesis_beacon_state, GenesisError}; use slot_clock::SlotClock; use spec::ChainSpec; use ssz::ssz_encode; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use types::Hash256; -pub use crate::block_processing::Outcome as BlockProcessingOutcome; +pub use self::block_processing::Outcome as BlockProcessingOutcome; + +#[derive(Debug, PartialEq)] +pub struct CheckPoint { + block_root: Hash256, + state_root: Hash256, + slot: u64, +} #[derive(Debug, PartialEq)] pub enum BeaconChainError { @@ -29,6 +39,8 @@ pub struct BeaconChain { pub leaf_blocks: HashSet, pub canonical_leaf_block: Hash256, pub spec: ChainSpec, + latest_attestation_targets: HashMap, + finalized_checkpoint: CheckPoint, } impl BeaconChain @@ -57,6 +69,12 @@ where let mut leaf_blocks = HashSet::new(); leaf_blocks.insert(block_root); + let finalized_checkpoint = CheckPoint { + block_root, + state_root, + slot: genesis_block.slot, + }; + Ok(Self { block_store, state_store, @@ -64,6 +82,8 @@ where leaf_blocks, canonical_leaf_block: block_root, spec, + latest_attestation_targets: HashMap::new(), + finalized_checkpoint, }) } } diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index 08660cbc4..26b6e362e 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -1,14 +1,23 @@ extern crate slog; +mod beacon_chain; mod config; mod rpc; use std::path::PathBuf; +use self::beacon_chain::BeaconChain; use crate::config::LighthouseConfig; use crate::rpc::start_server; use clap::{App, Arg}; +use db::{ + stores::{BeaconBlockStore, BeaconStateStore}, + MemoryDB, +}; use slog::{error, info, o, Drain}; +use slot_clock::SystemTimeSlotClock; +use spec::ChainSpec; +use std::sync::Arc; fn main() { let decorator = slog_term::TermDecorator::new().build(); @@ -58,6 +67,23 @@ fn main() { "data_dir" => &config.data_dir.to_str(), "port" => &config.p2p_listen_port); + // Specification (presently fixed to foundation). + let spec = ChainSpec::foundation(); + + // Database (presently in-memory) + let db = Arc::new(MemoryDB::open()); + let block_store = Arc::new(BeaconBlockStore::new(db.clone())); + let state_store = Arc::new(BeaconStateStore::new(db.clone())); + + // Slot clock + let slot_clock = SystemTimeSlotClock::new(spec.genesis_time, spec.slot_duration) + .expect("Unable to load SystemTimeSlotClock"); + + // Genesis chain + // TODO: persist chain to storage. + let _chain_result = + BeaconChain::genesis(state_store.clone(), block_store.clone(), slot_clock, spec); + let _server = start_server(log.clone()); loop { diff --git a/eth2/types/src/readers/block_reader.rs b/eth2/types/src/readers/block_reader.rs index 91a2852ac..d87bc5caf 100644 --- a/eth2/types/src/readers/block_reader.rs +++ b/eth2/types/src/readers/block_reader.rs @@ -1,3 +1,4 @@ +use super::state_reader::BeaconStateReader; use crate::{BeaconBlock, Hash256}; use std::fmt::Debug; From 1256ba0d010c8264bc23f401efb73613da4b9661 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 23 Jan 2019 19:25:05 +1100 Subject: [PATCH 003/107] Implement very raw state transition logic --- Cargo.toml | 1 + beacon_node/Cargo.toml | 1 + beacon_node/beacon_chain/Cargo.toml | 18 + .../beacon_chain/src/block_processing.rs | 136 +++++ .../src}/block_production.rs | 54 +- .../mod.rs => beacon_chain/src/lib.rs} | 3 +- .../src}/lmd_ghost.rs | 0 .../beacon_chain/src/state_transition.rs | 484 ++++++++++++++++++ .../src}/transition.rs | 0 .../tests/chain.rs} | 5 +- .../src/beacon_chain/block_processing.rs | 72 --- beacon_node/src/main.rs | 3 +- eth2/genesis/src/beacon_block.rs | 4 +- eth2/spec/src/foundation.rs | 1 + eth2/types/src/attestation.rs | 24 +- eth2/types/src/attestation_data.rs | 42 +- .../src/attestation_data_and_custody_bit.rs | 20 +- eth2/types/src/beacon_block.rs | 2 +- eth2/types/src/beacon_state.rs | 12 +- eth2/types/src/lib.rs | 2 + eth2/types/src/pending_attestation_record.rs | 10 +- eth2/types/src/validator_record.rs | 6 + .../slot_clock/src/system_time_slot_clock.rs | 6 +- eth2/validator_induction/src/inductor.rs | 1 + 24 files changed, 748 insertions(+), 159 deletions(-) create mode 100644 beacon_node/beacon_chain/Cargo.toml create mode 100644 beacon_node/beacon_chain/src/block_processing.rs rename beacon_node/{src/beacon_chain => beacon_chain/src}/block_production.rs (55%) rename beacon_node/{src/beacon_chain/mod.rs => beacon_chain/src/lib.rs} (98%) rename beacon_node/{src/beacon_chain => beacon_chain/src}/lmd_ghost.rs (100%) create mode 100644 beacon_node/beacon_chain/src/state_transition.rs rename beacon_node/{src/beacon_chain => beacon_chain/src}/transition.rs (100%) rename beacon_node/{src/beacon_chain/chain_test.rs => beacon_chain/tests/chain.rs} (90%) delete mode 100644 beacon_node/src/beacon_chain/block_processing.rs diff --git a/Cargo.toml b/Cargo.toml index c73e9f22d..bcaaef87e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ members = [ "eth2/validator_shuffling", "beacon_node", "beacon_node/db", + "beacon_node/beacon_chain", "protos", "validator_client", ] diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 1db739d7c..7b44bb0f9 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -6,6 +6,7 @@ edition = "2018" [dependencies] bls = { path = "../eth2/utils/bls" } +beacon_chain = { path = "beacon_chain" } grpcio = { version = "0.4", default-features = false, features = ["protobuf-codec"] } protobuf = "2.0.2" protos = { path = "../protos" } diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml new file mode 100644 index 000000000..d6dac9a4b --- /dev/null +++ b/beacon_node/beacon_chain/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "beacon_chain" +version = "0.1.0" +authors = ["Paul Hauner "] +edition = "2018" + +[dependencies] +bls = { path = "../../eth2/utils/bls" } +boolean-bitfield = { path = "../../eth2/utils/boolean-bitfield" } +db = { path = "../db" } +failure = "0.1" +failure_derive = "0.1" +genesis = { path = "../../eth2/genesis" } +serde_derive = "1.0" +slot_clock = { path = "../../eth2/utils/slot_clock" } +spec = { path = "../../eth2/spec" } +ssz = { path = "../../eth2/utils/ssz" } +types = { path = "../../eth2/types" } diff --git a/beacon_node/beacon_chain/src/block_processing.rs b/beacon_node/beacon_chain/src/block_processing.rs new file mode 100644 index 000000000..6c534bbce --- /dev/null +++ b/beacon_node/beacon_chain/src/block_processing.rs @@ -0,0 +1,136 @@ +use super::state_transition::Error as TransitionError; +use super::{BeaconChain, ClientDB, DBError, SlotClock}; +use slot_clock::{SystemTimeSlotClockError, TestingSlotClockError}; +use ssz::{ssz_encode, Encodable}; +use types::{ + readers::{BeaconBlockReader, BeaconStateReader}, + Hash256, +}; + +#[derive(Debug, PartialEq)] +pub enum Outcome { + FutureSlot, + Processed, + NewCanonicalBlock, + NewReorgBlock, + NewForkBlock, + StateTransitionFailed(TransitionError), + StateRootMismatch, +} + +#[derive(Debug, PartialEq)] +pub enum Error { + DBError(String), + SlotClockError(SystemTimeSlotClockError), + + NotImplemented, + PresentSlotIsNone, + UnableToDecodeBlock, + MissingParentState(Hash256), + InvalidParentState(Hash256), + MissingBeaconBlock(Hash256), + InvalidBeaconBlock(Hash256), + MissingParentBlock(Hash256), + NoBlockProducer, + StateSlotMismatch, + BadBlockSignature, + BadRandaoSignature, + MaxProposerSlashingsExceeded, + BadProposerSlashing, + MaxAttestationsExceeded, + BadAttestation, + NoBlockRoot, + MaxDepositsExceeded, + MaxExitsExceeded, + BadExit, + BadCustodyReseeds, + BadCustodyChallenges, + BadCustodyResponses, +} + +impl BeaconChain +where + T: ClientDB, + U: SlotClock, + Error: From<::Error>, +{ + pub fn process_block(&mut self, block: V) -> Result + where + V: BeaconBlockReader + Encodable + Sized, + { + let block = block + .into_beacon_block() + .ok_or(Error::UnableToDecodeBlock)?; + let block_root = block.canonical_root(); + + let present_slot = self + .slot_clock + .present_slot()? + .ok_or(Error::PresentSlotIsNone)?; + + // Block from future slots (i.e., greater than the present slot) should not be processed. + if block.slot() > present_slot { + return Ok(Outcome::FutureSlot); + } + + let parent_block_root = block.parent_root(); + + let parent_block = self + .block_store + .get_reader(&parent_block_root)? + .ok_or(Error::MissingParentBlock(parent_block_root))?; + + let parent_state_root = parent_block.parent_root(); + let parent_state = self + .state_store + .get_reader(&parent_state_root)? + .ok_or(Error::MissingParentState(parent_state_root))? + .into_beacon_state() + .ok_or(Error::InvalidParentState(parent_state_root))?; + + let state = match self.state_transition(parent_state, &block) { + Ok(state) => state, + Err(error) => return Ok(Outcome::StateTransitionFailed(error)), + }; + + let state_root = state.canonical_root(); + + if block.state_root != state_root { + return Ok(Outcome::StateRootMismatch); + } + + // Store the block and state. + self.block_store.put(&block_root, &ssz_encode(&block)[..])?; + self.state_store.put(&state_root, &ssz_encode(&state)[..])?; + + // Update leaf blocks so the implementation can track the chain heads. + if self.leaf_blocks.contains(&block.parent_root()) { + self.leaf_blocks.remove(&block.parent_root()); + } + if self.canonical_leaf_block == block.parent_root() { + self.canonical_leaf_block = block_root; + } + self.leaf_blocks.insert(block_root); + + // The block was sucessfully processed. + Ok(Outcome::Processed) + } +} + +impl From for Error { + fn from(e: DBError) -> Error { + Error::DBError(e.message) + } +} + +impl From for Error { + fn from(_: TestingSlotClockError) -> Error { + unreachable!(); // Testing clock never throws an error. + } +} + +impl From for Error { + fn from(e: SystemTimeSlotClockError) -> Error { + Error::SlotClockError(e) + } +} diff --git a/beacon_node/src/beacon_chain/block_production.rs b/beacon_node/beacon_chain/src/block_production.rs similarity index 55% rename from beacon_node/src/beacon_chain/block_production.rs rename to beacon_node/beacon_chain/src/block_production.rs index ba781d6e9..f6219fc87 100644 --- a/beacon_node/src/beacon_chain/block_production.rs +++ b/beacon_node/beacon_chain/src/block_production.rs @@ -1,13 +1,16 @@ +use super::state_transition::Error as TransitionError; use super::{BeaconChain, ClientDB, DBError, SlotClock}; +use bls::Signature; use slot_clock::TestingSlotClockError; use types::{ readers::{BeaconBlockReader, BeaconStateReader}, - BeaconBlock, BeaconState, Hash256, + BeaconBlock, BeaconBlockBody, BeaconState, Hash256, }; #[derive(Debug, PartialEq)] pub enum Error { DBError(String), + StateTransitionError(TransitionError), PresentSlotIsNone, } @@ -17,43 +20,48 @@ where U: SlotClock, Error: From<::Error>, { - pub fn produce_block(&mut self) -> Result<(BeaconBlock, BeaconState), Error> { - /* - * Important: this code is a big stub and only exists to ensure that tests pass. - * - * https://github.com/sigp/lighthouse/issues/107 - */ + pub fn produce_block( + &mut self, + randao_reveal: Signature, + ) -> Result<(BeaconBlock, BeaconState), Error> { let present_slot = self .slot_clock .present_slot()? .ok_or(Error::PresentSlotIsNone)?; + let parent_root = self.canonical_leaf_block; let parent_block_reader = self .block_store .get_reader(&parent_root)? .ok_or_else(|| Error::DBError("Block not found.".to_string()))?; - let parent_state_reader = self + let parent_state = self .state_store .get_reader(&parent_block_reader.state_root())? - .ok_or_else(|| Error::DBError("State not found.".to_string()))?; + .ok_or_else(|| Error::DBError("State not found.".to_string()))? + .into_beacon_state() + .ok_or_else(|| Error::DBError("State invalid.".to_string()))?; - let parent_block = parent_block_reader - .into_beacon_block() - .ok_or_else(|| Error::DBError("Bad parent block SSZ.".to_string()))?; let mut block = BeaconBlock { slot: present_slot, parent_root, state_root: Hash256::zero(), // Updated after the state is calculated. - ..parent_block + randao_reveal: randao_reveal, + candidate_pow_receipt_root: Hash256::zero(), // TODO: replace w/ eth1 data. + signature: self.spec.empty_signature.clone(), // To be completed by a validator. + body: BeaconBlockBody { + proposer_slashings: vec![], + casper_slashings: vec![], + attestations: vec![], + custody_reseeds: vec![], + custody_challenges: vec![], + custody_responses: vec![], + deposits: vec![], + exits: vec![], + }, }; - let parent_state = parent_state_reader - .into_beacon_state() - .ok_or_else(|| Error::DBError("Bad parent block SSZ.".to_string()))?; - let state = BeaconState { - slot: present_slot, - ..parent_state - }; + let state = self.state_transition(parent_state, &block)?; + let state_root = state.canonical_root(); block.state_root = state_root; @@ -68,6 +76,12 @@ impl From for Error { } } +impl From for Error { + fn from(e: TransitionError) -> Error { + Error::StateTransitionError(e) + } +} + impl From for Error { fn from(_: TestingSlotClockError) -> Error { unreachable!(); // Testing clock never throws an error. diff --git a/beacon_node/src/beacon_chain/mod.rs b/beacon_node/beacon_chain/src/lib.rs similarity index 98% rename from beacon_node/src/beacon_chain/mod.rs rename to beacon_node/beacon_chain/src/lib.rs index be1d6ce9d..6c95fcaa4 100644 --- a/beacon_node/src/beacon_chain/mod.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -1,8 +1,7 @@ mod block_processing; mod block_production; -#[cfg(test)] -mod chain_test; mod lmd_ghost; +mod state_transition; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, diff --git a/beacon_node/src/beacon_chain/lmd_ghost.rs b/beacon_node/beacon_chain/src/lmd_ghost.rs similarity index 100% rename from beacon_node/src/beacon_chain/lmd_ghost.rs rename to beacon_node/beacon_chain/src/lmd_ghost.rs diff --git a/beacon_node/beacon_chain/src/state_transition.rs b/beacon_node/beacon_chain/src/state_transition.rs new file mode 100644 index 000000000..afa805a23 --- /dev/null +++ b/beacon_node/beacon_chain/src/state_transition.rs @@ -0,0 +1,484 @@ +use super::{BeaconChain, ClientDB, DBError, SlotClock}; +use bls::{AggregatePublicKey, AggregateSignature, PublicKey, Signature}; +use boolean_bitfield::BooleanBitfield; +use slot_clock::{SystemTimeSlotClockError, TestingSlotClockError}; +use ssz::ssz_encode; +use types::{ + readers::{BeaconBlockReader, BeaconStateReader}, + AttestationData, AttestationDataAndCustodyBit, BeaconBlock, BeaconState, Exit, ForkData, + Hash256, PendingAttestationRecord, ProposalSignedData, +}; + +// TODO: define elsehwere. +const DOMAIN_ATTESTATION: u64 = 1; +const DOMAIN_PROPOSAL: u64 = 2; +const DOMAIN_EXIT: u64 = 3; +const DOMAIN_RANDAO: u64 = 4; + +macro_rules! ensure { + ($condition: expr, $result: expr) => { + if !$condition { + return Err($result); + } + }; +} + +#[derive(Debug, PartialEq)] +pub enum Error { + DBError(String), + StateAlreadyTransitioned, + NotImplemented, + PresentSlotIsNone, + UnableToDecodeBlock, + MissingParentState(Hash256), + InvalidParentState(Hash256), + MissingBeaconBlock(Hash256), + InvalidBeaconBlock(Hash256), + MissingParentBlock(Hash256), + NoBlockProducer, + StateSlotMismatch, + BadBlockSignature, + BadRandaoSignature, + MaxProposerSlashingsExceeded, + BadProposerSlashing, + MaxAttestationsExceeded, + BadAttestation, + NoBlockRoot, + MaxDepositsExceeded, + MaxExitsExceeded, + BadExit, + BadCustodyReseeds, + BadCustodyChallenges, + BadCustodyResponses, + SlotClockError(SystemTimeSlotClockError), +} + +impl BeaconChain +where + T: ClientDB, + U: SlotClock, +{ + pub fn state_transition( + &self, + mut state: BeaconState, + block: &BeaconBlock, + ) -> Result { + ensure!(state.slot < block.slot, Error::StateAlreadyTransitioned); + + for _ in state.slot..block.slot { + self.per_slot_processing(&mut state, &block.parent_root)?; + } + + /* + * Slot + */ + + ensure!(block.slot() == state.slot, Error::StateSlotMismatch); + + /* + * Proposer Signature + */ + + let block_without_signature_root = { + let mut block_without_signature = block.clone(); + block_without_signature.signature = self.spec.empty_signature.clone(); + block_without_signature.canonical_root() + }; + + let proposal_root = { + let proposal = ProposalSignedData { + slot: state.slot, + shard: self.spec.beacon_chain_shard_number, + block_root: block_without_signature_root, + }; + hash_tree_root(&proposal) + }; + + let block_proposer_index = + get_beacon_proposer_index(&state, block.slot, self.spec.epoch_length) + .ok_or(Error::NoBlockProducer)?; + let block_proposer = &state.validator_registry[block_proposer_index]; + + ensure!( + bls_verify( + &block_proposer.pubkey, + &proposal_root, + &block.signature, + get_domain(&state.fork_data, state.slot, DOMAIN_PROPOSAL) + ), + Error::BadBlockSignature + ); + + /* + * RANDAO + */ + + ensure!( + bls_verify( + &block_proposer.pubkey, + &ssz_encode(&block_proposer.proposer_slots), + &block.randao_reveal, + get_domain(&state.fork_data, state.slot, DOMAIN_RANDAO) + ), + Error::BadRandaoSignature + ); + + let new_mix = { + let mut mix = state.latest_randao_mixes + [(state.slot % self.spec.latest_randao_mixes_length) as usize] + .to_vec(); + mix.append(&mut ssz_encode(&block.randao_reveal)); + hash(&mix) + }; + + state.latest_randao_mixes[(state.slot % self.spec.latest_randao_mixes_length) as usize] = + new_mix; + + /* + * Eth1 data + */ + + // TODO: Eth1 data stuff. + + /* + * OPERATIONS + */ + + /* + * Proposer slashings + */ + + ensure!( + block.body.proposer_slashings.len() as u64 <= self.spec.max_proposer_slashings, + Error::MaxProposerSlashingsExceeded + ); + for proposer_slashing in &block.body.proposer_slashings { + let proposer = state + .validator_registry + .get(proposer_slashing.proposer_index as usize) + .ok_or(Error::BadProposerSlashing)?; + ensure!( + proposer_slashing.proposal_data_1.slot == proposer_slashing.proposal_data_2.slot, + Error::BadProposerSlashing + ); + ensure!( + proposer_slashing.proposal_data_1.shard == proposer_slashing.proposal_data_2.shard, + Error::BadProposerSlashing + ); + ensure!( + proposer_slashing.proposal_data_1.block_root + != proposer_slashing.proposal_data_2.block_root, + Error::BadProposerSlashing + ); + ensure!( + proposer.penalized_slot > state.slot, + Error::BadProposerSlashing + ); + ensure!( + bls_verify( + &proposer.pubkey, + &hash_tree_root(&proposer_slashing.proposal_data_1), + &proposer_slashing.proposal_signature_1, + get_domain( + &state.fork_data, + proposer_slashing.proposal_data_1.slot, + DOMAIN_PROPOSAL + ) + ), + Error::BadProposerSlashing + ); + ensure!( + bls_verify( + &proposer.pubkey, + &hash_tree_root(&proposer_slashing.proposal_data_2), + &proposer_slashing.proposal_signature_2, + get_domain( + &state.fork_data, + proposer_slashing.proposal_data_2.slot, + DOMAIN_PROPOSAL + ) + ), + Error::BadProposerSlashing + ); + penalize_validator(&state, proposer_slashing.proposer_index as usize); + } + + /* + * Attestations + */ + ensure!( + block.body.attestations.len() as u64 <= self.spec.max_attestations, + Error::MaxAttestationsExceeded + ); + + for attestation in &block.body.attestations { + ensure!( + attestation.data.slot + self.spec.min_attestation_inclusion_delay <= state.slot, + Error::BadAttestation + ); + ensure!( + attestation.data.slot + self.spec.epoch_length >= state.slot, + Error::BadAttestation + ); + if state.justified_slot >= state.slot - (state.slot % self.spec.epoch_length) { + ensure!( + attestation.data.justified_slot == state.justified_slot, + Error::BadAttestation + ); + } else { + ensure!( + attestation.data.justified_slot == state.previous_justified_slot, + Error::BadAttestation + ); + } + ensure!( + attestation.data.justified_block_root + == *get_block_root( + &state, + attestation.data.justified_slot, + self.spec.latest_block_roots_length + ) + .ok_or(Error::NoBlockRoot)?, + Error::BadAttestation + ); + ensure!( + (attestation.data.latest_crosslink_root + == state.latest_crosslinks[attestation.data.shard as usize].shard_block_root) + || (attestation.data.shard_block_root + == state.latest_crosslinks[attestation.data.shard as usize] + .shard_block_root), + Error::BadAttestation + ); + let participants = get_attestation_participants( + &state, + &attestation.data, + &attestation.aggregation_bitfield, + ); + let mut group_public_key = AggregatePublicKey::new(); + for participant in participants { + group_public_key.add( + state.validator_registry[participant as usize] + .pubkey + .as_raw(), + ) + } + let attestation_message = { + let attestation_data_and_custody_bit = AttestationDataAndCustodyBit { + data: attestation.data.clone(), + custody_bit: false, + }; + hash_tree_root(&attestation_data_and_custody_bit).to_vec() + }; + // Signature verification. + ensure!( + bls_verify_aggregate( + &group_public_key, + &attestation_message[..], + &attestation.aggregate_signature, + get_domain(&state.fork_data, attestation.data.slot, DOMAIN_ATTESTATION) + ), + Error::BadProposerSlashing + ); + ensure!( + attestation.data.shard_block_root == self.spec.zero_hash, + Error::BadAttestation + ); + let pending_attestation = PendingAttestationRecord { + data: attestation.data.clone(), + aggregation_bitfield: attestation.aggregation_bitfield.clone(), + custody_bitfield: attestation.custody_bitfield.clone(), + slot_included: state.slot, + }; + state.latest_attestations.push(pending_attestation); + } + + /* + * Deposits + */ + ensure!( + block.body.deposits.len() as u64 <= self.spec.max_deposits, + Error::MaxDepositsExceeded + ); + + // TODO: process deposits. + + /* + * Exits + */ + + ensure!( + block.body.exits.len() as u64 <= self.spec.max_exits, + Error::MaxExitsExceeded + ); + + for exit in &block.body.exits { + let validator = state + .validator_registry + .get(exit.validator_index as usize) + .ok_or(Error::BadExit)?; + ensure!( + validator.exit_slot > state.slot + self.spec.entry_exit_delay, + Error::BadExit + ); + ensure!(state.slot >= exit.slot, Error::BadExit); + let exit_message = { + let exit_struct = Exit { + slot: exit.slot, + validator_index: exit.validator_index, + signature: self.spec.empty_signature.clone(), + }; + hash_tree_root(&exit_struct) + }; + ensure!( + bls_verify( + &validator.pubkey, + &exit_message, + &exit.signature, + get_domain(&state.fork_data, exit.slot, DOMAIN_EXIT) + ), + Error::BadProposerSlashing + ); + initiate_validator_exit(&state, exit.validator_index); + } + + /* + * Custody + */ + ensure!( + block.body.custody_reseeds.is_empty(), + Error::BadCustodyReseeds + ); + ensure!( + block.body.custody_challenges.is_empty(), + Error::BadCustodyChallenges + ); + ensure!( + block.body.custody_responses.is_empty(), + Error::BadCustodyResponses + ); + + Ok(state) + } + + fn per_slot_processing( + &self, + state: &mut BeaconState, + previous_block_root: &Hash256, + ) -> Result<(), Error> { + let epoch_length = self.spec.epoch_length; + let latest_randao_mixes_length = self.spec.latest_randao_mixes_length; + let latest_block_roots_length = self.spec.latest_block_roots_length; + + // Misc counters. + state.slot += 1; + let block_proposer = get_beacon_proposer_index(&state, state.slot, epoch_length) + .ok_or(Error::NoBlockProducer)?; + state.validator_registry[block_proposer].proposer_slots += 1; + state.latest_randao_mixes[(state.slot % latest_randao_mixes_length) as usize] = + state.latest_randao_mixes[((state.slot - 1) % latest_randao_mixes_length) as usize]; + + // Block roots. + state.latest_block_roots + [((state.slot - 1) % self.spec.latest_block_roots_length) as usize] = + *previous_block_root; + + if state.slot % latest_block_roots_length == 0 { + let root = merkle_root(&state.latest_block_roots[..]); + state.batched_block_roots.push(root); + } + Ok(()) + } +} + +fn initiate_validator_exit(_state: &BeaconState, _index: u32) { + // TODO: stubbed out. +} + +fn get_attestation_participants( + _state: &BeaconState, + _attestation_data: &AttestationData, + _aggregation_bitfield: &BooleanBitfield, +) -> Vec { + vec![0, 1] +} + +fn get_block_root( + state: &BeaconState, + slot: u64, + latest_block_roots_length: u64, +) -> Option<&Hash256> { + // TODO: test + if state.slot <= slot + latest_block_roots_length && slot <= state.slot { + state + .latest_block_roots + .get((slot % latest_block_roots_length) as usize) + } else { + None + } +} + +fn penalize_validator(_state: &BeaconState, _proposer_index: usize) { + // TODO: stubbed out. +} + +fn hash(_input: &T) -> Hash256 { + // TODO: stubbed out. + Hash256::zero() +} + +fn get_domain(_fork: &ForkData, _slot: u64, _domain_type: u64) -> u64 { + // TODO: stubbed out. + 0 +} + +fn bls_verify(_pubkey: &PublicKey, _message: &[u8], _signature: &Signature, _domain: u64) -> bool { + // TODO: stubbed out. + true +} + +fn bls_verify_aggregate( + _pubkey: &AggregatePublicKey, + _message: &[u8], + _signature: &AggregateSignature, + _domain: u64, +) -> bool { + // TODO: stubbed out. + true +} + +fn hash_tree_root(_input: &T) -> Hash256 { + // TODO: stubbed out. + Hash256::zero() +} + +fn merkle_root(_input: &[Hash256]) -> Hash256 { + // TODO: stubbed out. + Hash256::zero() +} + +fn get_beacon_proposer_index( + _state: &BeaconState, + _slot: u64, + _epoch_length: u64, +) -> Option { + // TODO: stubbed out. + Some(0) +} + +impl From for Error { + fn from(e: DBError) -> Error { + Error::DBError(e.message) + } +} + +impl From for Error { + fn from(_: TestingSlotClockError) -> Error { + unreachable!(); // Testing clock never throws an error. + } +} + +impl From for Error { + fn from(e: SystemTimeSlotClockError) -> Error { + Error::SlotClockError(e) + } +} diff --git a/beacon_node/src/beacon_chain/transition.rs b/beacon_node/beacon_chain/src/transition.rs similarity index 100% rename from beacon_node/src/beacon_chain/transition.rs rename to beacon_node/beacon_chain/src/transition.rs diff --git a/beacon_node/src/beacon_chain/chain_test.rs b/beacon_node/beacon_chain/tests/chain.rs similarity index 90% rename from beacon_node/src/beacon_chain/chain_test.rs rename to beacon_node/beacon_chain/tests/chain.rs index 3d7d54d36..9c76bf59d 100644 --- a/beacon_node/src/beacon_chain/chain_test.rs +++ b/beacon_node/beacon_chain/tests/chain.rs @@ -1,4 +1,5 @@ -use super::{BeaconChain, BlockProcessingOutcome}; +use beacon_chain::{BeaconChain, BlockProcessingOutcome}; +use bls::Signature; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, MemoryDB, @@ -43,7 +44,7 @@ fn it_produces() { fn it_processes_a_block_it_produces() { let (_db, mut chain) = in_memory_test_chain(ChainSpec::foundation()); let (block, _state) = chain.produce_block().unwrap(); - let (outcome, new_block_hash) = chain.process_block(&block).unwrap(); + let (outcome, new_block_hash) = chain.process_block(block).unwrap(); assert_eq!(outcome, BlockProcessingOutcome::Processed); assert_eq!(chain.canonical_leaf_block, new_block_hash); } diff --git a/beacon_node/src/beacon_chain/block_processing.rs b/beacon_node/src/beacon_chain/block_processing.rs deleted file mode 100644 index b24a6f1f8..000000000 --- a/beacon_node/src/beacon_chain/block_processing.rs +++ /dev/null @@ -1,72 +0,0 @@ -use super::{BeaconChain, ClientDB, DBError, SlotClock}; -use slot_clock::TestingSlotClockError; -use ssz::{ssz_encode, Encodable}; -use types::{readers::BeaconBlockReader, Hash256}; - -#[derive(Debug, PartialEq)] -pub enum Outcome { - FutureSlot, - Processed, - - NewCanonicalBlock, - NewReorgBlock, - NewForkBlock, -} - -#[derive(Debug, PartialEq)] -pub enum Error { - DBError(String), - NotImplemented, - PresentSlotIsNone, -} - -impl BeaconChain -where - T: ClientDB, - U: SlotClock, - Error: From<::Error>, -{ - pub fn process_block(&mut self, block: &V) -> Result<(Outcome, Hash256), Error> - where - V: BeaconBlockReader + Encodable + Sized, - { - let block_root = block.canonical_root(); - - let present_slot = self - .slot_clock - .present_slot()? - .ok_or(Error::PresentSlotIsNone)?; - - // Block from future slots (i.e., greater than the present slot) should not be processed. - if block.slot() > present_slot { - return Ok((Outcome::FutureSlot, block_root)); - } - - // TODO: block processing has been removed. - // https://github.com/sigp/lighthouse/issues/98 - - // Update leaf blocks. - self.block_store.put(&block_root, &ssz_encode(block)[..])?; - if self.leaf_blocks.contains(&block.parent_root()) { - self.leaf_blocks.remove(&block.parent_root()); - } - if self.canonical_leaf_block == block.parent_root() { - self.canonical_leaf_block = block_root; - } - self.leaf_blocks.insert(block_root); - - Ok((Outcome::Processed, block_root)) - } -} - -impl From for Error { - fn from(e: DBError) -> Error { - Error::DBError(e.message) - } -} - -impl From for Error { - fn from(_: TestingSlotClockError) -> Error { - unreachable!(); // Testing clock never throws an error. - } -} diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index 26b6e362e..f9e77723a 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -1,14 +1,13 @@ extern crate slog; -mod beacon_chain; mod config; mod rpc; use std::path::PathBuf; -use self::beacon_chain::BeaconChain; use crate::config::LighthouseConfig; use crate::rpc::start_server; +use beacon_chain::BeaconChain; use clap::{App, Arg}; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, diff --git a/eth2/genesis/src/beacon_block.rs b/eth2/genesis/src/beacon_block.rs index 3fefff6a0..9c44ece6c 100644 --- a/eth2/genesis/src/beacon_block.rs +++ b/eth2/genesis/src/beacon_block.rs @@ -7,7 +7,7 @@ pub fn genesis_beacon_block(state_root: Hash256, spec: &ChainSpec) -> BeaconBloc slot: spec.genesis_slot_number, parent_root: spec.zero_hash, state_root, - randao_reveal: spec.zero_hash, + randao_reveal: spec.empty_signature.clone(), candidate_pow_receipt_root: spec.zero_hash, signature: spec.empty_signature.clone(), body: BeaconBlockBody { @@ -47,7 +47,7 @@ mod tests { assert!(genesis_block.slot == 0); assert!(genesis_block.parent_root.is_zero()); - assert!(genesis_block.randao_reveal.is_zero()); + assert_eq!(genesis_block.randao_reveal, spec.empty_signature); assert!(genesis_block.candidate_pow_receipt_root.is_zero()); // aka deposit_root } diff --git a/eth2/spec/src/foundation.rs b/eth2/spec/src/foundation.rs index 81bee7c4a..9aa2b1971 100644 --- a/eth2/spec/src/foundation.rs +++ b/eth2/spec/src/foundation.rs @@ -107,6 +107,7 @@ fn initial_validators_for_testing() -> Vec { let validator_record = ValidatorRecord { pubkey: keypair.pk.clone(), withdrawal_credentials: Hash256::zero(), + proposer_slots: 0, randao_commitment: Hash256::zero(), randao_layers: 0, activation_slot: u64::max_value(), diff --git a/eth2/types/src/attestation.rs b/eth2/types/src/attestation.rs index fb8b946a5..bef35bc78 100644 --- a/eth2/types/src/attestation.rs +++ b/eth2/types/src/attestation.rs @@ -7,32 +7,32 @@ use rand::RngCore; #[derive(Debug, Clone, PartialEq)] pub struct Attestation { pub data: AttestationData, - pub participation_bitfield: Bitfield, + pub aggregation_bitfield: Bitfield, pub custody_bitfield: Bitfield, - pub aggregate_sig: AggregateSignature, + pub aggregate_signature: AggregateSignature, } impl Encodable for Attestation { fn ssz_append(&self, s: &mut SszStream) { s.append(&self.data); - s.append(&self.participation_bitfield); + s.append(&self.aggregation_bitfield); s.append(&self.custody_bitfield); - s.append(&self.aggregate_sig); + s.append(&self.aggregate_signature); } } impl Decodable for Attestation { fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { let (data, i) = AttestationData::ssz_decode(bytes, i)?; - let (participation_bitfield, i) = Bitfield::ssz_decode(bytes, i)?; + let (aggregation_bitfield, i) = Bitfield::ssz_decode(bytes, i)?; let (custody_bitfield, i) = Bitfield::ssz_decode(bytes, i)?; - let (aggregate_sig, i) = AggregateSignature::ssz_decode(bytes, i)?; + let (aggregate_signature, i) = AggregateSignature::ssz_decode(bytes, i)?; let attestation_record = Self { data, - participation_bitfield, + aggregation_bitfield, custody_bitfield, - aggregate_sig, + aggregate_signature, }; Ok((attestation_record, i)) } @@ -42,9 +42,9 @@ impl Attestation { pub fn zero() -> Self { Self { data: AttestationData::zero(), - participation_bitfield: Bitfield::new(), + aggregation_bitfield: Bitfield::new(), custody_bitfield: Bitfield::new(), - aggregate_sig: AggregateSignature::new(), + aggregate_signature: AggregateSignature::new(), } } } @@ -53,9 +53,9 @@ impl TestRandom for Attestation { fn random_for_test(rng: &mut T) -> Self { Self { data: <_>::random_for_test(rng), - participation_bitfield: <_>::random_for_test(rng), + aggregation_bitfield: <_>::random_for_test(rng), custody_bitfield: <_>::random_for_test(rng), - aggregate_sig: <_>::random_for_test(rng), + aggregate_signature: <_>::random_for_test(rng), } } } diff --git a/eth2/types/src/attestation_data.rs b/eth2/types/src/attestation_data.rs index e1d70e2b0..f4eccb86e 100644 --- a/eth2/types/src/attestation_data.rs +++ b/eth2/types/src/attestation_data.rs @@ -8,10 +8,10 @@ pub const SSZ_ATTESTION_DATA_LENGTH: usize = { 8 + // shard 32 + // beacon_block_hash 32 + // epoch_boundary_hash - 32 + // shard_block_hash - 32 + // latest_crosslink_hash + 32 + // shard_block_root + 32 + // latest_crosslink_root 8 + // justified_slot - 32 // justified_block_hash + 32 // justified_block_root }; #[derive(Debug, Clone, PartialEq, Default)] @@ -20,10 +20,10 @@ pub struct AttestationData { pub shard: u64, pub beacon_block_hash: Hash256, pub epoch_boundary_hash: Hash256, - pub shard_block_hash: Hash256, - pub latest_crosslink_hash: Hash256, + pub shard_block_root: Hash256, + pub latest_crosslink_root: Hash256, pub justified_slot: u64, - pub justified_block_hash: Hash256, + pub justified_block_root: Hash256, } impl AttestationData { @@ -33,10 +33,10 @@ impl AttestationData { shard: 0, beacon_block_hash: Hash256::zero(), epoch_boundary_hash: Hash256::zero(), - shard_block_hash: Hash256::zero(), - latest_crosslink_hash: Hash256::zero(), + shard_block_root: Hash256::zero(), + latest_crosslink_root: Hash256::zero(), justified_slot: 0, - justified_block_hash: Hash256::zero(), + justified_block_root: Hash256::zero(), } } @@ -53,10 +53,10 @@ impl Encodable for AttestationData { s.append(&self.shard); s.append(&self.beacon_block_hash); s.append(&self.epoch_boundary_hash); - s.append(&self.shard_block_hash); - s.append(&self.latest_crosslink_hash); + s.append(&self.shard_block_root); + s.append(&self.latest_crosslink_root); s.append(&self.justified_slot); - s.append(&self.justified_block_hash); + s.append(&self.justified_block_root); } } @@ -66,20 +66,20 @@ impl Decodable for AttestationData { let (shard, i) = u64::ssz_decode(bytes, i)?; let (beacon_block_hash, i) = Hash256::ssz_decode(bytes, i)?; let (epoch_boundary_hash, i) = Hash256::ssz_decode(bytes, i)?; - let (shard_block_hash, i) = Hash256::ssz_decode(bytes, i)?; - let (latest_crosslink_hash, i) = Hash256::ssz_decode(bytes, i)?; + let (shard_block_root, i) = Hash256::ssz_decode(bytes, i)?; + let (latest_crosslink_root, i) = Hash256::ssz_decode(bytes, i)?; let (justified_slot, i) = u64::ssz_decode(bytes, i)?; - let (justified_block_hash, i) = Hash256::ssz_decode(bytes, i)?; + let (justified_block_root, i) = Hash256::ssz_decode(bytes, i)?; let attestation_data = AttestationData { slot, shard, beacon_block_hash, epoch_boundary_hash, - shard_block_hash, - latest_crosslink_hash, + shard_block_root, + latest_crosslink_root, justified_slot, - justified_block_hash, + justified_block_root, }; Ok((attestation_data, i)) } @@ -92,10 +92,10 @@ impl TestRandom for AttestationData { shard: <_>::random_for_test(rng), beacon_block_hash: <_>::random_for_test(rng), epoch_boundary_hash: <_>::random_for_test(rng), - shard_block_hash: <_>::random_for_test(rng), - latest_crosslink_hash: <_>::random_for_test(rng), + shard_block_root: <_>::random_for_test(rng), + latest_crosslink_root: <_>::random_for_test(rng), justified_slot: <_>::random_for_test(rng), - justified_block_hash: <_>::random_for_test(rng), + justified_block_root: <_>::random_for_test(rng), } } } diff --git a/eth2/types/src/attestation_data_and_custody_bit.rs b/eth2/types/src/attestation_data_and_custody_bit.rs index 66584b0d9..f53614d6b 100644 --- a/eth2/types/src/attestation_data_and_custody_bit.rs +++ b/eth2/types/src/attestation_data_and_custody_bit.rs @@ -1,7 +1,7 @@ use super::ssz::{Decodable, DecodeError, Encodable, SszStream}; -use rand::RngCore; -use crate::test_utils::TestRandom; use super::AttestationData; +use crate::test_utils::TestRandom; +use rand::RngCore; #[derive(Debug, Clone, PartialEq, Default)] pub struct AttestationDataAndCustodyBit { @@ -12,19 +12,16 @@ pub struct AttestationDataAndCustodyBit { impl Encodable for AttestationDataAndCustodyBit { fn ssz_append(&self, s: &mut SszStream) { s.append(&self.data); - s.append(&self.custody_bit); + // TODO: deal with bools } } impl Decodable for AttestationDataAndCustodyBit { fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { let (data, i) = <_>::ssz_decode(bytes, i)?; - let (custody_bit, i) = <_>::ssz_decode(bytes, i)?; + let custody_bit = false; - let attestation_data_and_custody_bit = AttestationDataAndCustodyBit { - data, - custody_bit, - }; + let attestation_data_and_custody_bit = AttestationDataAndCustodyBit { data, custody_bit }; Ok((attestation_data_and_custody_bit, i)) } @@ -34,16 +31,17 @@ impl TestRandom for AttestationDataAndCustodyBit { fn random_for_test(rng: &mut T) -> Self { Self { data: <_>::random_for_test(rng), - custody_bit: <_>::random_for_test(rng), + // TODO: deal with bools + custody_bit: false, } } } #[cfg(test)] mod test { - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use super::*; use super::super::ssz::ssz_encode; + use super::*; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/beacon_block.rs b/eth2/types/src/beacon_block.rs index 30a1bd83b..ee80bcd9a 100644 --- a/eth2/types/src/beacon_block.rs +++ b/eth2/types/src/beacon_block.rs @@ -10,7 +10,7 @@ pub struct BeaconBlock { pub slot: u64, pub parent_root: Hash256, pub state_root: Hash256, - pub randao_reveal: Hash256, + pub randao_reveal: Signature, pub candidate_pow_receipt_root: Hash256, pub signature: Signature, pub body: BeaconBlockBody, diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index c55ce8ec9..74f15d3ce 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -1,10 +1,10 @@ -use super::candidate_pow_receipt_root_record::CandidatePoWReceiptRootRecord; -use super::crosslink_record::CrosslinkRecord; -use super::fork_data::ForkData; -use super::pending_attestation_record::PendingAttestationRecord; -use super::validator_record::ValidatorRecord; -use super::Hash256; +use crate::candidate_pow_receipt_root_record::CandidatePoWReceiptRootRecord; +use crate::crosslink_record::CrosslinkRecord; +use crate::fork_data::ForkData; +use crate::pending_attestation_record::PendingAttestationRecord; use crate::test_utils::TestRandom; +use crate::validator_record::ValidatorRecord; +use crate::Hash256; use hashing::canonical_hash; use rand::RngCore; use ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream}; diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index f00c19d03..e23fe2ebc 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -7,6 +7,7 @@ pub mod test_utils; pub mod attestation; pub mod attestation_data; +pub mod attestation_data_and_custody_bit; pub mod beacon_block; pub mod beacon_block_body; pub mod beacon_state; @@ -35,6 +36,7 @@ use std::collections::HashMap; pub use crate::attestation::Attestation; pub use crate::attestation_data::AttestationData; +pub use crate::attestation_data_and_custody_bit::AttestationDataAndCustodyBit; pub use crate::beacon_block::BeaconBlock; pub use crate::beacon_block_body::BeaconBlockBody; pub use crate::beacon_state::BeaconState; diff --git a/eth2/types/src/pending_attestation_record.rs b/eth2/types/src/pending_attestation_record.rs index 3bebf5676..a093b9efd 100644 --- a/eth2/types/src/pending_attestation_record.rs +++ b/eth2/types/src/pending_attestation_record.rs @@ -6,7 +6,7 @@ use rand::RngCore; #[derive(Debug, Clone, PartialEq)] pub struct PendingAttestationRecord { pub data: AttestationData, - pub participation_bitfield: Bitfield, + pub aggregation_bitfield: Bitfield, pub custody_bitfield: Bitfield, pub slot_included: u64, } @@ -14,7 +14,7 @@ pub struct PendingAttestationRecord { impl Encodable for PendingAttestationRecord { fn ssz_append(&self, s: &mut SszStream) { s.append(&self.data); - s.append(&self.participation_bitfield); + s.append(&self.aggregation_bitfield); s.append(&self.custody_bitfield); s.append(&self.slot_included); } @@ -23,14 +23,14 @@ impl Encodable for PendingAttestationRecord { impl Decodable for PendingAttestationRecord { fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { let (data, i) = <_>::ssz_decode(bytes, i)?; - let (participation_bitfield, i) = <_>::ssz_decode(bytes, i)?; + let (aggregation_bitfield, i) = <_>::ssz_decode(bytes, i)?; let (custody_bitfield, i) = <_>::ssz_decode(bytes, i)?; let (slot_included, i) = <_>::ssz_decode(bytes, i)?; Ok(( Self { data, - participation_bitfield, + aggregation_bitfield, custody_bitfield, slot_included, }, @@ -43,7 +43,7 @@ impl TestRandom for PendingAttestationRecord { fn random_for_test(rng: &mut T) -> Self { Self { data: <_>::random_for_test(rng), - participation_bitfield: <_>::random_for_test(rng), + aggregation_bitfield: <_>::random_for_test(rng), custody_bitfield: <_>::random_for_test(rng), slot_included: <_>::random_for_test(rng), } diff --git a/eth2/types/src/validator_record.rs b/eth2/types/src/validator_record.rs index 5a8cde8ab..50550f6e1 100644 --- a/eth2/types/src/validator_record.rs +++ b/eth2/types/src/validator_record.rs @@ -47,6 +47,7 @@ fn status_flag_from_byte(flag: u8) -> Result, StatusFlagsDec pub struct ValidatorRecord { pub pubkey: PublicKey, pub withdrawal_credentials: Hash256, + pub proposer_slots: u64, pub randao_commitment: Hash256, pub randao_layers: u64, pub activation_slot: u64, @@ -73,6 +74,7 @@ impl Default for ValidatorRecord { Self { pubkey: PublicKey::default(), withdrawal_credentials: Hash256::default(), + proposer_slots: 0, randao_commitment: Hash256::default(), randao_layers: 0, activation_slot: std::u64::MAX, @@ -99,6 +101,7 @@ impl Encodable for ValidatorRecord { fn ssz_append(&self, s: &mut SszStream) { s.append(&self.pubkey); s.append(&self.withdrawal_credentials); + s.append(&self.proposer_slots); s.append(&self.randao_commitment); s.append(&self.randao_layers); s.append(&self.activation_slot); @@ -117,6 +120,7 @@ impl Decodable for ValidatorRecord { fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { let (pubkey, i) = <_>::ssz_decode(bytes, i)?; let (withdrawal_credentials, i) = <_>::ssz_decode(bytes, i)?; + let (proposer_slots, i) = <_>::ssz_decode(bytes, i)?; let (randao_commitment, i) = <_>::ssz_decode(bytes, i)?; let (randao_layers, i) = <_>::ssz_decode(bytes, i)?; let (activation_slot, i) = <_>::ssz_decode(bytes, i)?; @@ -135,6 +139,7 @@ impl Decodable for ValidatorRecord { Self { pubkey, withdrawal_credentials, + proposer_slots, randao_commitment, randao_layers, activation_slot, @@ -157,6 +162,7 @@ impl TestRandom for ValidatorRecord { Self { pubkey: <_>::random_for_test(rng), withdrawal_credentials: <_>::random_for_test(rng), + proposer_slots: <_>::random_for_test(rng), randao_commitment: <_>::random_for_test(rng), randao_layers: <_>::random_for_test(rng), activation_slot: <_>::random_for_test(rng), diff --git a/eth2/utils/slot_clock/src/system_time_slot_clock.rs b/eth2/utils/slot_clock/src/system_time_slot_clock.rs index 8c1e2f66c..5c5f2e0ea 100644 --- a/eth2/utils/slot_clock/src/system_time_slot_clock.rs +++ b/eth2/utils/slot_clock/src/system_time_slot_clock.rs @@ -3,10 +3,10 @@ use std::time::{Duration, SystemTime}; pub use std::time::SystemTimeError; -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub enum Error { SlotDurationIsZero, - SystemTimeError(SystemTimeError), + SystemTimeError(String), } /// Determines the present slot based upon the present system time. @@ -51,7 +51,7 @@ impl SlotClock for SystemTimeSlotClock { impl From for Error { fn from(e: SystemTimeError) -> Error { - Error::SystemTimeError(e) + Error::SystemTimeError(format!("{:?}", e)) } } diff --git a/eth2/validator_induction/src/inductor.rs b/eth2/validator_induction/src/inductor.rs index 9dd5089ba..05e6dfe3b 100644 --- a/eth2/validator_induction/src/inductor.rs +++ b/eth2/validator_induction/src/inductor.rs @@ -42,6 +42,7 @@ pub fn process_deposit( let validator = ValidatorRecord { pubkey: deposit_input.pubkey.clone(), withdrawal_credentials: deposit_input.withdrawal_credentials, + proposer_slots: 0, randao_commitment: deposit_input.randao_commitment, randao_layers: 0, activation_slot: spec.far_future_slot, From af6437eb13afdf901fb4a56c0303ac4e3e1c584c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 23 Jan 2019 20:59:59 +1100 Subject: [PATCH 004/107] Update validator client for types change --- validator_client/src/block_producer/grpc.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/validator_client/src/block_producer/grpc.rs b/validator_client/src/block_producer/grpc.rs index 033965b16..ca47ee3a6 100644 --- a/validator_client/src/block_producer/grpc.rs +++ b/validator_client/src/block_producer/grpc.rs @@ -25,12 +25,15 @@ impl BeaconNode for BeaconBlockServiceClient { let (signature, _) = Signature::ssz_decode(block.get_signature(), 0) .map_err(|_| BeaconNodeError::DecodeFailure)?; + let (randao_reveal, _) = Signature::ssz_decode(block.get_randao_reveal(), 0) + .map_err(|_| BeaconNodeError::DecodeFailure)?; + // TODO: this conversion is incomplete; fix it. Ok(Some(BeaconBlock { slot: block.get_slot(), parent_root: Hash256::zero(), state_root: Hash256::zero(), - randao_reveal: Hash256::from(block.get_randao_reveal()), + randao_reveal, candidate_pow_receipt_root: Hash256::zero(), signature, body: BeaconBlockBody { From 8e935f93bc93f370b537c9251c8bea231de4a3ea Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 23 Jan 2019 21:01:46 +1100 Subject: [PATCH 005/107] Replace EpochDutiesMap with trait in BlockProducer --- validator_client/src/block_producer/grpc.rs | 2 +- validator_client/src/block_producer/mod.rs | 35 +++++----- .../src/block_producer/service.rs | 8 +-- validator_client/src/block_producer/traits.rs | 9 +++ validator_client/src/duties/mod.rs | 70 ++++++++++++++++--- validator_client/src/main.rs | 2 +- 6 files changed, 92 insertions(+), 34 deletions(-) diff --git a/validator_client/src/block_producer/grpc.rs b/validator_client/src/block_producer/grpc.rs index ca47ee3a6..20ced3d0c 100644 --- a/validator_client/src/block_producer/grpc.rs +++ b/validator_client/src/block_producer/grpc.rs @@ -63,7 +63,7 @@ impl BeaconNode for BeaconBlockServiceClient { let mut grpc_block = GrpcBeaconBlock::new(); grpc_block.set_slot(block.slot); grpc_block.set_block_root(vec![0]); - grpc_block.set_randao_reveal(block.randao_reveal.to_vec()); + grpc_block.set_randao_reveal(ssz_encode(&block.randao_reveal)); grpc_block.set_signature(ssz_encode(&block.signature)); req.set_block(grpc_block); diff --git a/validator_client/src/block_producer/mod.rs b/validator_client/src/block_producer/mod.rs index f3cd0199b..e0ea220b6 100644 --- a/validator_client/src/block_producer/mod.rs +++ b/validator_client/src/block_producer/mod.rs @@ -2,10 +2,9 @@ mod grpc; mod service; #[cfg(test)] mod test_node; -mod traits; +pub mod traits; -use self::traits::{BeaconNode, BeaconNodeError}; -use super::EpochDutiesMap; +use self::traits::{BeaconNode, BeaconNodeError, DutiesReader, DutiesReaderError}; use slot_clock::SlotClock; use spec::ChainSpec; use std::sync::{Arc, RwLock}; @@ -45,19 +44,19 @@ pub enum Error { /// Ensures that messages are not slashable. /// /// Relies upon an external service to keep the `EpochDutiesMap` updated. -pub struct BlockProducer { +pub struct BlockProducer { pub last_processed_slot: u64, spec: Arc, - epoch_map: Arc>, + epoch_map: Arc, slot_clock: Arc>, beacon_node: Arc, } -impl BlockProducer { +impl BlockProducer { /// Returns a new instance where `last_processed_slot == 0`. pub fn new( spec: Arc, - epoch_map: Arc>, + epoch_map: Arc, slot_clock: Arc>, beacon_node: Arc, ) -> Self { @@ -71,7 +70,7 @@ impl BlockProducer { } } -impl BlockProducer { +impl BlockProducer { /// "Poll" to see if the validator is required to take any action. /// /// The slot clock will be read and any new actions undertaken. @@ -90,13 +89,14 @@ impl BlockProducer { // If this is a new slot. if slot > self.last_processed_slot { - let is_block_production_slot = { - let epoch_map = self.epoch_map.read().map_err(|_| Error::EpochMapPoisoned)?; - match epoch_map.get(&epoch) { - None => return Ok(PollOutcome::ProducerDutiesUnknown(slot)), - Some(duties) => duties.is_block_production_slot(slot), - } - }; + let is_block_production_slot = + match self.epoch_map.is_block_production_slot(epoch, slot) { + Ok(result) => result, + Err(DutiesReaderError::UnknownEpoch) => { + return Ok(PollOutcome::ProducerDutiesUnknown(slot)) + } + Err(DutiesReaderError::Poisoned) => return Err(Error::EpochMapPoisoned), + }; if is_block_production_slot { self.last_processed_slot = slot; @@ -178,6 +178,7 @@ mod tests { use super::test_node::TestBeaconNode; use super::*; use crate::duties::EpochDuties; + use crate::duties::EpochDutiesMap; use slot_clock::TestingSlotClock; use types::test_utils::{SeedableRng, TestRandom, XorShiftRng}; @@ -191,7 +192,7 @@ mod tests { let mut rng = XorShiftRng::from_seed([42; 16]); let spec = Arc::new(ChainSpec::foundation()); - let epoch_map = Arc::new(RwLock::new(EpochDutiesMap::new())); + let epoch_map = Arc::new(EpochDutiesMap::new()); let slot_clock = Arc::new(RwLock::new(TestingSlotClock::new(0))); let beacon_node = Arc::new(TestBeaconNode::default()); @@ -213,7 +214,7 @@ mod tests { ..std::default::Default::default() }; let produce_epoch = produce_slot / spec.epoch_length; - epoch_map.write().unwrap().insert(produce_epoch, duties); + epoch_map.insert(produce_epoch, duties); // One slot before production slot... slot_clock.write().unwrap().set_slot(produce_slot - 1); diff --git a/validator_client/src/block_producer/service.rs b/validator_client/src/block_producer/service.rs index ffdb33029..822df76c1 100644 --- a/validator_client/src/block_producer/service.rs +++ b/validator_client/src/block_producer/service.rs @@ -1,15 +1,15 @@ -use super::traits::BeaconNode; +use super::traits::{BeaconNode, DutiesReader}; use super::{BlockProducer, PollOutcome as BlockProducerPollOutcome, SlotClock}; use slog::{error, info, warn, Logger}; use std::time::Duration; -pub struct BlockProducerService { - pub block_producer: BlockProducer, +pub struct BlockProducerService { + pub block_producer: BlockProducer, pub poll_interval_millis: u64, pub log: Logger, } -impl BlockProducerService { +impl BlockProducerService { /// Run a loop which polls the block producer each `poll_interval_millis` millseconds. /// /// Logs the results of the polls. diff --git a/validator_client/src/block_producer/traits.rs b/validator_client/src/block_producer/traits.rs index be1c73bda..e16af2460 100644 --- a/validator_client/src/block_producer/traits.rs +++ b/validator_client/src/block_producer/traits.rs @@ -17,3 +17,12 @@ pub trait BeaconNode: Send + Sync { /// Returns `true` if the publish was sucessful. fn publish_beacon_block(&self, block: BeaconBlock) -> Result; } + +pub enum DutiesReaderError { + UnknownEpoch, + Poisoned, +} + +pub trait DutiesReader: Send + Sync { + fn is_block_production_slot(&self, epoch: u64, slot: u64) -> Result; +} diff --git a/validator_client/src/duties/mod.rs b/validator_client/src/duties/mod.rs index 4656715ba..8e7019533 100644 --- a/validator_client/src/duties/mod.rs +++ b/validator_client/src/duties/mod.rs @@ -5,6 +5,7 @@ mod test_node; mod traits; use self::traits::{BeaconNode, BeaconNodeError}; +use super::block_producer::traits::{DutiesReader, DutiesReaderError}; use bls::PublicKey; use slot_clock::SlotClock; use spec::ChainSpec; @@ -36,8 +37,52 @@ impl EpochDuties { } } +pub enum EpochDutiesMapError { + Poisoned, +} + /// Maps an `epoch` to some `EpochDuties` for a single validator. -pub type EpochDutiesMap = HashMap; +pub struct EpochDutiesMap { + pub map: RwLock>, +} + +impl EpochDutiesMap { + pub fn new() -> Self { + Self { + map: RwLock::new(HashMap::new()), + } + } + + pub fn get(&self, epoch: u64) -> Result, EpochDutiesMapError> { + let map = self.map.read().map_err(|_| EpochDutiesMapError::Poisoned)?; + match map.get(&epoch) { + Some(duties) => Ok(Some(duties.clone())), + None => Ok(None), + } + } + + pub fn insert( + &self, + epoch: u64, + epoch_duties: EpochDuties, + ) -> Result, EpochDutiesMapError> { + let mut map = self + .map + .write() + .map_err(|_| EpochDutiesMapError::Poisoned)?; + Ok(map.insert(epoch, epoch_duties)) + } +} + +impl DutiesReader for EpochDutiesMap { + fn is_block_production_slot(&self, epoch: u64, slot: u64) -> Result { + let map = self.map.read().map_err(|_| DutiesReaderError::Poisoned)?; + let duties = map + .get(&epoch) + .ok_or_else(|| DutiesReaderError::UnknownEpoch)?; + Ok(duties.is_block_production_slot(slot)) + } +} #[derive(Debug, PartialEq, Clone, Copy)] pub enum PollOutcome { @@ -68,7 +113,7 @@ pub enum Error { /// /// There is a single `DutiesManager` per validator instance. pub struct DutiesManager { - pub duties_map: Arc>, + pub duties_map: Arc, /// The validator's public key. pub pubkey: PublicKey, pub spec: Arc, @@ -95,14 +140,9 @@ impl DutiesManager { .ok_or(Error::EpochLengthIsZero)?; if let Some(duties) = self.beacon_node.request_shuffling(epoch, &self.pubkey)? { - let mut map = self - .duties_map - .write() - .map_err(|_| Error::EpochMapPoisoned)?; - // If these duties were known, check to see if they're updates or identical. - let result = if let Some(known_duties) = map.get(&epoch) { - if *known_duties == duties { + let result = if let Some(known_duties) = self.duties_map.get(epoch)? { + if known_duties == duties { Ok(PollOutcome::NoChange(epoch)) } else { Ok(PollOutcome::DutiesChanged(epoch, duties)) @@ -110,7 +150,7 @@ impl DutiesManager { } else { Ok(PollOutcome::NewDuties(epoch, duties)) }; - map.insert(epoch, duties); + self.duties_map.insert(epoch, duties)?; result } else { Ok(PollOutcome::UnknownValidatorOrEpoch(epoch)) @@ -124,6 +164,14 @@ impl From for Error { } } +impl From for Error { + fn from(e: EpochDutiesMapError) -> Error { + match e { + EpochDutiesMapError::Poisoned => Error::EpochMapPoisoned, + } + } +} + #[cfg(test)] mod tests { use super::test_node::TestBeaconNode; @@ -139,7 +187,7 @@ mod tests { #[test] pub fn polling() { let spec = Arc::new(ChainSpec::foundation()); - let duties_map = Arc::new(RwLock::new(EpochDutiesMap::new())); + let duties_map = Arc::new(EpochDutiesMap::new()); let keypair = Keypair::random(); let slot_clock = Arc::new(RwLock::new(TestingSlotClock::new(0))); let beacon_node = Arc::new(TestBeaconNode::default()); diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index bbbc0b4c3..c1775f826 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -107,7 +107,7 @@ fn main() { for keypair in keypairs { info!(log, "Starting validator services"; "validator" => keypair.pk.concatenated_hex_id()); - let duties_map = Arc::new(RwLock::new(EpochDutiesMap::new())); + let duties_map = Arc::new(EpochDutiesMap::new()); // Spawn a new thread to maintain the validator's `EpochDuties`. let duties_manager_thread = { From 403eefb7b494a8914a073978723e3ea7c24b18d5 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 23 Jan 2019 21:21:18 +1100 Subject: [PATCH 006/107] Move block_proposer into separate crate --- Cargo.toml | 1 + validator_client/block_proposer/Cargo.toml | 10 + validator_client/block_proposer/src/lib.rs | 263 ++++++++++++++++++ .../block_proposer/src/test_node.rs | 47 ++++ validator_client/block_proposer/src/traits.rs | 28 ++ 5 files changed, 349 insertions(+) create mode 100644 validator_client/block_proposer/Cargo.toml create mode 100644 validator_client/block_proposer/src/lib.rs create mode 100644 validator_client/block_proposer/src/test_node.rs create mode 100644 validator_client/block_proposer/src/traits.rs diff --git a/Cargo.toml b/Cargo.toml index bcaaef87e..fdedbf6bb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,4 +19,5 @@ members = [ "beacon_node/beacon_chain", "protos", "validator_client", + "validator_client/block_proposer", ] diff --git a/validator_client/block_proposer/Cargo.toml b/validator_client/block_proposer/Cargo.toml new file mode 100644 index 000000000..460ca863e --- /dev/null +++ b/validator_client/block_proposer/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "block_proposer" +version = "0.1.0" +authors = ["Paul Hauner "] +edition = "2018" + +[dependencies] +slot_clock = { path = "../../eth2/utils/slot_clock" } +spec = { path = "../../eth2/spec" } +types = { path = "../../eth2/types" } diff --git a/validator_client/block_proposer/src/lib.rs b/validator_client/block_proposer/src/lib.rs new file mode 100644 index 000000000..112684dbb --- /dev/null +++ b/validator_client/block_proposer/src/lib.rs @@ -0,0 +1,263 @@ +#[cfg(test)] +mod test_node; +mod traits; + +use slot_clock::SlotClock; +use spec::ChainSpec; +use std::sync::{Arc, RwLock}; +use types::BeaconBlock; + +pub use self::traits::{BeaconNode, BeaconNodeError, DutiesReader, DutiesReaderError}; + +#[derive(Debug, PartialEq)] +pub enum PollOutcome { + /// A new block was produced. + BlockProduced(u64), + /// A block was not produced as it would have been slashable. + SlashableBlockNotProduced(u64), + /// The validator duties did not require a block to be produced. + BlockProductionNotRequired(u64), + /// The duties for the present epoch were not found. + ProducerDutiesUnknown(u64), + /// The slot has already been processed, execution was skipped. + SlotAlreadyProcessed(u64), + /// The Beacon Node was unable to produce a block at that slot. + BeaconNodeUnableToProduceBlock(u64), +} + +#[derive(Debug, PartialEq)] +pub enum Error { + SlotClockError, + SlotUnknowable, + EpochMapPoisoned, + SlotClockPoisoned, + EpochLengthIsZero, + BeaconNodeError(BeaconNodeError), +} + +/// A polling state machine which performs block production duties, based upon some epoch duties +/// (`EpochDutiesMap`) and a concept of time (`SlotClock`). +/// +/// Ensures that messages are not slashable. +/// +/// Relies upon an external service to keep the `EpochDutiesMap` updated. +pub struct BlockProducer { + pub last_processed_slot: u64, + spec: Arc, + epoch_map: Arc, + slot_clock: Arc>, + beacon_node: Arc, +} + +impl BlockProducer { + /// Returns a new instance where `last_processed_slot == 0`. + pub fn new( + spec: Arc, + epoch_map: Arc, + slot_clock: Arc>, + beacon_node: Arc, + ) -> Self { + Self { + last_processed_slot: 0, + spec, + epoch_map, + slot_clock, + beacon_node, + } + } +} + +impl BlockProducer { + /// "Poll" to see if the validator is required to take any action. + /// + /// The slot clock will be read and any new actions undertaken. + pub fn poll(&mut self) -> Result { + let slot = self + .slot_clock + .read() + .map_err(|_| Error::SlotClockPoisoned)? + .present_slot() + .map_err(|_| Error::SlotClockError)? + .ok_or(Error::SlotUnknowable)?; + + let epoch = slot + .checked_div(self.spec.epoch_length) + .ok_or(Error::EpochLengthIsZero)?; + + // If this is a new slot. + if slot > self.last_processed_slot { + let is_block_production_slot = + match self.epoch_map.is_block_production_slot(epoch, slot) { + Ok(result) => result, + Err(DutiesReaderError::UnknownEpoch) => { + return Ok(PollOutcome::ProducerDutiesUnknown(slot)) + } + Err(DutiesReaderError::Poisoned) => return Err(Error::EpochMapPoisoned), + }; + + if is_block_production_slot { + self.last_processed_slot = slot; + + self.produce_block(slot) + } else { + Ok(PollOutcome::BlockProductionNotRequired(slot)) + } + } else { + Ok(PollOutcome::SlotAlreadyProcessed(slot)) + } + } + + /// Produce a block at some slot. + /// + /// Assumes that a block is required at this slot (does not check the duties). + /// + /// Ensures the message is not slashable. + /// + /// !!! UNSAFE !!! + /// + /// The slash-protection code is not yet implemented. There is zero protection against + /// slashing. + fn produce_block(&mut self, slot: u64) -> Result { + if let Some(block) = self.beacon_node.produce_beacon_block(slot)? { + if self.safe_to_produce(&block) { + let block = self.sign_block(block); + self.beacon_node.publish_beacon_block(block)?; + Ok(PollOutcome::BlockProduced(slot)) + } else { + Ok(PollOutcome::SlashableBlockNotProduced(slot)) + } + } else { + Ok(PollOutcome::BeaconNodeUnableToProduceBlock(slot)) + } + } + + /// Consumes a block, returning that block signed by the validators private key. + /// + /// Important: this function will not check to ensure the block is not slashable. This must be + /// done upstream. + fn sign_block(&mut self, block: BeaconBlock) -> BeaconBlock { + // TODO: sign the block + // https://github.com/sigp/lighthouse/issues/160 + self.store_produce(&block); + block + } + + /// Returns `true` if signing a block is safe (non-slashable). + /// + /// !!! UNSAFE !!! + /// + /// Important: this function is presently stubbed-out. It provides ZERO SAFETY. + fn safe_to_produce(&self, _block: &BeaconBlock) -> bool { + // TODO: ensure the producer doesn't produce slashable blocks. + // https://github.com/sigp/lighthouse/issues/160 + true + } + + /// Record that a block was produced so that slashable votes may not be made in the future. + /// + /// !!! UNSAFE !!! + /// + /// Important: this function is presently stubbed-out. It provides ZERO SAFETY. + fn store_produce(&mut self, _block: &BeaconBlock) { + // TODO: record this block production to prevent future slashings. + // https://github.com/sigp/lighthouse/issues/160 + } +} + +impl From for Error { + fn from(e: BeaconNodeError) -> Error { + Error::BeaconNodeError(e) + } +} + +#[cfg(test)] +mod tests { + use super::test_node::TestBeaconNode; + use super::*; + use slot_clock::TestingSlotClock; + use std::collections::HashMap; + use types::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + + // TODO: implement more thorough testing. + // https://github.com/sigp/lighthouse/issues/160 + // + // These tests should serve as a good example for future tests. + + type EpochMap = HashMap; + + impl DutiesReader for EpochMap { + fn is_block_production_slot( + &self, + epoch: u64, + slot: u64, + ) -> Result { + match self.get(&epoch) { + Some(s) if *s == slot => Ok(true), + Some(s) if *s != slot => Ok(false), + _ => Err(DutiesReaderError::UnknownEpoch), + } + } + } + + #[test] + pub fn polling() { + let mut rng = XorShiftRng::from_seed([42; 16]); + + let spec = Arc::new(ChainSpec::foundation()); + let slot_clock = Arc::new(RwLock::new(TestingSlotClock::new(0))); + let beacon_node = Arc::new(TestBeaconNode::default()); + + let mut epoch_map = EpochMap::new(); + let produce_slot = 100; + let produce_epoch = produce_slot / spec.epoch_length; + epoch_map.insert(produce_epoch, produce_slot); + let epoch_map = Arc::new(epoch_map); + + let mut block_producer = BlockProducer::new( + spec.clone(), + epoch_map.clone(), + slot_clock.clone(), + beacon_node.clone(), + ); + + // Configure responses from the BeaconNode. + beacon_node.set_next_produce_result(Ok(Some(BeaconBlock::random_for_test(&mut rng)))); + beacon_node.set_next_publish_result(Ok(true)); + + // One slot before production slot... + slot_clock.write().unwrap().set_slot(produce_slot - 1); + assert_eq!( + block_producer.poll(), + Ok(PollOutcome::BlockProductionNotRequired(produce_slot - 1)) + ); + + // On the produce slot... + slot_clock.write().unwrap().set_slot(produce_slot); + assert_eq!( + block_producer.poll(), + Ok(PollOutcome::BlockProduced(produce_slot)) + ); + + // Trying the same produce slot again... + slot_clock.write().unwrap().set_slot(produce_slot); + assert_eq!( + block_producer.poll(), + Ok(PollOutcome::SlotAlreadyProcessed(produce_slot)) + ); + + // One slot after the produce slot... + slot_clock.write().unwrap().set_slot(produce_slot + 1); + assert_eq!( + block_producer.poll(), + Ok(PollOutcome::BlockProductionNotRequired(produce_slot + 1)) + ); + + // In an epoch without known duties... + let slot = (produce_epoch + 1) * spec.epoch_length; + slot_clock.write().unwrap().set_slot(slot); + assert_eq!( + block_producer.poll(), + Ok(PollOutcome::ProducerDutiesUnknown(slot)) + ); + } +} diff --git a/validator_client/block_proposer/src/test_node.rs b/validator_client/block_proposer/src/test_node.rs new file mode 100644 index 000000000..e99613e8f --- /dev/null +++ b/validator_client/block_proposer/src/test_node.rs @@ -0,0 +1,47 @@ +use super::traits::{BeaconNode, BeaconNodeError}; +use std::sync::RwLock; +use types::BeaconBlock; + +type ProduceResult = Result, BeaconNodeError>; +type PublishResult = Result; + +/// A test-only struct used to simulate a Beacon Node. +#[derive(Default)] +pub struct TestBeaconNode { + pub produce_input: RwLock>, + pub produce_result: RwLock>, + pub publish_input: RwLock>, + pub publish_result: RwLock>, +} + +impl TestBeaconNode { + /// Set the result to be returned when `produce_beacon_block` is called. + pub fn set_next_produce_result(&self, result: ProduceResult) { + *self.produce_result.write().unwrap() = Some(result); + } + + /// Set the result to be returned when `publish_beacon_block` is called. + pub fn set_next_publish_result(&self, result: PublishResult) { + *self.publish_result.write().unwrap() = Some(result); + } +} + +impl BeaconNode for TestBeaconNode { + /// Returns the value specified by the `set_next_produce_result`. + fn produce_beacon_block(&self, slot: u64) -> ProduceResult { + *self.produce_input.write().unwrap() = Some(slot); + match *self.produce_result.read().unwrap() { + Some(ref r) => r.clone(), + None => panic!("TestBeaconNode: produce_result == None"), + } + } + + /// Returns the value specified by the `set_next_publish_result`. + fn publish_beacon_block(&self, block: BeaconBlock) -> PublishResult { + *self.publish_input.write().unwrap() = Some(block); + match *self.publish_result.read().unwrap() { + Some(ref r) => r.clone(), + None => panic!("TestBeaconNode: publish_result == None"), + } + } +} diff --git a/validator_client/block_proposer/src/traits.rs b/validator_client/block_proposer/src/traits.rs new file mode 100644 index 000000000..e16af2460 --- /dev/null +++ b/validator_client/block_proposer/src/traits.rs @@ -0,0 +1,28 @@ +use types::BeaconBlock; + +#[derive(Debug, PartialEq, Clone)] +pub enum BeaconNodeError { + RemoteFailure(String), + DecodeFailure, +} + +/// Defines the methods required to produce and publish blocks on a Beacon Node. +pub trait BeaconNode: Send + Sync { + /// Request that the node produces a block. + /// + /// Returns Ok(None) if the Beacon Node is unable to produce at the given slot. + fn produce_beacon_block(&self, slot: u64) -> Result, BeaconNodeError>; + /// Request that the node publishes a block. + /// + /// Returns `true` if the publish was sucessful. + fn publish_beacon_block(&self, block: BeaconBlock) -> Result; +} + +pub enum DutiesReaderError { + UnknownEpoch, + Poisoned, +} + +pub trait DutiesReader: Send + Sync { + fn is_block_production_slot(&self, epoch: u64, slot: u64) -> Result; +} From 188434aaa026e06ac9ebff8d6d51dbd346739295 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 23 Jan 2019 21:41:47 +1100 Subject: [PATCH 007/107] Move block_propser -> eth/, impl for val client --- Cargo.toml | 2 +- .../block_producer}/Cargo.toml | 2 +- .../block_producer}/src/lib.rs | 0 .../block_producer}/src/test_node.rs | 0 .../block_producer}/src/traits.rs | 0 validator_client/Cargo.toml | 1 + validator_client/src/block_producer/mod.rs | 255 ------------------ .../src/block_producer/test_node.rs | 47 ---- validator_client/src/block_producer/traits.rs | 28 -- .../grpc.rs | 19 +- .../src/block_producer_service/mod.rs | 5 + .../service.rs | 6 +- validator_client/src/duties/mod.rs | 2 +- validator_client/src/main.rs | 7 +- 14 files changed, 34 insertions(+), 340 deletions(-) rename {validator_client/block_proposer => eth2/block_producer}/Cargo.toml (90%) rename {validator_client/block_proposer => eth2/block_producer}/src/lib.rs (100%) rename {validator_client/block_proposer => eth2/block_producer}/src/test_node.rs (100%) rename {validator_client/block_proposer => eth2/block_producer}/src/traits.rs (100%) delete mode 100644 validator_client/src/block_producer/mod.rs delete mode 100644 validator_client/src/block_producer/test_node.rs delete mode 100644 validator_client/src/block_producer/traits.rs rename validator_client/src/{block_producer => block_producer_service}/grpc.rs (86%) create mode 100644 validator_client/src/block_producer_service/mod.rs rename validator_client/src/{block_producer => block_producer_service}/service.rs (93%) diff --git a/Cargo.toml b/Cargo.toml index fdedbf6bb..26f7d293a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "eth2/attestation_validation", + "eth2/block_producer", "eth2/genesis", "eth2/naive_fork_choice", "eth2/spec", @@ -19,5 +20,4 @@ members = [ "beacon_node/beacon_chain", "protos", "validator_client", - "validator_client/block_proposer", ] diff --git a/validator_client/block_proposer/Cargo.toml b/eth2/block_producer/Cargo.toml similarity index 90% rename from validator_client/block_proposer/Cargo.toml rename to eth2/block_producer/Cargo.toml index 460ca863e..76ea78ea3 100644 --- a/validator_client/block_proposer/Cargo.toml +++ b/eth2/block_producer/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "block_proposer" +name = "block_producer" version = "0.1.0" authors = ["Paul Hauner "] edition = "2018" diff --git a/validator_client/block_proposer/src/lib.rs b/eth2/block_producer/src/lib.rs similarity index 100% rename from validator_client/block_proposer/src/lib.rs rename to eth2/block_producer/src/lib.rs diff --git a/validator_client/block_proposer/src/test_node.rs b/eth2/block_producer/src/test_node.rs similarity index 100% rename from validator_client/block_proposer/src/test_node.rs rename to eth2/block_producer/src/test_node.rs diff --git a/validator_client/block_proposer/src/traits.rs b/eth2/block_producer/src/traits.rs similarity index 100% rename from validator_client/block_proposer/src/traits.rs rename to eth2/block_producer/src/traits.rs diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index c23e6607a..c4f8b8f4a 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -5,6 +5,7 @@ authors = ["Paul Hauner "] edition = "2018" [dependencies] +block_producer = { path = "../eth2/block_producer" } bls = { path = "../eth2/utils/bls" } clap = "2.32.0" dirs = "1.0.3" diff --git a/validator_client/src/block_producer/mod.rs b/validator_client/src/block_producer/mod.rs deleted file mode 100644 index e0ea220b6..000000000 --- a/validator_client/src/block_producer/mod.rs +++ /dev/null @@ -1,255 +0,0 @@ -mod grpc; -mod service; -#[cfg(test)] -mod test_node; -pub mod traits; - -use self::traits::{BeaconNode, BeaconNodeError, DutiesReader, DutiesReaderError}; -use slot_clock::SlotClock; -use spec::ChainSpec; -use std::sync::{Arc, RwLock}; -use types::BeaconBlock; - -pub use self::service::BlockProducerService; - -#[derive(Debug, PartialEq)] -pub enum PollOutcome { - /// A new block was produced. - BlockProduced(u64), - /// A block was not produced as it would have been slashable. - SlashableBlockNotProduced(u64), - /// The validator duties did not require a block to be produced. - BlockProductionNotRequired(u64), - /// The duties for the present epoch were not found. - ProducerDutiesUnknown(u64), - /// The slot has already been processed, execution was skipped. - SlotAlreadyProcessed(u64), - /// The Beacon Node was unable to produce a block at that slot. - BeaconNodeUnableToProduceBlock(u64), -} - -#[derive(Debug, PartialEq)] -pub enum Error { - SlotClockError, - SlotUnknowable, - EpochMapPoisoned, - SlotClockPoisoned, - EpochLengthIsZero, - BeaconNodeError(BeaconNodeError), -} - -/// A polling state machine which performs block production duties, based upon some epoch duties -/// (`EpochDutiesMap`) and a concept of time (`SlotClock`). -/// -/// Ensures that messages are not slashable. -/// -/// Relies upon an external service to keep the `EpochDutiesMap` updated. -pub struct BlockProducer { - pub last_processed_slot: u64, - spec: Arc, - epoch_map: Arc, - slot_clock: Arc>, - beacon_node: Arc, -} - -impl BlockProducer { - /// Returns a new instance where `last_processed_slot == 0`. - pub fn new( - spec: Arc, - epoch_map: Arc, - slot_clock: Arc>, - beacon_node: Arc, - ) -> Self { - Self { - last_processed_slot: 0, - spec, - epoch_map, - slot_clock, - beacon_node, - } - } -} - -impl BlockProducer { - /// "Poll" to see if the validator is required to take any action. - /// - /// The slot clock will be read and any new actions undertaken. - pub fn poll(&mut self) -> Result { - let slot = self - .slot_clock - .read() - .map_err(|_| Error::SlotClockPoisoned)? - .present_slot() - .map_err(|_| Error::SlotClockError)? - .ok_or(Error::SlotUnknowable)?; - - let epoch = slot - .checked_div(self.spec.epoch_length) - .ok_or(Error::EpochLengthIsZero)?; - - // If this is a new slot. - if slot > self.last_processed_slot { - let is_block_production_slot = - match self.epoch_map.is_block_production_slot(epoch, slot) { - Ok(result) => result, - Err(DutiesReaderError::UnknownEpoch) => { - return Ok(PollOutcome::ProducerDutiesUnknown(slot)) - } - Err(DutiesReaderError::Poisoned) => return Err(Error::EpochMapPoisoned), - }; - - if is_block_production_slot { - self.last_processed_slot = slot; - - self.produce_block(slot) - } else { - Ok(PollOutcome::BlockProductionNotRequired(slot)) - } - } else { - Ok(PollOutcome::SlotAlreadyProcessed(slot)) - } - } - - /// Produce a block at some slot. - /// - /// Assumes that a block is required at this slot (does not check the duties). - /// - /// Ensures the message is not slashable. - /// - /// !!! UNSAFE !!! - /// - /// The slash-protection code is not yet implemented. There is zero protection against - /// slashing. - fn produce_block(&mut self, slot: u64) -> Result { - if let Some(block) = self.beacon_node.produce_beacon_block(slot)? { - if self.safe_to_produce(&block) { - let block = self.sign_block(block); - self.beacon_node.publish_beacon_block(block)?; - Ok(PollOutcome::BlockProduced(slot)) - } else { - Ok(PollOutcome::SlashableBlockNotProduced(slot)) - } - } else { - Ok(PollOutcome::BeaconNodeUnableToProduceBlock(slot)) - } - } - - /// Consumes a block, returning that block signed by the validators private key. - /// - /// Important: this function will not check to ensure the block is not slashable. This must be - /// done upstream. - fn sign_block(&mut self, block: BeaconBlock) -> BeaconBlock { - // TODO: sign the block - // https://github.com/sigp/lighthouse/issues/160 - self.store_produce(&block); - block - } - - /// Returns `true` if signing a block is safe (non-slashable). - /// - /// !!! UNSAFE !!! - /// - /// Important: this function is presently stubbed-out. It provides ZERO SAFETY. - fn safe_to_produce(&self, _block: &BeaconBlock) -> bool { - // TODO: ensure the producer doesn't produce slashable blocks. - // https://github.com/sigp/lighthouse/issues/160 - true - } - - /// Record that a block was produced so that slashable votes may not be made in the future. - /// - /// !!! UNSAFE !!! - /// - /// Important: this function is presently stubbed-out. It provides ZERO SAFETY. - fn store_produce(&mut self, _block: &BeaconBlock) { - // TODO: record this block production to prevent future slashings. - // https://github.com/sigp/lighthouse/issues/160 - } -} - -impl From for Error { - fn from(e: BeaconNodeError) -> Error { - Error::BeaconNodeError(e) - } -} - -#[cfg(test)] -mod tests { - use super::test_node::TestBeaconNode; - use super::*; - use crate::duties::EpochDuties; - use crate::duties::EpochDutiesMap; - use slot_clock::TestingSlotClock; - use types::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - - // TODO: implement more thorough testing. - // https://github.com/sigp/lighthouse/issues/160 - // - // These tests should serve as a good example for future tests. - - #[test] - pub fn polling() { - let mut rng = XorShiftRng::from_seed([42; 16]); - - let spec = Arc::new(ChainSpec::foundation()); - let epoch_map = Arc::new(EpochDutiesMap::new()); - let slot_clock = Arc::new(RwLock::new(TestingSlotClock::new(0))); - let beacon_node = Arc::new(TestBeaconNode::default()); - - let mut block_producer = BlockProducer::new( - spec.clone(), - epoch_map.clone(), - slot_clock.clone(), - beacon_node.clone(), - ); - - // Configure responses from the BeaconNode. - beacon_node.set_next_produce_result(Ok(Some(BeaconBlock::random_for_test(&mut rng)))); - beacon_node.set_next_publish_result(Ok(true)); - - // Setup some valid duties for the validator - let produce_slot = 100; - let duties = EpochDuties { - block_production_slot: Some(produce_slot), - ..std::default::Default::default() - }; - let produce_epoch = produce_slot / spec.epoch_length; - epoch_map.insert(produce_epoch, duties); - - // One slot before production slot... - slot_clock.write().unwrap().set_slot(produce_slot - 1); - assert_eq!( - block_producer.poll(), - Ok(PollOutcome::BlockProductionNotRequired(produce_slot - 1)) - ); - - // On the produce slot... - slot_clock.write().unwrap().set_slot(produce_slot); - assert_eq!( - block_producer.poll(), - Ok(PollOutcome::BlockProduced(produce_slot)) - ); - - // Trying the same produce slot again... - slot_clock.write().unwrap().set_slot(produce_slot); - assert_eq!( - block_producer.poll(), - Ok(PollOutcome::SlotAlreadyProcessed(produce_slot)) - ); - - // One slot after the produce slot... - slot_clock.write().unwrap().set_slot(produce_slot + 1); - assert_eq!( - block_producer.poll(), - Ok(PollOutcome::BlockProductionNotRequired(produce_slot + 1)) - ); - - // In an epoch without known duties... - let slot = (produce_epoch + 1) * spec.epoch_length; - slot_clock.write().unwrap().set_slot(slot); - assert_eq!( - block_producer.poll(), - Ok(PollOutcome::ProducerDutiesUnknown(slot)) - ); - } -} diff --git a/validator_client/src/block_producer/test_node.rs b/validator_client/src/block_producer/test_node.rs deleted file mode 100644 index e99613e8f..000000000 --- a/validator_client/src/block_producer/test_node.rs +++ /dev/null @@ -1,47 +0,0 @@ -use super::traits::{BeaconNode, BeaconNodeError}; -use std::sync::RwLock; -use types::BeaconBlock; - -type ProduceResult = Result, BeaconNodeError>; -type PublishResult = Result; - -/// A test-only struct used to simulate a Beacon Node. -#[derive(Default)] -pub struct TestBeaconNode { - pub produce_input: RwLock>, - pub produce_result: RwLock>, - pub publish_input: RwLock>, - pub publish_result: RwLock>, -} - -impl TestBeaconNode { - /// Set the result to be returned when `produce_beacon_block` is called. - pub fn set_next_produce_result(&self, result: ProduceResult) { - *self.produce_result.write().unwrap() = Some(result); - } - - /// Set the result to be returned when `publish_beacon_block` is called. - pub fn set_next_publish_result(&self, result: PublishResult) { - *self.publish_result.write().unwrap() = Some(result); - } -} - -impl BeaconNode for TestBeaconNode { - /// Returns the value specified by the `set_next_produce_result`. - fn produce_beacon_block(&self, slot: u64) -> ProduceResult { - *self.produce_input.write().unwrap() = Some(slot); - match *self.produce_result.read().unwrap() { - Some(ref r) => r.clone(), - None => panic!("TestBeaconNode: produce_result == None"), - } - } - - /// Returns the value specified by the `set_next_publish_result`. - fn publish_beacon_block(&self, block: BeaconBlock) -> PublishResult { - *self.publish_input.write().unwrap() = Some(block); - match *self.publish_result.read().unwrap() { - Some(ref r) => r.clone(), - None => panic!("TestBeaconNode: publish_result == None"), - } - } -} diff --git a/validator_client/src/block_producer/traits.rs b/validator_client/src/block_producer/traits.rs deleted file mode 100644 index e16af2460..000000000 --- a/validator_client/src/block_producer/traits.rs +++ /dev/null @@ -1,28 +0,0 @@ -use types::BeaconBlock; - -#[derive(Debug, PartialEq, Clone)] -pub enum BeaconNodeError { - RemoteFailure(String), - DecodeFailure, -} - -/// Defines the methods required to produce and publish blocks on a Beacon Node. -pub trait BeaconNode: Send + Sync { - /// Request that the node produces a block. - /// - /// Returns Ok(None) if the Beacon Node is unable to produce at the given slot. - fn produce_beacon_block(&self, slot: u64) -> Result, BeaconNodeError>; - /// Request that the node publishes a block. - /// - /// Returns `true` if the publish was sucessful. - fn publish_beacon_block(&self, block: BeaconBlock) -> Result; -} - -pub enum DutiesReaderError { - UnknownEpoch, - Poisoned, -} - -pub trait DutiesReader: Send + Sync { - fn is_block_production_slot(&self, epoch: u64, slot: u64) -> Result; -} diff --git a/validator_client/src/block_producer/grpc.rs b/validator_client/src/block_producer_service/grpc.rs similarity index 86% rename from validator_client/src/block_producer/grpc.rs rename to validator_client/src/block_producer_service/grpc.rs index 20ced3d0c..b3bc8f5cc 100644 --- a/validator_client/src/block_producer/grpc.rs +++ b/validator_client/src/block_producer_service/grpc.rs @@ -1,12 +1,25 @@ -use super::traits::{BeaconNode, BeaconNodeError}; +use block_producer::{BeaconNode, BeaconNodeError}; use protos::services::{ BeaconBlock as GrpcBeaconBlock, ProduceBeaconBlockRequest, PublishBeaconBlockRequest, }; use protos::services_grpc::BeaconBlockServiceClient; use ssz::{ssz_encode, Decodable}; +use std::sync::Arc; use types::{BeaconBlock, BeaconBlockBody, Hash256, Signature}; -impl BeaconNode for BeaconBlockServiceClient { +/// A newtype designed to wrap the gRPC-generated service so the `BeaconNode` trait may be +/// implemented upon it. +pub struct BeaconBlockGrpcClient { + client: Arc, +} + +impl BeaconBlockGrpcClient { + pub fn new(client: Arc) -> Self { + Self { client } + } +} + +impl BeaconNode for BeaconBlockGrpcClient { /// Request a Beacon Node (BN) to produce a new block at the supplied slot. /// /// Returns `None` if it is not possible to produce at the supplied slot. For example, if the @@ -16,6 +29,7 @@ impl BeaconNode for BeaconBlockServiceClient { req.set_slot(slot); let reply = self + .client .produce_beacon_block(&req) .map_err(|err| BeaconNodeError::RemoteFailure(format!("{:?}", err)))?; @@ -69,6 +83,7 @@ impl BeaconNode for BeaconBlockServiceClient { req.set_block(grpc_block); let reply = self + .client .publish_beacon_block(&req) .map_err(|err| BeaconNodeError::RemoteFailure(format!("{:?}", err)))?; diff --git a/validator_client/src/block_producer_service/mod.rs b/validator_client/src/block_producer_service/mod.rs new file mode 100644 index 000000000..a4ddf760a --- /dev/null +++ b/validator_client/src/block_producer_service/mod.rs @@ -0,0 +1,5 @@ +mod grpc; +mod service; + +pub use self::grpc::BeaconBlockGrpcClient; +pub use self::service::BlockProducerService; diff --git a/validator_client/src/block_producer/service.rs b/validator_client/src/block_producer_service/service.rs similarity index 93% rename from validator_client/src/block_producer/service.rs rename to validator_client/src/block_producer_service/service.rs index 822df76c1..6652058c8 100644 --- a/validator_client/src/block_producer/service.rs +++ b/validator_client/src/block_producer_service/service.rs @@ -1,6 +1,8 @@ -use super::traits::{BeaconNode, DutiesReader}; -use super::{BlockProducer, PollOutcome as BlockProducerPollOutcome, SlotClock}; +use block_producer::{ + BeaconNode, BlockProducer, DutiesReader, PollOutcome as BlockProducerPollOutcome, +}; use slog::{error, info, warn, Logger}; +use slot_clock::SlotClock; use std::time::Duration; pub struct BlockProducerService { diff --git a/validator_client/src/duties/mod.rs b/validator_client/src/duties/mod.rs index 8e7019533..8484ad80c 100644 --- a/validator_client/src/duties/mod.rs +++ b/validator_client/src/duties/mod.rs @@ -5,7 +5,7 @@ mod test_node; mod traits; use self::traits::{BeaconNode, BeaconNodeError}; -use super::block_producer::traits::{DutiesReader, DutiesReaderError}; +use block_producer::{DutiesReader, DutiesReaderError}; use bls::PublicKey; use slot_clock::SlotClock; use spec::ChainSpec; diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index c1775f826..65338cfe7 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -1,6 +1,7 @@ +use self::block_producer_service::{BeaconBlockGrpcClient, BlockProducerService}; use self::duties::{DutiesManager, DutiesManagerService, EpochDutiesMap}; -use crate::block_producer::{BlockProducer, BlockProducerService}; use crate::config::ClientConfig; +use block_producer::BlockProducer; use bls::Keypair; use clap::{App, Arg}; use grpcio::{ChannelBuilder, EnvBuilder}; @@ -12,7 +13,7 @@ use std::path::PathBuf; use std::sync::{Arc, RwLock}; use std::thread; -mod block_producer; +mod block_producer_service; mod config; mod duties; @@ -141,7 +142,7 @@ fn main() { let duties_map = duties_map.clone(); let slot_clock = slot_clock.clone(); let log = log.clone(); - let client = beacon_block_grpc_client.clone(); + let client = Arc::new(BeaconBlockGrpcClient::new(beacon_block_grpc_client.clone())); thread::spawn(move || { let block_producer = BlockProducer::new(spec, duties_map, slot_clock, client); let mut block_producer_service = BlockProducerService { From db6d40e614439884ce6b3123d3e12c19b3baabab Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 24 Jan 2019 09:20:25 +1100 Subject: [PATCH 008/107] Add `Signer` trait to block_producer --- eth2/block_producer/src/lib.rs | 27 ++++++++++++++++++++++----- eth2/block_producer/src/traits.rs | 7 ++++++- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/eth2/block_producer/src/lib.rs b/eth2/block_producer/src/lib.rs index 112684dbb..b2f17852b 100644 --- a/eth2/block_producer/src/lib.rs +++ b/eth2/block_producer/src/lib.rs @@ -7,7 +7,7 @@ use spec::ChainSpec; use std::sync::{Arc, RwLock}; use types::BeaconBlock; -pub use self::traits::{BeaconNode, BeaconNodeError, DutiesReader, DutiesReaderError}; +pub use self::traits::{BeaconNode, BeaconNodeError, DutiesReader, DutiesReaderError, Signer}; #[derive(Debug, PartialEq)] pub enum PollOutcome { @@ -41,21 +41,23 @@ pub enum Error { /// Ensures that messages are not slashable. /// /// Relies upon an external service to keep the `EpochDutiesMap` updated. -pub struct BlockProducer { +pub struct BlockProducer { pub last_processed_slot: u64, spec: Arc, epoch_map: Arc, slot_clock: Arc>, beacon_node: Arc, + signer: Arc, } -impl BlockProducer { +impl BlockProducer { /// Returns a new instance where `last_processed_slot == 0`. pub fn new( spec: Arc, epoch_map: Arc, slot_clock: Arc>, beacon_node: Arc, + signer: Arc, ) -> Self { Self { last_processed_slot: 0, @@ -63,11 +65,12 @@ impl BlockProducer { epoch_map, slot_clock, beacon_node, + signer, } } } -impl BlockProducer { +impl BlockProducer { /// "Poll" to see if the validator is required to take any action. /// /// The slot clock will be read and any new actions undertaken. @@ -176,7 +179,10 @@ mod tests { use super::*; use slot_clock::TestingSlotClock; use std::collections::HashMap; - use types::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use types::{ + test_utils::{SeedableRng, TestRandom, XorShiftRng}, + Signature, + }; // TODO: implement more thorough testing. // https://github.com/sigp/lighthouse/issues/160 @@ -199,6 +205,15 @@ mod tests { } } + struct TestSigner(); + + impl Signer for TestSigner { + fn bls_sign(_message: &[u8]) -> Option { + let mut rng = XorShiftRng::from_seed([42; 16]); + Some(Signature::random_for_test(&mut rng)) + } + } + #[test] pub fn polling() { let mut rng = XorShiftRng::from_seed([42; 16]); @@ -206,6 +221,7 @@ mod tests { let spec = Arc::new(ChainSpec::foundation()); let slot_clock = Arc::new(RwLock::new(TestingSlotClock::new(0))); let beacon_node = Arc::new(TestBeaconNode::default()); + let signer = Arc::new(TestSigner()); let mut epoch_map = EpochMap::new(); let produce_slot = 100; @@ -218,6 +234,7 @@ mod tests { epoch_map.clone(), slot_clock.clone(), beacon_node.clone(), + signer.clone(), ); // Configure responses from the BeaconNode. diff --git a/eth2/block_producer/src/traits.rs b/eth2/block_producer/src/traits.rs index e16af2460..c718064d2 100644 --- a/eth2/block_producer/src/traits.rs +++ b/eth2/block_producer/src/traits.rs @@ -1,4 +1,4 @@ -use types::BeaconBlock; +use types::{BeaconBlock, Signature}; #[derive(Debug, PartialEq, Clone)] pub enum BeaconNodeError { @@ -23,6 +23,11 @@ pub enum DutiesReaderError { Poisoned, } +/// Provides methods for a validator to determine their responsibilities for some slot. pub trait DutiesReader: Send + Sync { fn is_block_production_slot(&self, epoch: u64, slot: u64) -> Result; } + +pub trait Signer { + fn bls_sign(message: &[u8]) -> Option; +} From 2bcce37b3cf057141cae463c4f4fc84b43e730ba Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 24 Jan 2019 11:42:40 +1100 Subject: [PATCH 009/107] Remove outdated attestation_validation crate --- Cargo.toml | 1 - eth2/attestation_validation/Cargo.toml | 12 - .../src/block_inclusion.rs | 246 ------------------ eth2/attestation_validation/src/enums.rs | 37 --- .../src/justified_block.rs | 80 ------ .../src/justified_slot.rs | 39 --- eth2/attestation_validation/src/lib.rs | 22 -- eth2/attestation_validation/src/macros.rs | 19 -- .../attestation_validation/src/shard_block.rs | 46 ---- eth2/attestation_validation/src/signature.rs | 151 ----------- 10 files changed, 653 deletions(-) delete mode 100644 eth2/attestation_validation/Cargo.toml delete mode 100644 eth2/attestation_validation/src/block_inclusion.rs delete mode 100644 eth2/attestation_validation/src/enums.rs delete mode 100644 eth2/attestation_validation/src/justified_block.rs delete mode 100644 eth2/attestation_validation/src/justified_slot.rs delete mode 100644 eth2/attestation_validation/src/lib.rs delete mode 100644 eth2/attestation_validation/src/macros.rs delete mode 100644 eth2/attestation_validation/src/shard_block.rs delete mode 100644 eth2/attestation_validation/src/signature.rs diff --git a/Cargo.toml b/Cargo.toml index 26f7d293a..5a25e3484 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,5 @@ [workspace] members = [ - "eth2/attestation_validation", "eth2/block_producer", "eth2/genesis", "eth2/naive_fork_choice", diff --git a/eth2/attestation_validation/Cargo.toml b/eth2/attestation_validation/Cargo.toml deleted file mode 100644 index b944f90f9..000000000 --- a/eth2/attestation_validation/Cargo.toml +++ /dev/null @@ -1,12 +0,0 @@ -[package] -name = "attestation_validation" -version = "0.1.0" -authors = ["Paul Hauner "] -edition = "2018" - -[dependencies] -bls = { path = "../utils/bls" } -db = { path = "../../beacon_node/db" } -hashing = { path = "../utils/hashing" } -ssz = { path = "../utils/ssz" } -types = { path = "../types" } diff --git a/eth2/attestation_validation/src/block_inclusion.rs b/eth2/attestation_validation/src/block_inclusion.rs deleted file mode 100644 index 76a5c9797..000000000 --- a/eth2/attestation_validation/src/block_inclusion.rs +++ /dev/null @@ -1,246 +0,0 @@ -use super::{Error, Invalid, Outcome}; - -/// Check that an attestation is valid to be included in some block. -pub fn validate_attestation_for_block( - attestation_slot: u64, - block_slot: u64, - parent_block_slot: u64, - min_attestation_inclusion_delay: u64, - epoch_length: u64, -) -> Result { - /* - * There is a delay before an attestation may be included in a block, quantified by - * `slots` and defined as `min_attestation_inclusion_delay`. - * - * So, an attestation must be at least `min_attestation_inclusion_delay` slots "older" than the - * block it is contained in. - */ - verify_or!( - // TODO: this differs from the spec as it does not handle underflows correctly. - // https://github.com/sigp/lighthouse/issues/95 - attestation_slot < block_slot.saturating_sub(min_attestation_inclusion_delay - 1), - reject!(Invalid::AttestationTooRecent) - ); - - /* - * A block may not include attestations reference slots more than an epoch length + 1 prior to - * the block slot. - */ - verify_or!( - attestation_slot >= parent_block_slot.saturating_sub(epoch_length + 1), - reject!(Invalid::AttestationTooOld) - ); - - accept!() -} - -#[cfg(test)] -mod tests { - use super::*; - - /* - * Invalid::AttestationTooOld tests. - */ - - #[test] - fn test_inclusion_too_old_minimal() { - let min_attestation_inclusion_delay = 10; - let epoch_length = 20; - let block_slot = 100; - let parent_block_slot = block_slot - 1; - let attestation_slot = block_slot - min_attestation_inclusion_delay; - - let outcome = validate_attestation_for_block( - attestation_slot, - block_slot, - parent_block_slot, - min_attestation_inclusion_delay, - epoch_length, - ); - assert_eq!(outcome, Ok(Outcome::Valid)); - } - - #[test] - fn test_inclusion_too_old_maximal() { - let min_attestation_inclusion_delay = 10; - let epoch_length = 20; - let block_slot = 100; - let parent_block_slot = block_slot - 1; - let attestation_slot = block_slot - epoch_length + 1; - - let outcome = validate_attestation_for_block( - attestation_slot, - block_slot, - parent_block_slot, - min_attestation_inclusion_delay, - epoch_length, - ); - assert_eq!(outcome, Ok(Outcome::Valid)); - } - - #[test] - fn test_inclusion_too_old_saturating_non_zero_attestation_slot() { - let min_attestation_inclusion_delay = 10; - let epoch_length = 20; - let block_slot = epoch_length + 1; - let parent_block_slot = block_slot - 1; - let attestation_slot = block_slot - min_attestation_inclusion_delay; - - let outcome = validate_attestation_for_block( - attestation_slot, - block_slot, - parent_block_slot, - min_attestation_inclusion_delay, - epoch_length, - ); - assert_eq!(outcome, Ok(Outcome::Valid)); - } - - #[test] - fn test_inclusion_too_old_saturating_zero_attestation_slot() { - let min_attestation_inclusion_delay = 10; - let epoch_length = 20; - let block_slot = epoch_length + 1; - let parent_block_slot = block_slot - 1; - let attestation_slot = 0; - - let outcome = validate_attestation_for_block( - attestation_slot, - block_slot, - parent_block_slot, - min_attestation_inclusion_delay, - epoch_length, - ); - assert_eq!(outcome, Ok(Outcome::Valid)); - } - - #[test] - fn test_inclusion_too_old() { - let min_attestation_inclusion_delay = 10; - let epoch_length = 20; - let block_slot = epoch_length * 2; - let parent_block_slot = block_slot - 1; - let attestation_slot = parent_block_slot - (epoch_length + 2); - - let outcome = validate_attestation_for_block( - attestation_slot, - block_slot, - parent_block_slot, - min_attestation_inclusion_delay, - epoch_length, - ); - assert_eq!(outcome, Ok(Outcome::Invalid(Invalid::AttestationTooOld))); - } - - /* - * Invalid::AttestationTooRecent tests. - */ - - #[test] - fn test_inclusion_too_recent_minimal() { - let parent_block_slot = 99; - let min_attestation_inclusion_delay = 10; - let epoch_length = 20; - let block_slot = 100; - let attestation_slot = block_slot - min_attestation_inclusion_delay; - - let outcome = validate_attestation_for_block( - attestation_slot, - block_slot, - parent_block_slot, - min_attestation_inclusion_delay, - epoch_length, - ); - assert_eq!(outcome, Ok(Outcome::Valid)); - } - - #[test] - fn test_inclusion_too_recent_maximal() { - let parent_block_slot = 99; - let min_attestation_inclusion_delay = 10; - let epoch_length = 20; - let block_slot = 100; - let attestation_slot = block_slot - epoch_length; - - let outcome = validate_attestation_for_block( - attestation_slot, - block_slot, - parent_block_slot, - min_attestation_inclusion_delay, - epoch_length, - ); - assert_eq!(outcome, Ok(Outcome::Valid)); - } - - #[test] - fn test_inclusion_too_recent_insufficient() { - let parent_block_slot = 99; - let min_attestation_inclusion_delay = 10; - let epoch_length = 20; - let block_slot = 100; - let attestation_slot = block_slot - (min_attestation_inclusion_delay - 1); - - let outcome = validate_attestation_for_block( - attestation_slot, - block_slot, - parent_block_slot, - min_attestation_inclusion_delay, - epoch_length, - ); - assert_eq!(outcome, Ok(Outcome::Invalid(Invalid::AttestationTooRecent))); - } - - #[test] - fn test_inclusion_too_recent_first_possible_slot() { - let min_attestation_inclusion_delay = 10; - let epoch_length = 20; - let block_slot = min_attestation_inclusion_delay; - let attestation_slot = 0; - let parent_block_slot = block_slot - 1; - - let outcome = validate_attestation_for_block( - attestation_slot, - block_slot, - parent_block_slot, - min_attestation_inclusion_delay, - epoch_length, - ); - assert_eq!(outcome, Ok(Outcome::Valid)); - } - - #[test] - fn test_inclusion_too_recent_saturation_non_zero_slot() { - let min_attestation_inclusion_delay = 10; - let epoch_length = 20; - let block_slot = min_attestation_inclusion_delay - 1; - let parent_block_slot = block_slot - 1; - let attestation_slot = 0; - - let outcome = validate_attestation_for_block( - attestation_slot, - block_slot, - parent_block_slot, - min_attestation_inclusion_delay, - epoch_length, - ); - assert_eq!(outcome, Ok(Outcome::Invalid(Invalid::AttestationTooRecent))); - } - - #[test] - fn test_inclusion_too_recent_saturation_zero_slot() { - let min_attestation_inclusion_delay = 10; - let epoch_length = 20; - let block_slot = min_attestation_inclusion_delay - 1; - let parent_block_slot = block_slot - 1; - let attestation_slot = 0; - - let outcome = validate_attestation_for_block( - attestation_slot, - block_slot, - parent_block_slot, - min_attestation_inclusion_delay, - epoch_length, - ); - assert_eq!(outcome, Ok(Outcome::Invalid(Invalid::AttestationTooRecent))); - } -} diff --git a/eth2/attestation_validation/src/enums.rs b/eth2/attestation_validation/src/enums.rs deleted file mode 100644 index 6c94c628c..000000000 --- a/eth2/attestation_validation/src/enums.rs +++ /dev/null @@ -1,37 +0,0 @@ -/// Reasons why an `AttestationRecord` can be invalid. -#[derive(PartialEq, Debug)] -pub enum Invalid { - AttestationTooRecent, - AttestationTooOld, - JustifiedSlotImpermissable, - JustifiedBlockNotInChain, - JustifiedBlockHashMismatch, - UnknownShard, - ShardBlockHashMismatch, - SignatureInvalid, -} - -/// The outcome of validating the `AttestationRecord`. -/// -/// Distinct from the `Error` enum as an `Outcome` indicates that validation executed sucessfully -/// and determined the validity `AttestationRecord`. -#[derive(PartialEq, Debug)] -pub enum Outcome { - Valid, - Invalid(Invalid), -} - -/// Errors that prevent this function from correctly validating the `AttestationRecord`. -/// -/// Distinct from the `Outcome` enum as `Errors` indicate that validation encountered an unexpected -/// condition and was unable to perform its duty. -#[derive(PartialEq, Debug)] -pub enum Error { - BlockHasNoParent, - BadValidatorIndex, - UnableToLookupBlockAtSlot, - OutOfBoundsBitfieldIndex, - PublicKeyCorrupt, - NoPublicKeyForValidator, - DBError(String), -} diff --git a/eth2/attestation_validation/src/justified_block.rs b/eth2/attestation_validation/src/justified_block.rs deleted file mode 100644 index e910819c8..000000000 --- a/eth2/attestation_validation/src/justified_block.rs +++ /dev/null @@ -1,80 +0,0 @@ -use super::db::stores::{BeaconBlockAtSlotError, BeaconBlockStore}; -use super::db::ClientDB; -use super::types::AttestationData; -use super::types::Hash256; -use super::{Error, Invalid, Outcome}; -use std::sync::Arc; - -/// Verify that a attestation's `data.justified_block_hash` matches the local hash of the block at the -/// attestation's `data.justified_slot`. -/// -/// `chain_tip_block_hash` is the tip of the chain in which the justified block hash should exist -/// locally. As Lightouse stores multiple chains locally, it is possible to have multiple blocks at -/// the same slot. `chain_tip_block_hash` serves to restrict the lookup to a single chain, where -/// each slot may have exactly zero or one blocks. -pub fn validate_attestation_justified_block_hash( - data: &AttestationData, - chain_tip_block_hash: &Hash256, - block_store: &Arc>, -) -> Result -where - T: ClientDB + Sized, -{ - /* - * The `justified_block_hash` in the attestation must match exactly the hash of the block at - * that slot in the local chain. - * - * This condition also infers that the `justified_slot` specified in attestation must exist - * locally. - */ - match block_hash_at_slot(chain_tip_block_hash, data.justified_slot, block_store)? { - None => reject!(Invalid::JustifiedBlockNotInChain), - Some(local_justified_block_hash) => { - verify_or!( - data.justified_block_hash == local_justified_block_hash, - reject!(Invalid::JustifiedBlockHashMismatch) - ); - } - }; - accept!() -} - -/// Returns the hash (or None) of a block at a slot in the chain that is specified by -/// `chain_tip_hash`. -/// -/// Given that the database stores multiple chains, it is possible for there to be multiple blocks -/// at the given slot. `chain_tip_hash` specifies exactly which chain should be used. -fn block_hash_at_slot( - chain_tip_hash: &Hash256, - slot: u64, - block_store: &Arc>, -) -> Result, Error> -where - T: ClientDB + Sized, -{ - match block_store.block_at_slot(&chain_tip_hash, slot)? { - None => Ok(None), - Some((hash_bytes, _)) => Ok(Some(Hash256::from(&hash_bytes[..]))), - } -} - -impl From for Error { - fn from(e: BeaconBlockAtSlotError) -> Self { - match e { - BeaconBlockAtSlotError::DBError(s) => Error::DBError(s), - _ => Error::UnableToLookupBlockAtSlot, - } - } -} - -#[cfg(test)] -mod tests { - /* - * TODO: Implement tests. - * - * These tests will require the `BeaconBlock` and `BeaconBlockBody` updates, which are not - * yet included in the code base. Adding tests now will result in duplicated work. - * - * https://github.com/sigp/lighthouse/issues/97 - */ -} diff --git a/eth2/attestation_validation/src/justified_slot.rs b/eth2/attestation_validation/src/justified_slot.rs deleted file mode 100644 index ea71f2616..000000000 --- a/eth2/attestation_validation/src/justified_slot.rs +++ /dev/null @@ -1,39 +0,0 @@ -use super::types::{AttestationData, BeaconState}; -use super::{Error, Invalid, Outcome}; - -/// Verify that an attestation's `data.justified_slot` matches the justified slot known to the -/// `state`. -/// -/// In the case that an attestation references a slot _before_ the latest state transition, is -/// acceptable for the attestation to reference the previous known `justified_slot`. If this were -/// not the case, all attestations created _prior_ to the last state recalculation would be rejected -/// if a block was justified in that state recalculation. It is both ideal and likely that blocks -/// will be justified during a state recalcuation. -pub fn validate_attestation_justified_slot( - data: &AttestationData, - state: &BeaconState, - epoch_length: u64, -) -> Result { - let permissable_justified_slot = if data.slot >= state.slot - (state.slot % epoch_length) { - state.justified_slot - } else { - state.previous_justified_slot - }; - verify_or!( - data.justified_slot == permissable_justified_slot, - reject!(Invalid::JustifiedSlotImpermissable) - ); - accept!() -} - -#[cfg(test)] -mod tests { - /* - * TODO: Implement tests. - * - * These tests will require the `BeaconBlock` and `BeaconBlockBody` updates, which are not - * yet included in the code base. Adding tests now will result in duplicated work. - * - * https://github.com/sigp/lighthouse/issues/97 - */ -} diff --git a/eth2/attestation_validation/src/lib.rs b/eth2/attestation_validation/src/lib.rs deleted file mode 100644 index 825371ed0..000000000 --- a/eth2/attestation_validation/src/lib.rs +++ /dev/null @@ -1,22 +0,0 @@ -extern crate bls; -extern crate db; -extern crate hashing; -extern crate ssz; -extern crate types; - -#[macro_use] -mod macros; - -mod block_inclusion; -mod enums; -mod justified_block; -mod justified_slot; -mod shard_block; -mod signature; - -pub use crate::block_inclusion::validate_attestation_for_block; -pub use crate::enums::{Error, Invalid, Outcome}; -pub use crate::justified_block::validate_attestation_justified_block_hash; -pub use crate::justified_slot::validate_attestation_justified_slot; -pub use crate::shard_block::validate_attestation_data_shard_block_hash; -pub use crate::signature::validate_attestation_signature; diff --git a/eth2/attestation_validation/src/macros.rs b/eth2/attestation_validation/src/macros.rs deleted file mode 100644 index faae00fcf..000000000 --- a/eth2/attestation_validation/src/macros.rs +++ /dev/null @@ -1,19 +0,0 @@ -macro_rules! verify_or { - ($condition: expr, $result: expr) => { - if !$condition { - $result - } - }; -} - -macro_rules! reject { - ($result: expr) => { - return Ok(Outcome::Invalid($result)); - }; -} - -macro_rules! accept { - () => { - Ok(Outcome::Valid) - }; -} diff --git a/eth2/attestation_validation/src/shard_block.rs b/eth2/attestation_validation/src/shard_block.rs deleted file mode 100644 index e1d9487de..000000000 --- a/eth2/attestation_validation/src/shard_block.rs +++ /dev/null @@ -1,46 +0,0 @@ -use super::db::ClientDB; -use super::types::{AttestationData, BeaconState}; -use super::{Error, Invalid, Outcome}; - -/// Check that an attestation is valid with reference to some state. -pub fn validate_attestation_data_shard_block_hash( - data: &AttestationData, - state: &BeaconState, -) -> Result -where - T: ClientDB + Sized, -{ - /* - * The `shard_block_hash` in the state's `latest_crosslinks` must match either the - * `latest_crosslink_hash` or the `shard_block_hash` on the attestation. - * - * TODO: figure out the reasoning behind this. - */ - match state.latest_crosslinks.get(data.shard as usize) { - None => reject!(Invalid::UnknownShard), - Some(crosslink) => { - let local_shard_block_hash = crosslink.shard_block_root; - let shard_block_hash_is_permissable = { - (local_shard_block_hash == data.latest_crosslink_hash) - || (local_shard_block_hash == data.shard_block_hash) - }; - verify_or!( - shard_block_hash_is_permissable, - reject!(Invalid::ShardBlockHashMismatch) - ); - } - }; - accept!() -} - -#[cfg(test)] -mod tests { - /* - * TODO: Implement tests. - * - * These tests will require the `BeaconBlock` and `BeaconBlockBody` updates, which are not - * yet included in the code base. Adding tests now will result in duplicated work. - * - * https://github.com/sigp/lighthouse/issues/97 - */ -} diff --git a/eth2/attestation_validation/src/signature.rs b/eth2/attestation_validation/src/signature.rs deleted file mode 100644 index 7d08be337..000000000 --- a/eth2/attestation_validation/src/signature.rs +++ /dev/null @@ -1,151 +0,0 @@ -use super::bls::{AggregatePublicKey, AggregateSignature}; -use super::db::stores::{ValidatorStore, ValidatorStoreError}; -use super::db::ClientDB; -use super::types::{AttestationData, Bitfield, BitfieldError}; -use super::{Error, Invalid, Outcome}; - -/// Validate that some signature is correct for some attestation data and known validator set. -pub fn validate_attestation_signature( - attestation_data: &AttestationData, - participation_bitfield: &Bitfield, - aggregate_signature: &AggregateSignature, - attestation_indices: &[usize], - validator_store: &ValidatorStore, -) -> Result -where - T: ClientDB + Sized, -{ - let mut agg_pub_key = AggregatePublicKey::new(); - - for i in 0..attestation_indices.len() { - let voted = participation_bitfield.get(i)?; - if voted { - // De-reference the attestation index into a canonical ValidatorRecord index. - let validator = *attestation_indices.get(i).ok_or(Error::BadValidatorIndex)?; - // Load the public key. - let pub_key = validator_store - .get_public_key_by_index(validator)? - .ok_or(Error::NoPublicKeyForValidator)?; - // Aggregate the public key. - agg_pub_key.add(&pub_key.as_raw()); - } - } - - let signed_message = attestation_data_signing_message(attestation_data); - verify_or!( - // TODO: ensure "domain" for aggregate signatures is included. - // https://github.com/sigp/lighthouse/issues/91 - aggregate_signature.verify(&signed_message, &agg_pub_key), - reject!(Invalid::SignatureInvalid) - ); - - accept!() -} - -fn attestation_data_signing_message(attestation_data: &AttestationData) -> Vec { - let mut signed_message = attestation_data.canonical_root().to_vec(); - signed_message.append(&mut vec![0]); - signed_message -} - -impl From for Error { - fn from(error: ValidatorStoreError) -> Self { - match error { - ValidatorStoreError::DBError(s) => Error::DBError(s), - ValidatorStoreError::DecodeError => Error::PublicKeyCorrupt, - } - } -} - -impl From for Error { - fn from(_error: BitfieldError) -> Self { - Error::OutOfBoundsBitfieldIndex - } -} - -#[cfg(test)] -mod tests { - use super::super::bls::{Keypair, Signature}; - use super::super::db::MemoryDB; - use super::*; - use std::sync::Arc; - - /* - * TODO: Test cases are not comprehensive. - * https://github.com/sigp/lighthouse/issues/94 - */ - - #[test] - fn test_signature_verification() { - let attestation_data = AttestationData::zero(); - let message = attestation_data_signing_message(&attestation_data); - let signing_keypairs = vec![ - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - ]; - let non_signing_keypairs = vec![ - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - ]; - /* - * Signing keypairs first, then non-signing - */ - let mut all_keypairs = signing_keypairs.clone(); - all_keypairs.append(&mut non_signing_keypairs.clone()); - - let attestation_indices: Vec = (0..all_keypairs.len()).collect(); - let mut bitfield = Bitfield::from_elem(all_keypairs.len(), false); - for i in 0..signing_keypairs.len() { - bitfield.set(i, true).unwrap(); - } - - let db = Arc::new(MemoryDB::open()); - let store = ValidatorStore::new(db); - - for (i, keypair) in all_keypairs.iter().enumerate() { - store.put_public_key_by_index(i, &keypair.pk).unwrap(); - } - - let mut agg_sig = AggregateSignature::new(); - for keypair in &signing_keypairs { - let sig = Signature::new(&message, &keypair.sk); - agg_sig.add(&sig); - } - - /* - * Test using all valid parameters. - */ - let outcome = validate_attestation_signature( - &attestation_data, - &bitfield, - &agg_sig, - &attestation_indices, - &store, - ) - .unwrap(); - assert_eq!(outcome, Outcome::Valid); - - /* - * Add another validator to the bitfield, run validation will all other - * parameters the same and assert that it fails. - */ - bitfield.set(signing_keypairs.len() + 1, true).unwrap(); - let outcome = validate_attestation_signature( - &attestation_data, - &bitfield, - &agg_sig, - &attestation_indices, - &store, - ) - .unwrap(); - assert_eq!(outcome, Outcome::Invalid(Invalid::SignatureInvalid)); - } -} From 8c0e634aa48c7726533b14480c76f014842c5c9a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 24 Jan 2019 11:50:34 +1100 Subject: [PATCH 010/107] Add Signer to validator client --- eth2/block_producer/src/lib.rs | 82 ++++++++++--------- .../beacon_node.rs} | 2 +- .../src/test_utils/epoch_map.rs | 14 ++++ eth2/block_producer/src/test_utils/mod.rs | 7 ++ eth2/block_producer/src/test_utils/signer.rs | 31 +++++++ eth2/block_producer/src/traits.rs | 6 +- .../src/block_producer_service/service.rs | 11 ++- validator_client/src/main.rs | 6 +- 8 files changed, 110 insertions(+), 49 deletions(-) rename eth2/block_producer/src/{test_node.rs => test_utils/beacon_node.rs} (97%) create mode 100644 eth2/block_producer/src/test_utils/epoch_map.rs create mode 100644 eth2/block_producer/src/test_utils/mod.rs create mode 100644 eth2/block_producer/src/test_utils/signer.rs diff --git a/eth2/block_producer/src/lib.rs b/eth2/block_producer/src/lib.rs index b2f17852b..fc06ffa46 100644 --- a/eth2/block_producer/src/lib.rs +++ b/eth2/block_producer/src/lib.rs @@ -1,11 +1,10 @@ -#[cfg(test)] -mod test_node; +pub mod test_utils; mod traits; use slot_clock::SlotClock; use spec::ChainSpec; use std::sync::{Arc, RwLock}; -use types::BeaconBlock; +use types::{BeaconBlock, Hash256, ProposalSignedData}; pub use self::traits::{BeaconNode, BeaconNodeError, DutiesReader, DutiesReaderError, Signer}; @@ -23,6 +22,8 @@ pub enum PollOutcome { SlotAlreadyProcessed(u64), /// The Beacon Node was unable to produce a block at that slot. BeaconNodeUnableToProduceBlock(u64), + /// The signer failed to sign the message. + SignerRejection(u64), } #[derive(Debug, PartialEq)] @@ -123,9 +124,12 @@ impl BlockProducer Result { if let Some(block) = self.beacon_node.produce_beacon_block(slot)? { if self.safe_to_produce(&block) { - let block = self.sign_block(block); - self.beacon_node.publish_beacon_block(block)?; - Ok(PollOutcome::BlockProduced(slot)) + if let Some(block) = self.sign_block(block) { + self.beacon_node.publish_beacon_block(block)?; + Ok(PollOutcome::BlockProduced(slot)) + } else { + Ok(PollOutcome::SignerRejection(slot)) + } } else { Ok(PollOutcome::SlashableBlockNotProduced(slot)) } @@ -138,11 +142,30 @@ impl BlockProducer BeaconBlock { - // TODO: sign the block - // https://github.com/sigp/lighthouse/issues/160 + fn sign_block(&mut self, mut block: BeaconBlock) -> Option { self.store_produce(&block); - block + + let proposal_root = { + let block_without_signature_root = { + let mut block_without_signature = block.clone(); + block_without_signature.signature = self.spec.empty_signature.clone(); + block_without_signature.canonical_root() + }; + let proposal = ProposalSignedData { + slot: block.slot, + shard: self.spec.beacon_chain_shard_number, + block_root: block_without_signature_root, + }; + hash_tree_root(&proposal) + }; + + match self.signer.bls_sign(&proposal_root[..]) { + None => None, + Some(signature) => { + block.signature = signature; + Some(block) + } + } } /// Returns `true` if signing a block is safe (non-slashable). @@ -167,6 +190,11 @@ impl BlockProducer(_input: &T) -> Hash256 { + // TODO: stubbed out. + Hash256::zero() +} + impl From for Error { fn from(e: BeaconNodeError) -> Error { Error::BeaconNodeError(e) @@ -175,13 +203,12 @@ impl From for Error { #[cfg(test)] mod tests { - use super::test_node::TestBeaconNode; + use super::test_utils::{TestBeaconNode, TestEpochMap, TestSigner}; use super::*; use slot_clock::TestingSlotClock; - use std::collections::HashMap; use types::{ test_utils::{SeedableRng, TestRandom, XorShiftRng}, - Signature, + Keypair, }; // TODO: implement more thorough testing. @@ -189,31 +216,6 @@ mod tests { // // These tests should serve as a good example for future tests. - type EpochMap = HashMap; - - impl DutiesReader for EpochMap { - fn is_block_production_slot( - &self, - epoch: u64, - slot: u64, - ) -> Result { - match self.get(&epoch) { - Some(s) if *s == slot => Ok(true), - Some(s) if *s != slot => Ok(false), - _ => Err(DutiesReaderError::UnknownEpoch), - } - } - } - - struct TestSigner(); - - impl Signer for TestSigner { - fn bls_sign(_message: &[u8]) -> Option { - let mut rng = XorShiftRng::from_seed([42; 16]); - Some(Signature::random_for_test(&mut rng)) - } - } - #[test] pub fn polling() { let mut rng = XorShiftRng::from_seed([42; 16]); @@ -221,9 +223,9 @@ mod tests { let spec = Arc::new(ChainSpec::foundation()); let slot_clock = Arc::new(RwLock::new(TestingSlotClock::new(0))); let beacon_node = Arc::new(TestBeaconNode::default()); - let signer = Arc::new(TestSigner()); + let signer = Arc::new(TestSigner::new(Keypair::random())); - let mut epoch_map = EpochMap::new(); + let mut epoch_map = TestEpochMap::new(); let produce_slot = 100; let produce_epoch = produce_slot / spec.epoch_length; epoch_map.insert(produce_epoch, produce_slot); diff --git a/eth2/block_producer/src/test_node.rs b/eth2/block_producer/src/test_utils/beacon_node.rs similarity index 97% rename from eth2/block_producer/src/test_node.rs rename to eth2/block_producer/src/test_utils/beacon_node.rs index e99613e8f..bca01d9c7 100644 --- a/eth2/block_producer/src/test_node.rs +++ b/eth2/block_producer/src/test_utils/beacon_node.rs @@ -1,4 +1,4 @@ -use super::traits::{BeaconNode, BeaconNodeError}; +use crate::traits::{BeaconNode, BeaconNodeError}; use std::sync::RwLock; use types::BeaconBlock; diff --git a/eth2/block_producer/src/test_utils/epoch_map.rs b/eth2/block_producer/src/test_utils/epoch_map.rs new file mode 100644 index 000000000..290660afd --- /dev/null +++ b/eth2/block_producer/src/test_utils/epoch_map.rs @@ -0,0 +1,14 @@ +use crate::{DutiesReader, DutiesReaderError}; +use std::collections::HashMap; + +pub type TestEpochMap = HashMap; + +impl DutiesReader for TestEpochMap { + fn is_block_production_slot(&self, epoch: u64, slot: u64) -> Result { + match self.get(&epoch) { + Some(s) if *s == slot => Ok(true), + Some(s) if *s != slot => Ok(false), + _ => Err(DutiesReaderError::UnknownEpoch), + } + } +} diff --git a/eth2/block_producer/src/test_utils/mod.rs b/eth2/block_producer/src/test_utils/mod.rs new file mode 100644 index 000000000..0dd384b12 --- /dev/null +++ b/eth2/block_producer/src/test_utils/mod.rs @@ -0,0 +1,7 @@ +mod beacon_node; +mod epoch_map; +mod signer; + +pub use self::beacon_node::TestBeaconNode; +pub use self::epoch_map::TestEpochMap; +pub use self::signer::TestSigner; diff --git a/eth2/block_producer/src/test_utils/signer.rs b/eth2/block_producer/src/test_utils/signer.rs new file mode 100644 index 000000000..d43c0feb0 --- /dev/null +++ b/eth2/block_producer/src/test_utils/signer.rs @@ -0,0 +1,31 @@ +use crate::traits::Signer; +use std::sync::RwLock; +use types::{Keypair, Signature}; + +/// A test-only struct used to simulate a Beacon Node. +pub struct TestSigner { + keypair: Keypair, + should_sign: RwLock, +} + +impl TestSigner { + /// Produce a new TestSigner with signing enabled by default. + pub fn new(keypair: Keypair) -> Self { + Self { + keypair, + should_sign: RwLock::new(true), + } + } + + /// If set to `false`, the service will refuse to sign all messages. Otherwise, all messages + /// will be signed. + pub fn enable_signing(&self, enabled: bool) { + *self.should_sign.write().unwrap() = enabled; + } +} + +impl Signer for TestSigner { + fn bls_sign(&self, message: &[u8]) -> Option { + Some(Signature::new(message, &self.keypair.sk)) + } +} diff --git a/eth2/block_producer/src/traits.rs b/eth2/block_producer/src/traits.rs index c718064d2..c9716701b 100644 --- a/eth2/block_producer/src/traits.rs +++ b/eth2/block_producer/src/traits.rs @@ -18,16 +18,18 @@ pub trait BeaconNode: Send + Sync { fn publish_beacon_block(&self, block: BeaconBlock) -> Result; } +#[derive(Debug, PartialEq, Clone)] pub enum DutiesReaderError { UnknownEpoch, Poisoned, } -/// Provides methods for a validator to determine their responsibilities for some slot. +/// Informs a validator of their duties (e.g., block production). pub trait DutiesReader: Send + Sync { fn is_block_production_slot(&self, epoch: u64, slot: u64) -> Result; } +/// Signs message using an internally-maintained private key. pub trait Signer { - fn bls_sign(message: &[u8]) -> Option; + fn bls_sign(&self, message: &[u8]) -> Option; } diff --git a/validator_client/src/block_producer_service/service.rs b/validator_client/src/block_producer_service/service.rs index 6652058c8..3966d42f9 100644 --- a/validator_client/src/block_producer_service/service.rs +++ b/validator_client/src/block_producer_service/service.rs @@ -1,17 +1,17 @@ use block_producer::{ - BeaconNode, BlockProducer, DutiesReader, PollOutcome as BlockProducerPollOutcome, + BeaconNode, BlockProducer, DutiesReader, PollOutcome as BlockProducerPollOutcome, Signer, }; use slog::{error, info, warn, Logger}; use slot_clock::SlotClock; use std::time::Duration; -pub struct BlockProducerService { - pub block_producer: BlockProducer, +pub struct BlockProducerService { + pub block_producer: BlockProducer, pub poll_interval_millis: u64, pub log: Logger, } -impl BlockProducerService { +impl BlockProducerService { /// Run a loop which polls the block producer each `poll_interval_millis` millseconds. /// /// Logs the results of the polls. @@ -39,6 +39,9 @@ impl BlockProducerService Ok(BlockProducerPollOutcome::BeaconNodeUnableToProduceBlock(slot)) => { error!(self.log, "Beacon node unable to produce block"; "slot" => slot) } + Ok(BlockProducerPollOutcome::SignerRejection(slot)) => { + error!(self.log, "The cryptographic signer refused to sign the block"; "slot" => slot) + } }; std::thread::sleep(Duration::from_millis(self.poll_interval_millis)); diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index 65338cfe7..cd626b2fc 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -1,7 +1,7 @@ use self::block_producer_service::{BeaconBlockGrpcClient, BlockProducerService}; use self::duties::{DutiesManager, DutiesManagerService, EpochDutiesMap}; use crate::config::ClientConfig; -use block_producer::BlockProducer; +use block_producer::{test_utils::TestSigner, BlockProducer}; use bls::Keypair; use clap::{App, Arg}; use grpcio::{ChannelBuilder, EnvBuilder}; @@ -139,12 +139,14 @@ fn main() { // Spawn a new thread to perform block production for the validator. let producer_thread = { let spec = spec.clone(); + let signer = Arc::new(TestSigner::new(keypair.clone())); let duties_map = duties_map.clone(); let slot_clock = slot_clock.clone(); let log = log.clone(); let client = Arc::new(BeaconBlockGrpcClient::new(beacon_block_grpc_client.clone())); thread::spawn(move || { - let block_producer = BlockProducer::new(spec, duties_map, slot_clock, client); + let block_producer = + BlockProducer::new(spec, duties_map, slot_clock, client, signer); let mut block_producer_service = BlockProducerService { block_producer, poll_interval_millis, From eae904270b66357d38f5f7eaf809fee4f751e2ce Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 24 Jan 2019 11:51:25 +1100 Subject: [PATCH 011/107] Derive Clone for SystemTimeSlotClock --- eth2/utils/slot_clock/src/system_time_slot_clock.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/eth2/utils/slot_clock/src/system_time_slot_clock.rs b/eth2/utils/slot_clock/src/system_time_slot_clock.rs index 5c5f2e0ea..ba62b3b93 100644 --- a/eth2/utils/slot_clock/src/system_time_slot_clock.rs +++ b/eth2/utils/slot_clock/src/system_time_slot_clock.rs @@ -10,6 +10,7 @@ pub enum Error { } /// Determines the present slot based upon the present system time. +#[derive(Clone)] pub struct SystemTimeSlotClock { genesis_seconds: u64, slot_duration_seconds: u64, From a4ac5b0d3ed5930d01245c94973ecf29267957e4 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 24 Jan 2019 11:51:48 +1100 Subject: [PATCH 012/107] Begin impl block_producer in beacon chain tests --- beacon_node/beacon_chain/Cargo.toml | 1 + beacon_node/beacon_chain/tests/chain.rs | 71 ++++++++++++++++++++++++- 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index d6dac9a4b..7e157dffb 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -5,6 +5,7 @@ authors = ["Paul Hauner "] edition = "2018" [dependencies] +block_producer = { path = "../../eth2/block_producer" } bls = { path = "../../eth2/utils/bls" } boolean-bitfield = { path = "../../eth2/utils/boolean-bitfield" } db = { path = "../db" } diff --git a/beacon_node/beacon_chain/tests/chain.rs b/beacon_node/beacon_chain/tests/chain.rs index 9c76bf59d..390d48ea4 100644 --- a/beacon_node/beacon_chain/tests/chain.rs +++ b/beacon_node/beacon_chain/tests/chain.rs @@ -1,12 +1,77 @@ use beacon_chain::{BeaconChain, BlockProcessingOutcome}; -use bls::Signature; +#[cfg(test)] +use block_producer::{ + test_utils::{TestEpochMap, TestSigner}, + BeaconNode as BeaconBlockNode, BeaconNodeError as BeaconBlockNodeError, BlockProducer, +}; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, MemoryDB, }; use slot_clock::TestingSlotClock; use spec::ChainSpec; -use std::sync::Arc; +use std::sync::{Arc, RwLock}; +use types::{BeaconBlock, Keypair}; + +struct DirectBeaconNode(); + +impl BeaconBlockNode for DirectBeaconNode { + fn produce_beacon_block(&self, slot: u64) -> Result, BeaconBlockNodeError> { + Err(BeaconBlockNodeError::DecodeFailure) + } + + /// Returns the value specified by the `set_next_publish_result`. + fn publish_beacon_block(&self, block: BeaconBlock) -> Result { + Err(BeaconBlockNodeError::DecodeFailure) + } +} + +struct Validator { + block_producer: BlockProducer, + spec: Arc, + epoch_map: Arc, + keypair: Keypair, + beacon_node: Arc, + slot_clock: Arc>, + signer: Arc, +} + +impl Validator { + pub fn new() -> Self { + let spec = Arc::new(ChainSpec::foundation()); + let keypair = Keypair::random(); + let slot_clock = Arc::new(RwLock::new(TestingSlotClock::new(0))); + let signer = Arc::new(TestSigner::new(keypair.clone())); + let beacon_node = Arc::new(DirectBeaconNode()); + let epoch_map = Arc::new(TestEpochMap::new()); + + let block_producer = BlockProducer::new( + spec.clone(), + epoch_map.clone(), + slot_clock.clone(), + beacon_node.clone(), + signer.clone(), + ); + + Self { + block_producer, + spec, + epoch_map, + keypair, + beacon_node, + slot_clock, + signer, + } + } +} + +fn generate_validators(n: usize) -> Vec { + let mut validators = Vec::with_capacity(n); + for _ in 0..n { + validators.push(Validator::new()); + } + validators +} fn in_memory_test_stores() -> ( Arc, @@ -34,6 +99,7 @@ fn it_constructs() { let (_db, _chain) = in_memory_test_chain(ChainSpec::foundation()); } +/* #[test] fn it_produces() { let (_db, mut chain) = in_memory_test_chain(ChainSpec::foundation()); @@ -48,3 +114,4 @@ fn it_processes_a_block_it_produces() { assert_eq!(outcome, BlockProcessingOutcome::Processed); assert_eq!(chain.canonical_leaf_block, new_block_hash); } +*/ From d009b85637b36cd44af69e5b1d5ba4d6ca56c2ec Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 24 Jan 2019 11:52:19 +1100 Subject: [PATCH 013/107] Export bls::Keypair from types crate --- eth2/types/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index e23fe2ebc..4a82ec0be 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -67,4 +67,4 @@ pub type AttesterMap = HashMap<(u64, u64), Vec>; /// Maps a slot to a block proposer. pub type ProposerMap = HashMap; -pub use bls::{AggregatePublicKey, AggregateSignature, PublicKey, Signature}; +pub use bls::{AggregatePublicKey, AggregateSignature, Keypair, PublicKey, Signature}; From b29934aed4828448b37b43f8d6c3d2d0b1a153c8 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 24 Jan 2019 12:10:03 +1100 Subject: [PATCH 014/107] Update state transition as per spec --- beacon_node/beacon_chain/src/block_production.rs | 8 ++++++-- beacon_node/beacon_chain/src/state_transition.rs | 9 ++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/beacon_node/beacon_chain/src/block_production.rs b/beacon_node/beacon_chain/src/block_production.rs index f6219fc87..50524410f 100644 --- a/beacon_node/beacon_chain/src/block_production.rs +++ b/beacon_node/beacon_chain/src/block_production.rs @@ -4,7 +4,7 @@ use bls::Signature; use slot_clock::TestingSlotClockError; use types::{ readers::{BeaconBlockReader, BeaconStateReader}, - BeaconBlock, BeaconBlockBody, BeaconState, Hash256, + BeaconBlock, BeaconBlockBody, BeaconState, Eth1Data, Hash256, }; #[derive(Debug, PartialEq)] @@ -46,7 +46,11 @@ where parent_root, state_root: Hash256::zero(), // Updated after the state is calculated. randao_reveal: randao_reveal, - candidate_pow_receipt_root: Hash256::zero(), // TODO: replace w/ eth1 data. + eth1_data: Eth1Data { + // TODO: replace with real data + deposit_root: Hash256::zero(), + block_hash: Hash256::zero(), + }, signature: self.spec.empty_signature.clone(), // To be completed by a validator. body: BeaconBlockBody { proposer_slashings: vec![], diff --git a/beacon_node/beacon_chain/src/state_transition.rs b/beacon_node/beacon_chain/src/state_transition.rs index afa805a23..4a41d6f2c 100644 --- a/beacon_node/beacon_chain/src/state_transition.rs +++ b/beacon_node/beacon_chain/src/state_transition.rs @@ -4,9 +4,8 @@ use boolean_bitfield::BooleanBitfield; use slot_clock::{SystemTimeSlotClockError, TestingSlotClockError}; use ssz::ssz_encode; use types::{ - readers::{BeaconBlockReader, BeaconStateReader}, - AttestationData, AttestationDataAndCustodyBit, BeaconBlock, BeaconState, Exit, ForkData, - Hash256, PendingAttestationRecord, ProposalSignedData, + readers::BeaconBlockReader, AttestationData, AttestationDataAndCustodyBit, BeaconBlock, + BeaconState, Exit, Fork, Hash256, PendingAttestation, ProposalSignedData, }; // TODO: define elsehwere. @@ -283,7 +282,7 @@ where attestation.data.shard_block_root == self.spec.zero_hash, Error::BadAttestation ); - let pending_attestation = PendingAttestationRecord { + let pending_attestation = PendingAttestation { data: attestation.data.clone(), aggregation_bitfield: attestation.aggregation_bitfield.clone(), custody_bitfield: attestation.custody_bitfield.clone(), @@ -426,7 +425,7 @@ fn hash(_input: &T) -> Hash256 { Hash256::zero() } -fn get_domain(_fork: &ForkData, _slot: u64, _domain_type: u64) -> u64 { +fn get_domain(_fork: &Fork, _slot: u64, _domain_type: u64) -> u64 { // TODO: stubbed out. 0 } From 6fd3a1a03e0cb272409b1c7fda2e06e2ffdc4772 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 24 Jan 2019 17:05:48 +1100 Subject: [PATCH 015/107] Add progress on test rig --- .../beacon_chain/src/attestation_targets.rs | 50 +++++++++ beacon_node/beacon_chain/src/block_graph.rs | 37 +++++++ .../beacon_chain/src/block_processing.rs | 22 ++-- .../beacon_chain/src/block_production.rs | 16 +-- .../beacon_chain/src/canonical_head.rs | 34 ++++++ beacon_node/beacon_chain/src/info.rs | 49 +++++++++ beacon_node/beacon_chain/src/lib.rs | 96 ++++++++++++----- beacon_node/beacon_chain/src/lmd_ghost.rs | 16 +-- beacon_node/beacon_chain/tests/chain.rs | 101 +++--------------- .../tests/utils/direct_beacon_node.rs | 58 ++++++++++ .../beacon_chain/tests/utils/direct_duties.rs | 37 +++++++ beacon_node/beacon_chain/tests/utils/mod.rs | 9 ++ .../beacon_chain/tests/utils/test_rig.rs | 85 +++++++++++++++ .../beacon_chain/tests/utils/validator.rs | 61 +++++++++++ eth2/block_producer/Cargo.toml | 1 + eth2/block_producer/src/lib.rs | 26 ++++- .../src/test_utils/beacon_node.rs | 26 ++++- eth2/block_producer/src/traits.rs | 13 ++- eth2/spec/src/lib.rs | 2 +- 19 files changed, 595 insertions(+), 144 deletions(-) create mode 100644 beacon_node/beacon_chain/src/attestation_targets.rs create mode 100644 beacon_node/beacon_chain/src/block_graph.rs create mode 100644 beacon_node/beacon_chain/src/canonical_head.rs create mode 100644 beacon_node/beacon_chain/src/info.rs create mode 100644 beacon_node/beacon_chain/tests/utils/direct_beacon_node.rs create mode 100644 beacon_node/beacon_chain/tests/utils/direct_duties.rs create mode 100644 beacon_node/beacon_chain/tests/utils/mod.rs create mode 100644 beacon_node/beacon_chain/tests/utils/test_rig.rs create mode 100644 beacon_node/beacon_chain/tests/utils/validator.rs diff --git a/beacon_node/beacon_chain/src/attestation_targets.rs b/beacon_node/beacon_chain/src/attestation_targets.rs new file mode 100644 index 000000000..6971d5b4e --- /dev/null +++ b/beacon_node/beacon_chain/src/attestation_targets.rs @@ -0,0 +1,50 @@ +use crate::{BeaconChain, CheckPoint, ClientDB, SlotClock}; +use std::collections::HashMap; +use std::sync::RwLockReadGuard; +use types::{BeaconBlock, BeaconState, Hash256}; + +pub struct AttestationTargets { + map: HashMap, +} + +impl AttestationTargets { + pub fn new() -> Self { + Self { + map: HashMap::new(), + } + } + + pub fn get(&self, validator_index: u64) -> Option<&Hash256> { + self.map.get(&validator_index) + } + + pub fn insert(&mut self, validator_index: u64, block_hash: Hash256) -> Option { + self.map.insert(validator_index, block_hash) + } +} + +impl BeaconChain +where + T: ClientDB, + U: SlotClock, +{ + pub fn insert_latest_attestation_target(&self, validator_index: u64, block_root: Hash256) { + let mut targets = self + .latest_attestation_targets + .write() + .expect("CRITICAL: CanonicalHead poisioned."); + targets.insert(validator_index, block_root); + } + + pub fn get_latest_attestation_target(&self, validator_index: u64) -> Option { + let targets = self + .latest_attestation_targets + .read() + .expect("CRITICAL: CanonicalHead poisioned."); + + match targets.get(validator_index) { + Some(hash) => Some(hash.clone()), + None => None, + } + } +} diff --git a/beacon_node/beacon_chain/src/block_graph.rs b/beacon_node/beacon_chain/src/block_graph.rs new file mode 100644 index 000000000..8a0f6e61d --- /dev/null +++ b/beacon_node/beacon_chain/src/block_graph.rs @@ -0,0 +1,37 @@ +use crate::{BeaconChain, CheckPoint, ClientDB, SlotClock}; +use std::collections::HashSet; +use std::sync::{RwLock, RwLockReadGuard}; +use types::{BeaconBlock, BeaconState, Hash256}; + +pub struct BlockGraph { + pub leaves: RwLock>, +} + +impl BlockGraph { + pub fn new() -> Self { + Self { + leaves: RwLock::new(HashSet::new()), + } + } + /// Add a new leaf to the block hash graph. Returns `true` if the leaf was built upon another + /// leaf. + pub fn add_leaf(&self, parent: &Hash256, leaf: Hash256) -> bool { + let mut leaves = self + .leaves + .write() + .expect("CRITICAL: BlockGraph poisioned."); + + if leaves.contains(parent) { + leaves.remove(parent); + leaves.insert(leaf); + true + } else { + leaves.insert(leaf); + false + } + } + + pub fn leaves(&self) -> RwLockReadGuard> { + self.leaves.read().expect("CRITICAL: BlockGraph poisioned.") + } +} diff --git a/beacon_node/beacon_chain/src/block_processing.rs b/beacon_node/beacon_chain/src/block_processing.rs index 6c534bbce..f99958dfb 100644 --- a/beacon_node/beacon_chain/src/block_processing.rs +++ b/beacon_node/beacon_chain/src/block_processing.rs @@ -54,7 +54,7 @@ where U: SlotClock, Error: From<::Error>, { - pub fn process_block(&mut self, block: V) -> Result + pub fn process_block(&self, block: V) -> Result where V: BeaconBlockReader + Encodable + Sized, { @@ -103,14 +103,20 @@ where self.block_store.put(&block_root, &ssz_encode(&block)[..])?; self.state_store.put(&state_root, &ssz_encode(&state)[..])?; - // Update leaf blocks so the implementation can track the chain heads. - if self.leaf_blocks.contains(&block.parent_root()) { - self.leaf_blocks.remove(&block.parent_root()); + self.block_graph + .add_leaf(&parent_block_root, block_root.clone()); + + // If the parent block was the parent_block, automatically update the canonical head. + // + // TODO: this is a first-in-best-dressed scenario that is not ideal -- find a solution. + if self.canonical_head().beacon_block_root == parent_block_root { + self.update_canonical_head( + block.clone(), + block_root.clone(), + state.clone(), + state_root.clone(), + ); } - if self.canonical_leaf_block == block.parent_root() { - self.canonical_leaf_block = block_root; - } - self.leaf_blocks.insert(block_root); // The block was sucessfully processed. Ok(Outcome::Processed) diff --git a/beacon_node/beacon_chain/src/block_production.rs b/beacon_node/beacon_chain/src/block_production.rs index 50524410f..c0a057ad2 100644 --- a/beacon_node/beacon_chain/src/block_production.rs +++ b/beacon_node/beacon_chain/src/block_production.rs @@ -18,18 +18,22 @@ impl BeaconChain where T: ClientDB, U: SlotClock, - Error: From<::Error>, { pub fn produce_block( - &mut self, + &self, randao_reveal: Signature, - ) -> Result<(BeaconBlock, BeaconState), Error> { + ) -> Result<(BeaconBlock, BeaconState), Error> + where + Error: From<::Error>, + { + // TODO: allow producing a block from a previous (or future?) slot. let present_slot = self .slot_clock - .present_slot()? + .present_slot() + .map_err(|e| e.into())? .ok_or(Error::PresentSlotIsNone)?; - let parent_root = self.canonical_leaf_block; + let parent_root = self.canonical_head().beacon_block_root; let parent_block_reader = self .block_store .get_reader(&parent_root)? @@ -43,7 +47,7 @@ where let mut block = BeaconBlock { slot: present_slot, - parent_root, + parent_root: parent_root.clone(), state_root: Hash256::zero(), // Updated after the state is calculated. randao_reveal: randao_reveal, eth1_data: Eth1Data { diff --git a/beacon_node/beacon_chain/src/canonical_head.rs b/beacon_node/beacon_chain/src/canonical_head.rs new file mode 100644 index 000000000..acccb5de7 --- /dev/null +++ b/beacon_node/beacon_chain/src/canonical_head.rs @@ -0,0 +1,34 @@ +use crate::{BeaconChain, CheckPoint, ClientDB, SlotClock}; +use std::sync::RwLockReadGuard; +use types::{BeaconBlock, BeaconState, Hash256}; + +impl BeaconChain +where + T: ClientDB, + U: SlotClock, +{ + pub fn update_canonical_head( + &self, + new_beacon_block: BeaconBlock, + new_beacon_block_root: Hash256, + new_beacon_state: BeaconState, + new_beacon_state_root: Hash256, + ) { + let mut canonical_head = self + .canonical_head + .write() + .expect("CRITICAL: CanonicalHead poisioned."); + canonical_head.update( + new_beacon_block, + new_beacon_block_root, + new_beacon_state, + new_beacon_state_root, + ); + } + + pub fn canonical_head(&self) -> RwLockReadGuard { + self.canonical_head + .read() + .expect("CRITICAL: CanonicalHead poisioned.") + } +} diff --git a/beacon_node/beacon_chain/src/info.rs b/beacon_node/beacon_chain/src/info.rs new file mode 100644 index 000000000..2025ba27a --- /dev/null +++ b/beacon_node/beacon_chain/src/info.rs @@ -0,0 +1,49 @@ +use super::{BeaconChain, ClientDB, SlotClock}; +use types::PublicKey; + +impl BeaconChain +where + T: ClientDB, + U: SlotClock, +{ + pub fn validator_index(&self, pubkey: &PublicKey) -> Option { + for (i, validator) in self + .canonical_head() + .beacon_state + .validator_registry + .iter() + .enumerate() + { + if validator.pubkey == *pubkey { + return Some(i); + } + } + None + } + + pub fn proposer_slots(&self, validator_index: usize) -> Option { + if let Some(validator) = self + .canonical_head() + .beacon_state + .validator_registry + .get(validator_index) + { + Some(validator.proposer_slots) + } else { + None + } + } + + pub fn present_slot(&self) -> Option { + match self.slot_clock.present_slot() { + Ok(some_slot) => some_slot, + _ => None, + } + } + + pub fn block_proposer(&self, slot: u64) -> Option { + //TODO: this is a stub; fix. + let validator_count = self.canonical_head().beacon_state.validator_registry.len(); + Some((slot as usize) % validator_count) + } +} diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 6c95fcaa4..b52dfdd73 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -1,8 +1,14 @@ +mod attestation_targets; +mod block_graph; mod block_processing; -mod block_production; +pub mod block_production; +mod canonical_head; +mod info; mod lmd_ghost; mod state_transition; +use self::attestation_targets::AttestationTargets; +use self::block_graph::BlockGraph; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, ClientDB, DBError, @@ -12,18 +18,11 @@ use slot_clock::SlotClock; use spec::ChainSpec; use ssz::ssz_encode; use std::collections::{HashMap, HashSet}; -use std::sync::Arc; -use types::Hash256; +use std::sync::{Arc, RwLock}; +use types::{BeaconBlock, BeaconState, Hash256, PublicKey}; pub use self::block_processing::Outcome as BlockProcessingOutcome; -#[derive(Debug, PartialEq)] -pub struct CheckPoint { - block_root: Hash256, - state_root: Hash256, - slot: u64, -} - #[derive(Debug, PartialEq)] pub enum BeaconChainError { InsufficientValidators, @@ -31,15 +30,51 @@ pub enum BeaconChainError { DBError(String), } +pub struct CheckPoint { + beacon_block: BeaconBlock, + beacon_block_root: Hash256, + beacon_state: BeaconState, + beacon_state_root: Hash256, +} + +impl CheckPoint { + pub fn new( + beacon_block: BeaconBlock, + beacon_block_root: Hash256, + beacon_state: BeaconState, + beacon_state_root: Hash256, + ) -> Self { + Self { + beacon_block, + beacon_block_root, + beacon_state, + beacon_state_root, + } + } + + pub fn update( + &mut self, + beacon_block: BeaconBlock, + beacon_block_root: Hash256, + beacon_state: BeaconState, + beacon_state_root: Hash256, + ) { + self.beacon_block = beacon_block; + self.beacon_block_root = beacon_block_root; + self.beacon_state = beacon_state; + self.beacon_state_root = beacon_state_root; + } +} + pub struct BeaconChain { pub block_store: Arc>, pub state_store: Arc>, pub slot_clock: U, - pub leaf_blocks: HashSet, - pub canonical_leaf_block: Hash256, + pub block_graph: BlockGraph, + canonical_head: RwLock, + finalized_head: RwLock, + pub latest_attestation_targets: RwLock, pub spec: ChainSpec, - latest_attestation_targets: HashMap, - finalized_checkpoint: CheckPoint, } impl BeaconChain @@ -65,24 +100,33 @@ where let block_root = genesis_block.canonical_root(); block_store.put(&block_root, &ssz_encode(&genesis_block)[..])?; - let mut leaf_blocks = HashSet::new(); - leaf_blocks.insert(block_root); + let block_graph = BlockGraph::new(); + block_graph.add_leaf(&Hash256::zero(), block_root.clone()); - let finalized_checkpoint = CheckPoint { - block_root, - state_root, - slot: genesis_block.slot, - }; + let finalized_head = RwLock::new(CheckPoint::new( + genesis_block.clone(), + block_root.clone(), + genesis_state.clone(), + state_root.clone(), + )); + let canonical_head = RwLock::new(CheckPoint::new( + genesis_block.clone(), + block_root.clone(), + genesis_state.clone(), + state_root.clone(), + )); + + let latest_attestation_targets = RwLock::new(AttestationTargets::new()); Ok(Self { block_store, state_store, slot_clock, - leaf_blocks, - canonical_leaf_block: block_root, - spec, - latest_attestation_targets: HashMap::new(), - finalized_checkpoint, + block_graph, + finalized_head, + canonical_head, + latest_attestation_targets, + spec: spec, }) } } diff --git a/beacon_node/beacon_chain/src/lmd_ghost.rs b/beacon_node/beacon_chain/src/lmd_ghost.rs index 7f00a9276..e4aaa3483 100644 --- a/beacon_node/beacon_chain/src/lmd_ghost.rs +++ b/beacon_node/beacon_chain/src/lmd_ghost.rs @@ -4,13 +4,12 @@ use db::{ ClientDB, DBError, }; use slot_clock::TestingSlotClockError; -use ssz::{ssz_encode, Encodable}; use std::collections::HashSet; use std::sync::Arc; use types::{ readers::{BeaconBlockReader, BeaconStateReader}, validator_registry::get_active_validator_indices, - BeaconBlock, Hash256, + Hash256, }; #[derive(Debug, PartialEq)] @@ -53,7 +52,7 @@ where let mut attestation_targets = Vec::with_capacity(active_validator_indices.len()); for i in active_validator_indices { - if let Some(target) = self.latest_attestation_targets.get(&i) { + if let Some(target) = self.get_latest_attestation_target(i as u64) { attestation_targets.push(target); } } @@ -62,8 +61,11 @@ where let mut head_vote_count = 0; loop { - let child_hashes_and_slots = - get_child_hashes_and_slots(&self.block_store, &head_hash, &self.leaf_blocks)?; + let child_hashes_and_slots = get_child_hashes_and_slots( + &self.block_store, + &head_hash, + &self.block_graph.leaves(), + )?; if child_hashes_and_slots.len() == 0 { break; @@ -90,7 +92,7 @@ where fn get_vote_count( block_store: &Arc>, - attestation_targets: &[&Hash256], + attestation_targets: &[Hash256], block_root: &Hash256, slot: u64, ) -> Result { @@ -99,7 +101,7 @@ fn get_vote_count( let (root_at_slot, _) = block_store .block_at_slot(&block_root, slot)? .ok_or(Error::MissingBeaconBlock(*block_root))?; - if root_at_slot == *block_root { + if root_at_slot == *target { count += 1; } } diff --git a/beacon_node/beacon_chain/tests/chain.rs b/beacon_node/beacon_chain/tests/chain.rs index 390d48ea4..3c0a9b8f9 100644 --- a/beacon_node/beacon_chain/tests/chain.rs +++ b/beacon_node/beacon_chain/tests/chain.rs @@ -1,9 +1,7 @@ -use beacon_chain::{BeaconChain, BlockProcessingOutcome}; +use self::utils::TestRig; +use beacon_chain::BeaconChain; #[cfg(test)] -use block_producer::{ - test_utils::{TestEpochMap, TestSigner}, - BeaconNode as BeaconBlockNode, BeaconNodeError as BeaconBlockNodeError, BlockProducer, -}; +use block_producer::{test_utils::TestSigner, BlockProducer}; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, MemoryDB, @@ -11,92 +9,19 @@ use db::{ use slot_clock::TestingSlotClock; use spec::ChainSpec; use std::sync::{Arc, RwLock}; -use types::{BeaconBlock, Keypair}; +use types::{Keypair, Validator}; -struct DirectBeaconNode(); - -impl BeaconBlockNode for DirectBeaconNode { - fn produce_beacon_block(&self, slot: u64) -> Result, BeaconBlockNodeError> { - Err(BeaconBlockNodeError::DecodeFailure) - } - - /// Returns the value specified by the `set_next_publish_result`. - fn publish_beacon_block(&self, block: BeaconBlock) -> Result { - Err(BeaconBlockNodeError::DecodeFailure) - } -} - -struct Validator { - block_producer: BlockProducer, - spec: Arc, - epoch_map: Arc, - keypair: Keypair, - beacon_node: Arc, - slot_clock: Arc>, - signer: Arc, -} - -impl Validator { - pub fn new() -> Self { - let spec = Arc::new(ChainSpec::foundation()); - let keypair = Keypair::random(); - let slot_clock = Arc::new(RwLock::new(TestingSlotClock::new(0))); - let signer = Arc::new(TestSigner::new(keypair.clone())); - let beacon_node = Arc::new(DirectBeaconNode()); - let epoch_map = Arc::new(TestEpochMap::new()); - - let block_producer = BlockProducer::new( - spec.clone(), - epoch_map.clone(), - slot_clock.clone(), - beacon_node.clone(), - signer.clone(), - ); - - Self { - block_producer, - spec, - epoch_map, - keypair, - beacon_node, - slot_clock, - signer, - } - } -} - -fn generate_validators(n: usize) -> Vec { - let mut validators = Vec::with_capacity(n); - for _ in 0..n { - validators.push(Validator::new()); - } - validators -} - -fn in_memory_test_stores() -> ( - Arc, - Arc>, - Arc>, -) { - let db = Arc::new(MemoryDB::open()); - let block_store = Arc::new(BeaconBlockStore::new(db.clone())); - let state_store = Arc::new(BeaconStateStore::new(db.clone())); - (db, block_store, state_store) -} - -fn in_memory_test_chain( - spec: ChainSpec, -) -> (Arc, BeaconChain) { - let (db, block_store, state_store) = in_memory_test_stores(); - let slot_clock = TestingSlotClock::new(0); - - let chain = BeaconChain::genesis(state_store, block_store, slot_clock, spec); - (db, chain.unwrap()) -} +mod utils; #[test] -fn it_constructs() { - let (_db, _chain) = in_memory_test_chain(ChainSpec::foundation()); +fn rig_can_generate_validators() { + /* + let (_db, mut chain) = in_memory_test_chain(ChainSpec::foundation()); + let validators = generate_validators(2, &chain); + chain.spec = inject_validators_into_spec(chain.spec.clone(), &validators[..]); + */ + let mut rig = TestRig::new(ChainSpec::foundation()); + rig.generate_validators(2); } /* diff --git a/beacon_node/beacon_chain/tests/utils/direct_beacon_node.rs b/beacon_node/beacon_chain/tests/utils/direct_beacon_node.rs new file mode 100644 index 000000000..130a4d136 --- /dev/null +++ b/beacon_node/beacon_chain/tests/utils/direct_beacon_node.rs @@ -0,0 +1,58 @@ +use beacon_chain::{block_production::Error as BlockProductionError, BeaconChain}; +use block_producer::{BeaconNode as BeaconBlockNode, BeaconNodeError as BeaconBlockNodeError}; +use db::ClientDB; +use slot_clock::SlotClock; +use types::{BeaconBlock, PublicKey, Signature}; + +pub struct DirectBeaconNode<'a, T: ClientDB, U: SlotClock> { + beacon_chain: &'a BeaconChain, +} + +impl<'a, T: ClientDB, U: SlotClock> DirectBeaconNode<'a, T, U> { + pub fn new(beacon_chain: &'a BeaconChain) -> Self { + Self { beacon_chain } + } +} + +impl<'a, T: ClientDB, U: SlotClock> BeaconBlockNode for DirectBeaconNode<'a, T, U> +where + BlockProductionError: From<::Error>, +{ + fn proposer_nonce(&self, pubkey: &PublicKey) -> Result { + let validator_index = self + .beacon_chain + .validator_index(pubkey) + .ok_or_else(|| BeaconBlockNodeError::RemoteFailure("pubkey unknown.".to_string()))?; + + self.beacon_chain + .proposer_slots(validator_index) + .ok_or_else(|| { + BeaconBlockNodeError::RemoteFailure("validator_index unknown.".to_string()) + }) + } + + fn produce_beacon_block( + &self, + slot: u64, + randao_reveal: &Signature, + ) -> Result, BeaconBlockNodeError> +where { + let (block, _state) = self + .beacon_chain + .produce_block(randao_reveal.clone()) + .map_err(|e| BeaconBlockNodeError::RemoteFailure(format!("{:?}", e)))?; + + if block.slot == slot { + Ok(Some(block)) + } else { + Err(BeaconBlockNodeError::RemoteFailure( + "Unable to produce at non-current slot.".to_string(), + )) + } + } + + /// Returns the value specified by the `set_next_publish_result`. + fn publish_beacon_block(&self, block: BeaconBlock) -> Result { + Err(BeaconBlockNodeError::DecodeFailure) + } +} diff --git a/beacon_node/beacon_chain/tests/utils/direct_duties.rs b/beacon_node/beacon_chain/tests/utils/direct_duties.rs new file mode 100644 index 000000000..75177ac14 --- /dev/null +++ b/beacon_node/beacon_chain/tests/utils/direct_duties.rs @@ -0,0 +1,37 @@ +use beacon_chain::{block_production::Error as BlockProductionError, BeaconChain}; +use block_producer::{DutiesReader, DutiesReaderError}; +use db::ClientDB; +use slot_clock::SlotClock; +use types::PublicKey; + +pub struct DirectDuties<'a, T: ClientDB, U: SlotClock> { + beacon_chain: &'a BeaconChain, + pubkey: PublicKey, +} + +impl<'a, T: ClientDB, U: SlotClock> DirectDuties<'a, T, U> { + pub fn new(pubkey: PublicKey, beacon_chain: &'a BeaconChain) -> Self { + Self { + beacon_chain, + pubkey, + } + } +} + +impl<'a, T: ClientDB, U: SlotClock> DutiesReader for DirectDuties<'a, T, U> +where + BlockProductionError: From<::Error>, +{ + fn is_block_production_slot(&self, _epoch: u64, slot: u64) -> Result { + let validator_index = self + .beacon_chain + .validator_index(&self.pubkey) + .ok_or_else(|| DutiesReaderError::UnknownValidator)?; + + match self.beacon_chain.block_proposer(slot) { + Some(proposer) if proposer == validator_index => Ok(true), + Some(_) => Ok(false), + None => Err(DutiesReaderError::UnknownEpoch), + } + } +} diff --git a/beacon_node/beacon_chain/tests/utils/mod.rs b/beacon_node/beacon_chain/tests/utils/mod.rs new file mode 100644 index 000000000..6460fe421 --- /dev/null +++ b/beacon_node/beacon_chain/tests/utils/mod.rs @@ -0,0 +1,9 @@ +mod direct_beacon_node; +mod direct_duties; +mod test_rig; +mod validator; + +pub use self::direct_beacon_node::DirectBeaconNode; +pub use self::direct_duties::DirectDuties; +pub use self::test_rig::TestRig; +pub use self::validator::TestValidator; diff --git a/beacon_node/beacon_chain/tests/utils/test_rig.rs b/beacon_node/beacon_chain/tests/utils/test_rig.rs new file mode 100644 index 000000000..0bdf0a36b --- /dev/null +++ b/beacon_node/beacon_chain/tests/utils/test_rig.rs @@ -0,0 +1,85 @@ +use super::{DirectBeaconNode, DirectDuties, TestValidator}; +use beacon_chain::BeaconChain; +#[cfg(test)] +use block_producer::{test_utils::TestSigner, BlockProducer}; +use db::{ + stores::{BeaconBlockStore, BeaconStateStore}, + MemoryDB, +}; +use slot_clock::TestingSlotClock; +use spec::ChainSpec; +use std::sync::{Arc, RwLock}; +use types::{Keypair, Validator}; + +pub struct TestRig<'a> { + db: Arc, + beacon_chain: BeaconChain, + block_store: Arc>, + state_store: Arc>, + validators: Vec>, +} + +impl<'a> TestRig<'a> { + pub fn new(spec: ChainSpec) -> Self { + let db = Arc::new(MemoryDB::open()); + let block_store = Arc::new(BeaconBlockStore::new(db.clone())); + let state_store = Arc::new(BeaconStateStore::new(db.clone())); + + let slot_clock = TestingSlotClock::new(0); + + let mut beacon_chain = + BeaconChain::genesis(state_store.clone(), block_store.clone(), slot_clock, spec) + .unwrap(); + + /* + let validators = generate_validators(validator_count, &beacon_chain); + beacon_chain.spec = inject_validators_into_spec(beacon_chain.spec.clone(), &validators[..]); + */ + + Self { + db, + beacon_chain, + block_store, + state_store, + validators: vec![], + } + } + + pub fn generate_validators(&'a mut self, validator_count: usize) { + self.validators = Vec::with_capacity(validator_count); + for _ in 0..validator_count { + self.validators.push(TestValidator::new(&self.beacon_chain)); + } + self.beacon_chain.spec = + inject_validators_into_spec(self.beacon_chain.spec.clone(), &self.validators[..]); + } + + pub fn process_next_slot(&mut self) { + let slot = self + .beacon_chain + .present_slot() + .expect("Unable to determine slot.") + + 1; + self.beacon_chain.slot_clock.set_slot(slot); + + let block_proposer = self + .beacon_chain + .block_proposer(slot) + .expect("Unable to determine proposer."); + + let validator = self + .validators + .get(block_proposer) + .expect("Block proposer unknown"); + } +} + +fn inject_validators_into_spec(mut spec: ChainSpec, validators: &[TestValidator]) -> ChainSpec { + spec.initial_validators = Vec::with_capacity(validators.len()); + spec.initial_balances = Vec::with_capacity(validators.len()); + for validator in validators { + spec.initial_validators.push(validator.validator_record()); + spec.initial_balances.push(32_000_000_000); // 32 ETH + } + spec +} diff --git a/beacon_node/beacon_chain/tests/utils/validator.rs b/beacon_node/beacon_chain/tests/utils/validator.rs new file mode 100644 index 000000000..a6a348dce --- /dev/null +++ b/beacon_node/beacon_chain/tests/utils/validator.rs @@ -0,0 +1,61 @@ +use super::{DirectBeaconNode, DirectDuties}; +use beacon_chain::BeaconChain; +#[cfg(test)] +use block_producer::{test_utils::TestSigner, BlockProducer}; +use db::MemoryDB; +use slot_clock::TestingSlotClock; +use spec::ChainSpec; +use std::sync::{Arc, RwLock}; +use types::{Keypair, Validator}; + +pub struct TestValidator<'a> { + block_producer: BlockProducer< + TestingSlotClock, + DirectBeaconNode<'a, MemoryDB, TestingSlotClock>, + DirectDuties<'a, MemoryDB, TestingSlotClock>, + TestSigner, + >, + spec: Arc, + epoch_map: Arc>, + keypair: Keypair, + beacon_node: Arc>, + slot_clock: Arc>, + signer: Arc, +} + +impl<'a> TestValidator<'a> { + pub fn new(beacon_chain: &'a BeaconChain) -> Self { + let spec = Arc::new(ChainSpec::foundation()); + let keypair = Keypair::random(); + let slot_clock = Arc::new(RwLock::new(TestingSlotClock::new(0))); + let signer = Arc::new(TestSigner::new(keypair.clone())); + let beacon_node = Arc::new(DirectBeaconNode::new(beacon_chain)); + let epoch_map = Arc::new(DirectDuties::new(keypair.pk.clone(), beacon_chain)); + + let block_producer = BlockProducer::new( + spec.clone(), + keypair.pk.clone(), + epoch_map.clone(), + slot_clock.clone(), + beacon_node.clone(), + signer.clone(), + ); + + Self { + block_producer, + spec, + epoch_map, + keypair, + beacon_node, + slot_clock, + signer, + } + } + + pub fn validator_record(&self) -> Validator { + Validator { + pubkey: self.keypair.pk.clone(), + ..std::default::Default::default() + } + } +} diff --git a/eth2/block_producer/Cargo.toml b/eth2/block_producer/Cargo.toml index 76ea78ea3..7203dbe31 100644 --- a/eth2/block_producer/Cargo.toml +++ b/eth2/block_producer/Cargo.toml @@ -7,4 +7,5 @@ edition = "2018" [dependencies] slot_clock = { path = "../../eth2/utils/slot_clock" } spec = { path = "../../eth2/spec" } +ssz = { path = "../../eth2/utils/ssz" } types = { path = "../../eth2/types" } diff --git a/eth2/block_producer/src/lib.rs b/eth2/block_producer/src/lib.rs index fc06ffa46..48c36fa9f 100644 --- a/eth2/block_producer/src/lib.rs +++ b/eth2/block_producer/src/lib.rs @@ -3,8 +3,9 @@ mod traits; use slot_clock::SlotClock; use spec::ChainSpec; +use ssz::ssz_encode; use std::sync::{Arc, RwLock}; -use types::{BeaconBlock, Hash256, ProposalSignedData}; +use types::{BeaconBlock, Hash256, ProposalSignedData, PublicKey}; pub use self::traits::{BeaconNode, BeaconNodeError, DutiesReader, DutiesReaderError, Signer}; @@ -24,6 +25,8 @@ pub enum PollOutcome { BeaconNodeUnableToProduceBlock(u64), /// The signer failed to sign the message. SignerRejection(u64), + /// The public key for this validator is not an active validator. + ValidatorIsUnknown(u64), } #[derive(Debug, PartialEq)] @@ -44,6 +47,7 @@ pub enum Error { /// Relies upon an external service to keep the `EpochDutiesMap` updated. pub struct BlockProducer { pub last_processed_slot: u64, + pubkey: PublicKey, spec: Arc, epoch_map: Arc, slot_clock: Arc>, @@ -55,6 +59,7 @@ impl BlockProducer, + pubkey: PublicKey, epoch_map: Arc, slot_clock: Arc>, beacon_node: Arc, @@ -62,6 +67,7 @@ impl BlockProducer Self { Self { last_processed_slot: 0, + pubkey, spec, epoch_map, slot_clock, @@ -96,6 +102,9 @@ impl BlockProducer { return Ok(PollOutcome::ProducerDutiesUnknown(slot)) } + Err(DutiesReaderError::UnknownValidator) => { + return Ok(PollOutcome::ValidatorIsUnknown(slot)) + } Err(DutiesReaderError::Poisoned) => return Err(Error::EpochMapPoisoned), }; @@ -122,7 +131,20 @@ impl BlockProducer Result { - if let Some(block) = self.beacon_node.produce_beacon_block(slot)? { + let randao_reveal = { + let producer_nonce = self.beacon_node.proposer_nonce(&self.pubkey)?; + // TODO: add domain, etc to this message. + let message = ssz_encode(&producer_nonce); + match self.signer.bls_sign(&message) { + None => return Ok(PollOutcome::SignerRejection(slot)), + Some(signature) => signature, + } + }; + + if let Some(block) = self + .beacon_node + .produce_beacon_block(slot, &randao_reveal)? + { if self.safe_to_produce(&block) { if let Some(block) = self.sign_block(block) { self.beacon_node.publish_beacon_block(block)?; diff --git a/eth2/block_producer/src/test_utils/beacon_node.rs b/eth2/block_producer/src/test_utils/beacon_node.rs index bca01d9c7..59527ce60 100644 --- a/eth2/block_producer/src/test_utils/beacon_node.rs +++ b/eth2/block_producer/src/test_utils/beacon_node.rs @@ -1,20 +1,30 @@ use crate::traits::{BeaconNode, BeaconNodeError}; use std::sync::RwLock; -use types::BeaconBlock; +use types::{BeaconBlock, PublicKey, Signature}; +type NonceResult = Result; type ProduceResult = Result, BeaconNodeError>; type PublishResult = Result; /// A test-only struct used to simulate a Beacon Node. #[derive(Default)] pub struct TestBeaconNode { - pub produce_input: RwLock>, + pub nonce_input: RwLock>, + pub nonce_result: RwLock>, + + pub produce_input: RwLock>, pub produce_result: RwLock>, + pub publish_input: RwLock>, pub publish_result: RwLock>, } impl TestBeaconNode { + /// Set the result to be returned when `produce_beacon_block` is called. + pub fn set_next_nonce_result(&self, result: NonceResult) { + *self.nonce_result.write().unwrap() = Some(result); + } + /// Set the result to be returned when `produce_beacon_block` is called. pub fn set_next_produce_result(&self, result: ProduceResult) { *self.produce_result.write().unwrap() = Some(result); @@ -27,9 +37,17 @@ impl TestBeaconNode { } impl BeaconNode for TestBeaconNode { + fn proposer_nonce(&self, pubkey: &PublicKey) -> NonceResult { + *self.nonce_input.write().unwrap() = Some(pubkey.clone()); + match *self.nonce_result.read().unwrap() { + Some(ref r) => r.clone(), + None => panic!("TestBeaconNode: nonce_result == None"), + } + } + /// Returns the value specified by the `set_next_produce_result`. - fn produce_beacon_block(&self, slot: u64) -> ProduceResult { - *self.produce_input.write().unwrap() = Some(slot); + fn produce_beacon_block(&self, slot: u64, randao_reveal: &Signature) -> ProduceResult { + *self.produce_input.write().unwrap() = Some((slot, randao_reveal.clone())); match *self.produce_result.read().unwrap() { Some(ref r) => r.clone(), None => panic!("TestBeaconNode: produce_result == None"), diff --git a/eth2/block_producer/src/traits.rs b/eth2/block_producer/src/traits.rs index c9716701b..0145328a3 100644 --- a/eth2/block_producer/src/traits.rs +++ b/eth2/block_producer/src/traits.rs @@ -1,4 +1,4 @@ -use types::{BeaconBlock, Signature}; +use types::{BeaconBlock, PublicKey, Signature}; #[derive(Debug, PartialEq, Clone)] pub enum BeaconNodeError { @@ -8,10 +8,18 @@ pub enum BeaconNodeError { /// Defines the methods required to produce and publish blocks on a Beacon Node. pub trait BeaconNode: Send + Sync { + /// Requests the proposer nonce (presently named `proposer_slots`). + fn proposer_nonce(&self, pubkey: &PublicKey) -> Result; + /// Request that the node produces a block. /// /// Returns Ok(None) if the Beacon Node is unable to produce at the given slot. - fn produce_beacon_block(&self, slot: u64) -> Result, BeaconNodeError>; + fn produce_beacon_block( + &self, + slot: u64, + randao_reveal: &Signature, + ) -> Result, BeaconNodeError>; + /// Request that the node publishes a block. /// /// Returns `true` if the publish was sucessful. @@ -20,6 +28,7 @@ pub trait BeaconNode: Send + Sync { #[derive(Debug, PartialEq, Clone)] pub enum DutiesReaderError { + UnknownValidator, UnknownEpoch, Poisoned, } diff --git a/eth2/spec/src/lib.rs b/eth2/spec/src/lib.rs index 3e7d5dd2b..aa3906076 100644 --- a/eth2/spec/src/lib.rs +++ b/eth2/spec/src/lib.rs @@ -6,7 +6,7 @@ mod foundation; use bls::Signature; use types::{Address, Eth1Data, Hash256, Validator}; -#[derive(PartialEq, Debug)] +#[derive(PartialEq, Debug, Clone)] pub struct ChainSpec { /* * Misc From 271a4cf8a74d2347fea92d07fa9a5c37bcabf5d0 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 25 Jan 2019 11:23:59 +1100 Subject: [PATCH 016/107] Add `get_deserialized` fns to block & state stores. --- beacon_node/db/src/stores/beacon_block_store.rs | 12 ++++++++++++ beacon_node/db/src/stores/beacon_state_store.rs | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/beacon_node/db/src/stores/beacon_block_store.rs b/beacon_node/db/src/stores/beacon_block_store.rs index 3ec558c93..fcdd595f5 100644 --- a/beacon_node/db/src/stores/beacon_block_store.rs +++ b/beacon_node/db/src/stores/beacon_block_store.rs @@ -26,6 +26,18 @@ impl BeaconBlockStore { Self { db } } + pub fn get_deserialized(&self, hash: &Hash256) -> Result, DBError> { + match self.get(&hash)? { + None => Ok(None), + Some(ssz) => { + let (block, _) = BeaconBlock::ssz_decode(&ssz, 0).map_err(|_| DBError { + message: "Bad BeaconBlock SSZ.".to_string(), + })?; + Ok(Some(block)) + } + } + } + /// Retuns an object implementing `BeaconBlockReader`, or `None` (if hash not known). /// /// Note: Presently, this function fully deserializes a `BeaconBlock` and returns that. In the diff --git a/beacon_node/db/src/stores/beacon_state_store.rs b/beacon_node/db/src/stores/beacon_state_store.rs index a54e19249..ed22696cb 100644 --- a/beacon_node/db/src/stores/beacon_state_store.rs +++ b/beacon_node/db/src/stores/beacon_state_store.rs @@ -19,6 +19,18 @@ impl BeaconStateStore { Self { db } } + pub fn get_deserialized(&self, hash: &Hash256) -> Result, DBError> { + match self.get(&hash)? { + None => Ok(None), + Some(ssz) => { + let (state, _) = BeaconState::ssz_decode(&ssz, 0).map_err(|_| DBError { + message: "Bad State SSZ.".to_string(), + })?; + Ok(Some(state)) + } + } + } + /// Retuns an object implementing `BeaconStateReader`, or `None` (if hash not known). /// /// Note: Presently, this function fully deserializes a `BeaconState` and returns that. In the From 5f953f76d8b62e6b8c8dd0a8fddc27e19410803c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 25 Jan 2019 11:25:09 +1100 Subject: [PATCH 017/107] Make TestingSlotClock thread-safe --- eth2/utils/slot_clock/src/testing_slot_clock.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/eth2/utils/slot_clock/src/testing_slot_clock.rs b/eth2/utils/slot_clock/src/testing_slot_clock.rs index de1d7ddb3..330d47f1a 100644 --- a/eth2/utils/slot_clock/src/testing_slot_clock.rs +++ b/eth2/utils/slot_clock/src/testing_slot_clock.rs @@ -1,11 +1,12 @@ use super::SlotClock; +use std::sync::RwLock; #[derive(Debug, PartialEq)] pub enum Error {} /// Determines the present slot based upon the present system time. pub struct TestingSlotClock { - slot: u64, + slot: RwLock, } impl TestingSlotClock { @@ -13,11 +14,13 @@ impl TestingSlotClock { /// /// Returns an Error if `slot_duration_seconds == 0`. pub fn new(slot: u64) -> TestingSlotClock { - TestingSlotClock { slot } + TestingSlotClock { + slot: RwLock::new(slot), + } } - pub fn set_slot(&mut self, slot: u64) { - self.slot = slot; + pub fn set_slot(&self, slot: u64) { + *self.slot.write().expect("TestingSlotClock poisoned.") = slot; } } @@ -25,7 +28,8 @@ impl SlotClock for TestingSlotClock { type Error = Error; fn present_slot(&self) -> Result, Error> { - Ok(Some(self.slot)) + let slot = *self.slot.read().expect("TestingSlotClock poisoned."); + Ok(Some(slot)) } } @@ -35,7 +39,7 @@ mod tests { #[test] fn test_slot_now() { - let mut clock = TestingSlotClock::new(10); + let clock = TestingSlotClock::new(10); assert_eq!(clock.present_slot(), Ok(Some(10))); clock.set_slot(123); assert_eq!(clock.present_slot(), Ok(Some(123))); From 5fdad686fac5793f59ee504299e2ce817e34a0c0 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 25 Jan 2019 11:25:56 +1100 Subject: [PATCH 018/107] Add test for `state_root` to genesis --- eth2/genesis/src/beacon_block.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/eth2/genesis/src/beacon_block.rs b/eth2/genesis/src/beacon_block.rs index e79e3d2f4..478d76ded 100644 --- a/eth2/genesis/src/beacon_block.rs +++ b/eth2/genesis/src/beacon_block.rs @@ -32,12 +32,13 @@ mod tests { use bls::Signature; #[test] - fn test_genesis() { + fn test_state_root() { let spec = ChainSpec::foundation(); let state_root = Hash256::from("cats".as_bytes()); - // This only checks that the function runs without panic. - genesis_beacon_block(state_root, &spec); + let block = genesis_beacon_block(state_root, &spec); + + assert_eq!(block.state_root, state_root); } #[test] From 643fc20063c9af9e6df9f4f1ebd4f347ce76c282 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 25 Jan 2019 11:29:41 +1100 Subject: [PATCH 019/107] Update block_producer for upstream changes --- eth2/block_producer/src/lib.rs | 46 +++++++++++-------- .../src/test_utils/beacon_node.rs | 4 +- .../src/test_utils/epoch_map.rs | 19 ++++++-- eth2/block_producer/src/traits.rs | 10 +++- 4 files changed, 52 insertions(+), 27 deletions(-) diff --git a/eth2/block_producer/src/lib.rs b/eth2/block_producer/src/lib.rs index 48c36fa9f..3cf610355 100644 --- a/eth2/block_producer/src/lib.rs +++ b/eth2/block_producer/src/lib.rs @@ -7,7 +7,9 @@ use ssz::ssz_encode; use std::sync::{Arc, RwLock}; use types::{BeaconBlock, Hash256, ProposalSignedData, PublicKey}; -pub use self::traits::{BeaconNode, BeaconNodeError, DutiesReader, DutiesReaderError, Signer}; +pub use self::traits::{ + BeaconNode, BeaconNodeError, DutiesReader, DutiesReaderError, PublishOutcome, Signer, +}; #[derive(Debug, PartialEq)] pub enum PollOutcome { @@ -46,11 +48,11 @@ pub enum Error { /// /// Relies upon an external service to keep the `EpochDutiesMap` updated. pub struct BlockProducer { - pub last_processed_slot: u64, + pub last_processed_slot: Option, pubkey: PublicKey, spec: Arc, epoch_map: Arc, - slot_clock: Arc>, + slot_clock: Arc, beacon_node: Arc, signer: Arc, } @@ -61,12 +63,12 @@ impl BlockProducer, pubkey: PublicKey, epoch_map: Arc, - slot_clock: Arc>, + slot_clock: Arc, beacon_node: Arc, signer: Arc, ) -> Self { Self { - last_processed_slot: 0, + last_processed_slot: None, pubkey, spec, epoch_map, @@ -84,8 +86,6 @@ impl BlockProducer Result { let slot = self .slot_clock - .read() - .map_err(|_| Error::SlotClockPoisoned)? .present_slot() .map_err(|_| Error::SlotClockError)? .ok_or(Error::SlotUnknowable)?; @@ -95,21 +95,20 @@ impl BlockProducer self.last_processed_slot { - let is_block_production_slot = - match self.epoch_map.is_block_production_slot(epoch, slot) { - Ok(result) => result, - Err(DutiesReaderError::UnknownEpoch) => { - return Ok(PollOutcome::ProducerDutiesUnknown(slot)) - } - Err(DutiesReaderError::UnknownValidator) => { - return Ok(PollOutcome::ValidatorIsUnknown(slot)) - } - Err(DutiesReaderError::Poisoned) => return Err(Error::EpochMapPoisoned), - }; + if !self.is_processed_slot(slot) { + let is_block_production_slot = match self.epoch_map.is_block_production_slot(slot) { + Ok(result) => result, + Err(DutiesReaderError::UnknownEpoch) => { + return Ok(PollOutcome::ProducerDutiesUnknown(slot)) + } + Err(DutiesReaderError::UnknownValidator) => { + return Ok(PollOutcome::ValidatorIsUnknown(slot)) + } + Err(DutiesReaderError::Poisoned) => return Err(Error::EpochMapPoisoned), + }; if is_block_production_slot { - self.last_processed_slot = slot; + self.last_processed_slot = Some(slot); self.produce_block(slot) } else { @@ -120,6 +119,13 @@ impl BlockProducer bool { + match self.last_processed_slot { + Some(processed_slot) if processed_slot <= slot => true, + _ => false, + } + } + /// Produce a block at some slot. /// /// Assumes that a block is required at this slot (does not check the duties). diff --git a/eth2/block_producer/src/test_utils/beacon_node.rs b/eth2/block_producer/src/test_utils/beacon_node.rs index 59527ce60..f132ce6ec 100644 --- a/eth2/block_producer/src/test_utils/beacon_node.rs +++ b/eth2/block_producer/src/test_utils/beacon_node.rs @@ -1,10 +1,10 @@ -use crate::traits::{BeaconNode, BeaconNodeError}; +use crate::traits::{BeaconNode, BeaconNodeError, PublishOutcome}; use std::sync::RwLock; use types::{BeaconBlock, PublicKey, Signature}; type NonceResult = Result; type ProduceResult = Result, BeaconNodeError>; -type PublishResult = Result; +type PublishResult = Result; /// A test-only struct used to simulate a Beacon Node. #[derive(Default)] diff --git a/eth2/block_producer/src/test_utils/epoch_map.rs b/eth2/block_producer/src/test_utils/epoch_map.rs index 290660afd..4cb1fb9fb 100644 --- a/eth2/block_producer/src/test_utils/epoch_map.rs +++ b/eth2/block_producer/src/test_utils/epoch_map.rs @@ -1,11 +1,24 @@ use crate::{DutiesReader, DutiesReaderError}; use std::collections::HashMap; -pub type TestEpochMap = HashMap; +pub struct TestEpochMap { + epoch_length: u64, + map: HashMap, +} + +impl TestEpochMap { + fn new(epoch_length: u64) -> Self { + Self { + epoch_length, + map: HashMap::new(), + } + } +} impl DutiesReader for TestEpochMap { - fn is_block_production_slot(&self, epoch: u64, slot: u64) -> Result { - match self.get(&epoch) { + fn is_block_production_slot(&self, slot: u64) -> Result { + let epoch = slot / self.epoch_length; + match self.map.get(&epoch) { Some(s) if *s == slot => Ok(true), Some(s) if *s != slot => Ok(false), _ => Err(DutiesReaderError::UnknownEpoch), diff --git a/eth2/block_producer/src/traits.rs b/eth2/block_producer/src/traits.rs index 0145328a3..dada6ce3a 100644 --- a/eth2/block_producer/src/traits.rs +++ b/eth2/block_producer/src/traits.rs @@ -6,6 +6,12 @@ pub enum BeaconNodeError { DecodeFailure, } +#[derive(Debug, PartialEq, Clone)] +pub enum PublishOutcome { + ValidBlock, + InvalidBlock(String), +} + /// Defines the methods required to produce and publish blocks on a Beacon Node. pub trait BeaconNode: Send + Sync { /// Requests the proposer nonce (presently named `proposer_slots`). @@ -23,7 +29,7 @@ pub trait BeaconNode: Send + Sync { /// Request that the node publishes a block. /// /// Returns `true` if the publish was sucessful. - fn publish_beacon_block(&self, block: BeaconBlock) -> Result; + fn publish_beacon_block(&self, block: BeaconBlock) -> Result; } #[derive(Debug, PartialEq, Clone)] @@ -35,7 +41,7 @@ pub enum DutiesReaderError { /// Informs a validator of their duties (e.g., block production). pub trait DutiesReader: Send + Sync { - fn is_block_production_slot(&self, epoch: u64, slot: u64) -> Result; + fn is_block_production_slot(&self, slot: u64) -> Result; } /// Signs message using an internally-maintained private key. From f4f5b3a13c1329b59f419ab5c12960ca07cd511d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 25 Jan 2019 11:30:06 +1100 Subject: [PATCH 020/107] Update beacon_chain as per test bugs --- .../beacon_chain/src/block_processing.rs | 32 +++++--- .../beacon_chain/src/finalized_head.rs | 34 ++++++++ beacon_node/beacon_chain/src/lib.rs | 6 +- beacon_node/beacon_chain/src/lmd_ghost.rs | 26 ++++++- beacon_node/beacon_chain/tests/chain.rs | 5 +- .../tests/utils/direct_beacon_node.rs | 31 +++++--- .../beacon_chain/tests/utils/direct_duties.rs | 13 ++-- .../beacon_chain/tests/utils/test_rig.rs | 78 ++++++++++--------- .../beacon_chain/tests/utils/validator.rs | 49 ++++++++---- 9 files changed, 188 insertions(+), 86 deletions(-) create mode 100644 beacon_node/beacon_chain/src/finalized_head.rs diff --git a/beacon_node/beacon_chain/src/block_processing.rs b/beacon_node/beacon_chain/src/block_processing.rs index f99958dfb..32670f51d 100644 --- a/beacon_node/beacon_chain/src/block_processing.rs +++ b/beacon_node/beacon_chain/src/block_processing.rs @@ -8,16 +8,23 @@ use types::{ }; #[derive(Debug, PartialEq)] -pub enum Outcome { - FutureSlot, +pub enum ValidBlock { Processed, - NewCanonicalBlock, - NewReorgBlock, - NewForkBlock, +} + +#[derive(Debug, PartialEq)] +pub enum InvalidBlock { + FutureSlot, StateTransitionFailed(TransitionError), StateRootMismatch, } +#[derive(Debug, PartialEq)] +pub enum Outcome { + ValidBlock(ValidBlock), + InvalidBlock(InvalidBlock), +} + #[derive(Debug, PartialEq)] pub enum Error { DBError(String), @@ -70,7 +77,7 @@ where // Block from future slots (i.e., greater than the present slot) should not be processed. if block.slot() > present_slot { - return Ok(Outcome::FutureSlot); + return Ok(Outcome::InvalidBlock(InvalidBlock::FutureSlot)); } let parent_block_root = block.parent_root(); @@ -80,7 +87,8 @@ where .get_reader(&parent_block_root)? .ok_or(Error::MissingParentBlock(parent_block_root))?; - let parent_state_root = parent_block.parent_root(); + let parent_state_root = parent_block.state_root(); + let parent_state = self .state_store .get_reader(&parent_state_root)? @@ -90,13 +98,17 @@ where let state = match self.state_transition(parent_state, &block) { Ok(state) => state, - Err(error) => return Ok(Outcome::StateTransitionFailed(error)), + Err(error) => { + return Ok(Outcome::InvalidBlock(InvalidBlock::StateTransitionFailed( + error, + ))) + } }; let state_root = state.canonical_root(); if block.state_root != state_root { - return Ok(Outcome::StateRootMismatch); + return Ok(Outcome::InvalidBlock(InvalidBlock::StateRootMismatch)); } // Store the block and state. @@ -119,7 +131,7 @@ where } // The block was sucessfully processed. - Ok(Outcome::Processed) + Ok(Outcome::ValidBlock(ValidBlock::Processed)) } } diff --git a/beacon_node/beacon_chain/src/finalized_head.rs b/beacon_node/beacon_chain/src/finalized_head.rs new file mode 100644 index 000000000..28c883b4c --- /dev/null +++ b/beacon_node/beacon_chain/src/finalized_head.rs @@ -0,0 +1,34 @@ +use crate::{BeaconChain, CheckPoint, ClientDB, SlotClock}; +use std::sync::RwLockReadGuard; +use types::{BeaconBlock, BeaconState, Hash256}; + +impl BeaconChain +where + T: ClientDB, + U: SlotClock, +{ + pub fn update_finalized_head( + &self, + new_beacon_block: BeaconBlock, + new_beacon_block_root: Hash256, + new_beacon_state: BeaconState, + new_beacon_state_root: Hash256, + ) { + let mut finalized_head = self + .finalized_head + .write() + .expect("CRITICAL: finalized_head poisioned."); + finalized_head.update( + new_beacon_block, + new_beacon_block_root, + new_beacon_state, + new_beacon_state_root, + ); + } + + pub fn finalized_head(&self) -> RwLockReadGuard { + self.finalized_head + .read() + .expect("CRITICAL: finalized_head poisioned.") + } +} diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index b52dfdd73..8ae6d0f90 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -1,8 +1,9 @@ mod attestation_targets; mod block_graph; -mod block_processing; +pub mod block_processing; pub mod block_production; mod canonical_head; +mod finalized_head; mod info; mod lmd_ghost; mod state_transition; @@ -17,9 +18,8 @@ use genesis::{genesis_beacon_block, genesis_beacon_state, GenesisError}; use slot_clock::SlotClock; use spec::ChainSpec; use ssz::ssz_encode; -use std::collections::{HashMap, HashSet}; use std::sync::{Arc, RwLock}; -use types::{BeaconBlock, BeaconState, Hash256, PublicKey}; +use types::{BeaconBlock, BeaconState, Hash256}; pub use self::block_processing::Outcome as BlockProcessingOutcome; diff --git a/beacon_node/beacon_chain/src/lmd_ghost.rs b/beacon_node/beacon_chain/src/lmd_ghost.rs index e4aaa3483..6116834bb 100644 --- a/beacon_node/beacon_chain/src/lmd_ghost.rs +++ b/beacon_node/beacon_chain/src/lmd_ghost.rs @@ -32,7 +32,31 @@ where U: SlotClock, Error: From<::Error>, { - pub fn slow_lmd_ghost(&mut self, start_hash: &Hash256) -> Result { + pub fn fork_choice(&self) -> Result<(), Error> { + let present_head = &self.finalized_head().beacon_block_root; + + let new_head = self.slow_lmd_ghost(&self.finalized_head().beacon_block_root)?; + + if new_head != *present_head { + let block = self + .block_store + .get_deserialized(&new_head)? + .ok_or_else(|| Error::MissingBeaconBlock(new_head))?; + let block_root = block.canonical_root(); + + let state = self + .state_store + .get_deserialized(&block.state_root)? + .ok_or_else(|| Error::MissingBeaconState(block.state_root))?; + let state_root = state.canonical_root(); + + self.update_canonical_head(block, block_root, state, state_root); + } + + Ok(()) + } + + pub fn slow_lmd_ghost(&self, start_hash: &Hash256) -> Result { let start = self .block_store .get_reader(&start_hash)? diff --git a/beacon_node/beacon_chain/tests/chain.rs b/beacon_node/beacon_chain/tests/chain.rs index 3c0a9b8f9..52d001eed 100644 --- a/beacon_node/beacon_chain/tests/chain.rs +++ b/beacon_node/beacon_chain/tests/chain.rs @@ -20,8 +20,9 @@ fn rig_can_generate_validators() { let validators = generate_validators(2, &chain); chain.spec = inject_validators_into_spec(chain.spec.clone(), &validators[..]); */ - let mut rig = TestRig::new(ChainSpec::foundation()); - rig.generate_validators(2); + let validator_count = 2; + let mut rig = TestRig::new(ChainSpec::foundation(), validator_count); + rig.produce_next_slot(); } /* diff --git a/beacon_node/beacon_chain/tests/utils/direct_beacon_node.rs b/beacon_node/beacon_chain/tests/utils/direct_beacon_node.rs index 130a4d136..9fcd7c286 100644 --- a/beacon_node/beacon_chain/tests/utils/direct_beacon_node.rs +++ b/beacon_node/beacon_chain/tests/utils/direct_beacon_node.rs @@ -1,22 +1,27 @@ +use beacon_chain::block_processing::{Error as ProcessingError, Outcome as ProcessingOutcome}; use beacon_chain::{block_production::Error as BlockProductionError, BeaconChain}; -use block_producer::{BeaconNode as BeaconBlockNode, BeaconNodeError as BeaconBlockNodeError}; +use block_producer::{ + BeaconNode as BeaconBlockNode, BeaconNodeError as BeaconBlockNodeError, PublishOutcome, +}; use db::ClientDB; use slot_clock::SlotClock; +use std::sync::Arc; use types::{BeaconBlock, PublicKey, Signature}; -pub struct DirectBeaconNode<'a, T: ClientDB, U: SlotClock> { - beacon_chain: &'a BeaconChain, +pub struct DirectBeaconNode { + beacon_chain: Arc>, } -impl<'a, T: ClientDB, U: SlotClock> DirectBeaconNode<'a, T, U> { - pub fn new(beacon_chain: &'a BeaconChain) -> Self { +impl DirectBeaconNode { + pub fn new(beacon_chain: Arc>) -> Self { Self { beacon_chain } } } -impl<'a, T: ClientDB, U: SlotClock> BeaconBlockNode for DirectBeaconNode<'a, T, U> +impl BeaconBlockNode for DirectBeaconNode where BlockProductionError: From<::Error>, + ProcessingError: From<::Error>, { fn proposer_nonce(&self, pubkey: &PublicKey) -> Result { let validator_index = self @@ -51,8 +56,16 @@ where { } } - /// Returns the value specified by the `set_next_publish_result`. - fn publish_beacon_block(&self, block: BeaconBlock) -> Result { - Err(BeaconBlockNodeError::DecodeFailure) + fn publish_beacon_block( + &self, + block: BeaconBlock, + ) -> Result { + match self.beacon_chain.process_block(block) { + Ok(ProcessingOutcome::ValidBlock(_)) => Ok(PublishOutcome::ValidBlock), + Ok(ProcessingOutcome::InvalidBlock(reason)) => { + Ok(PublishOutcome::InvalidBlock(format!("{:?}", reason))) + } + Err(error) => Err(BeaconBlockNodeError::RemoteFailure(format!("{:?}", error))), + } } } diff --git a/beacon_node/beacon_chain/tests/utils/direct_duties.rs b/beacon_node/beacon_chain/tests/utils/direct_duties.rs index 75177ac14..ff2cd7223 100644 --- a/beacon_node/beacon_chain/tests/utils/direct_duties.rs +++ b/beacon_node/beacon_chain/tests/utils/direct_duties.rs @@ -2,15 +2,16 @@ use beacon_chain::{block_production::Error as BlockProductionError, BeaconChain} use block_producer::{DutiesReader, DutiesReaderError}; use db::ClientDB; use slot_clock::SlotClock; +use std::sync::Arc; use types::PublicKey; -pub struct DirectDuties<'a, T: ClientDB, U: SlotClock> { - beacon_chain: &'a BeaconChain, +pub struct DirectDuties { + beacon_chain: Arc>, pubkey: PublicKey, } -impl<'a, T: ClientDB, U: SlotClock> DirectDuties<'a, T, U> { - pub fn new(pubkey: PublicKey, beacon_chain: &'a BeaconChain) -> Self { +impl DirectDuties { + pub fn new(pubkey: PublicKey, beacon_chain: Arc>) -> Self { Self { beacon_chain, pubkey, @@ -18,11 +19,11 @@ impl<'a, T: ClientDB, U: SlotClock> DirectDuties<'a, T, U> { } } -impl<'a, T: ClientDB, U: SlotClock> DutiesReader for DirectDuties<'a, T, U> +impl DutiesReader for DirectDuties where BlockProductionError: From<::Error>, { - fn is_block_production_slot(&self, _epoch: u64, slot: u64) -> Result { + fn is_block_production_slot(&self, slot: u64) -> Result { let validator_index = self .beacon_chain .validator_index(&self.pubkey) diff --git a/beacon_node/beacon_chain/tests/utils/test_rig.rs b/beacon_node/beacon_chain/tests/utils/test_rig.rs index 0bdf0a36b..7bd494b23 100644 --- a/beacon_node/beacon_chain/tests/utils/test_rig.rs +++ b/beacon_node/beacon_chain/tests/utils/test_rig.rs @@ -11,75 +11,77 @@ use spec::ChainSpec; use std::sync::{Arc, RwLock}; use types::{Keypair, Validator}; -pub struct TestRig<'a> { +pub struct TestRig { db: Arc, - beacon_chain: BeaconChain, + beacon_chain: Arc>, block_store: Arc>, state_store: Arc>, - validators: Vec>, + validators: Vec, } -impl<'a> TestRig<'a> { - pub fn new(spec: ChainSpec) -> Self { +impl TestRig { + pub fn new(mut spec: ChainSpec, validator_count: usize) -> Self { let db = Arc::new(MemoryDB::open()); let block_store = Arc::new(BeaconBlockStore::new(db.clone())); let state_store = Arc::new(BeaconStateStore::new(db.clone())); let slot_clock = TestingSlotClock::new(0); - let mut beacon_chain = - BeaconChain::genesis(state_store.clone(), block_store.clone(), slot_clock, spec) - .unwrap(); + // Remove the validators present in the spec (if any). + spec.initial_validators = Vec::with_capacity(validator_count); + spec.initial_balances = Vec::with_capacity(validator_count); - /* - let validators = generate_validators(validator_count, &beacon_chain); - beacon_chain.spec = inject_validators_into_spec(beacon_chain.spec.clone(), &validators[..]); - */ + // Insert `validator_count` new `Validator` records into the spec, retaining the keypairs + // for later user. + let mut keypairs = Vec::with_capacity(validator_count); + for _ in 0..validator_count { + let keypair = Keypair::random(); + + spec.initial_validators.push(Validator { + pubkey: keypair.pk.clone(), + ..std::default::Default::default() + }); + spec.initial_balances.push(32_000_000_000); // 32 ETH + + keypairs.push(keypair); + } + + // Create the Beacon Chain + let beacon_chain = Arc::new( + BeaconChain::genesis(state_store.clone(), block_store.clone(), slot_clock, spec) + .unwrap(), + ); + + // Spawn the test validator instances. + let mut validators = Vec::with_capacity(validator_count); + for keypair in keypairs { + validators.push(TestValidator::new(keypair.clone(), beacon_chain.clone())); + } Self { db, beacon_chain, block_store, state_store, - validators: vec![], + validators, } } - pub fn generate_validators(&'a mut self, validator_count: usize) { - self.validators = Vec::with_capacity(validator_count); - for _ in 0..validator_count { - self.validators.push(TestValidator::new(&self.beacon_chain)); - } - self.beacon_chain.spec = - inject_validators_into_spec(self.beacon_chain.spec.clone(), &self.validators[..]); - } - - pub fn process_next_slot(&mut self) { + pub fn produce_next_slot(&mut self) { let slot = self .beacon_chain .present_slot() .expect("Unable to determine slot.") + 1; + self.beacon_chain.slot_clock.set_slot(slot); - let block_proposer = self + let proposer = self .beacon_chain .block_proposer(slot) .expect("Unable to determine proposer."); - let validator = self - .validators - .get(block_proposer) - .expect("Block proposer unknown"); + self.validators[proposer].set_slot(slot); + self.validators[proposer].produce_block().unwrap(); } } - -fn inject_validators_into_spec(mut spec: ChainSpec, validators: &[TestValidator]) -> ChainSpec { - spec.initial_validators = Vec::with_capacity(validators.len()); - spec.initial_balances = Vec::with_capacity(validators.len()); - for validator in validators { - spec.initial_validators.push(validator.validator_record()); - spec.initial_balances.push(32_000_000_000); // 32 ETH - } - spec -} diff --git a/beacon_node/beacon_chain/tests/utils/validator.rs b/beacon_node/beacon_chain/tests/utils/validator.rs index a6a348dce..755ea51d1 100644 --- a/beacon_node/beacon_chain/tests/utils/validator.rs +++ b/beacon_node/beacon_chain/tests/utils/validator.rs @@ -1,36 +1,46 @@ use super::{DirectBeaconNode, DirectDuties}; use beacon_chain::BeaconChain; #[cfg(test)] -use block_producer::{test_utils::TestSigner, BlockProducer}; +use block_producer::{test_utils::TestSigner, BlockProducer, Error as PollError}; use db::MemoryDB; use slot_clock::TestingSlotClock; use spec::ChainSpec; use std::sync::{Arc, RwLock}; use types::{Keypair, Validator}; -pub struct TestValidator<'a> { +pub use block_producer::PollOutcome; + +#[derive(Debug, PartialEq)] +pub enum ProduceError { + DidNotProduce(PollOutcome), + PollError(PollError), +} + +pub struct TestValidator { block_producer: BlockProducer< TestingSlotClock, - DirectBeaconNode<'a, MemoryDB, TestingSlotClock>, - DirectDuties<'a, MemoryDB, TestingSlotClock>, + DirectBeaconNode, + DirectDuties, TestSigner, >, spec: Arc, - epoch_map: Arc>, + epoch_map: Arc>, keypair: Keypair, - beacon_node: Arc>, - slot_clock: Arc>, + beacon_node: Arc>, + slot_clock: Arc, signer: Arc, } -impl<'a> TestValidator<'a> { - pub fn new(beacon_chain: &'a BeaconChain) -> Self { +impl TestValidator { + pub fn new( + keypair: Keypair, + beacon_chain: Arc>, + ) -> Self { let spec = Arc::new(ChainSpec::foundation()); - let keypair = Keypair::random(); - let slot_clock = Arc::new(RwLock::new(TestingSlotClock::new(0))); + let slot_clock = Arc::new(TestingSlotClock::new(0)); let signer = Arc::new(TestSigner::new(keypair.clone())); - let beacon_node = Arc::new(DirectBeaconNode::new(beacon_chain)); - let epoch_map = Arc::new(DirectDuties::new(keypair.pk.clone(), beacon_chain)); + let beacon_node = Arc::new(DirectBeaconNode::new(beacon_chain.clone())); + let epoch_map = Arc::new(DirectDuties::new(keypair.pk.clone(), beacon_chain.clone())); let block_producer = BlockProducer::new( spec.clone(), @@ -52,10 +62,15 @@ impl<'a> TestValidator<'a> { } } - pub fn validator_record(&self) -> Validator { - Validator { - pubkey: self.keypair.pk.clone(), - ..std::default::Default::default() + pub fn produce_block(&mut self) -> Result { + match self.block_producer.poll() { + Ok(PollOutcome::BlockProduced(slot)) => Ok(PollOutcome::BlockProduced(slot)), + Ok(outcome) => Err(ProduceError::DidNotProduce(outcome)), + Err(error) => Err(ProduceError::PollError(error)), } } + + pub fn set_slot(&mut self, slot: u64) { + self.slot_clock.set_slot(slot) + } } From 19417efa6308d15f804381fb92be9f96ace503bf Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 25 Jan 2019 13:02:59 +1100 Subject: [PATCH 021/107] Fix compile errors from block_producer upgrades --- eth2/block_producer/src/lib.rs | 30 +++---- .../src/test_utils/epoch_map.rs | 4 +- eth2/block_producer/src/traits.rs | 1 + .../src/block_producer_service/grpc.rs | 25 ++++-- .../src/block_producer_service/service.rs | 3 + validator_client/src/duties/duties_map.rs | 81 +++++++++++++++++ validator_client/src/duties/mod.rs | 88 ++----------------- validator_client/src/main.rs | 9 +- 8 files changed, 135 insertions(+), 106 deletions(-) create mode 100644 validator_client/src/duties/duties_map.rs diff --git a/eth2/block_producer/src/lib.rs b/eth2/block_producer/src/lib.rs index 3cf610355..74788eb60 100644 --- a/eth2/block_producer/src/lib.rs +++ b/eth2/block_producer/src/lib.rs @@ -4,7 +4,7 @@ mod traits; use slot_clock::SlotClock; use spec::ChainSpec; use ssz::ssz_encode; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use types::{BeaconBlock, Hash256, ProposalSignedData, PublicKey}; pub use self::traits::{ @@ -90,10 +90,6 @@ impl BlockProducer BlockProducer { return Ok(PollOutcome::ValidatorIsUnknown(slot)) } + Err(DutiesReaderError::EpochLengthIsZero) => return Err(Error::EpochLengthIsZero), Err(DutiesReaderError::Poisoned) => return Err(Error::EpochMapPoisoned), }; @@ -121,7 +118,7 @@ impl BlockProducer bool { match self.last_processed_slot { - Some(processed_slot) if processed_slot <= slot => true, + Some(processed_slot) if processed_slot >= slot => true, _ => false, } } @@ -249,18 +246,20 @@ mod tests { let mut rng = XorShiftRng::from_seed([42; 16]); let spec = Arc::new(ChainSpec::foundation()); - let slot_clock = Arc::new(RwLock::new(TestingSlotClock::new(0))); + let slot_clock = Arc::new(TestingSlotClock::new(0)); let beacon_node = Arc::new(TestBeaconNode::default()); let signer = Arc::new(TestSigner::new(Keypair::random())); - let mut epoch_map = TestEpochMap::new(); + let mut epoch_map = TestEpochMap::new(spec.epoch_length); let produce_slot = 100; let produce_epoch = produce_slot / spec.epoch_length; - epoch_map.insert(produce_epoch, produce_slot); + epoch_map.map.insert(produce_epoch, produce_slot); let epoch_map = Arc::new(epoch_map); + let keypair = Keypair::random(); let mut block_producer = BlockProducer::new( spec.clone(), + keypair.pk.clone(), epoch_map.clone(), slot_clock.clone(), beacon_node.clone(), @@ -269,31 +268,32 @@ mod tests { // Configure responses from the BeaconNode. beacon_node.set_next_produce_result(Ok(Some(BeaconBlock::random_for_test(&mut rng)))); - beacon_node.set_next_publish_result(Ok(true)); + beacon_node.set_next_publish_result(Ok(PublishOutcome::ValidBlock)); + beacon_node.set_next_nonce_result(Ok(0)); // One slot before production slot... - slot_clock.write().unwrap().set_slot(produce_slot - 1); + slot_clock.set_slot(produce_slot - 1); assert_eq!( block_producer.poll(), Ok(PollOutcome::BlockProductionNotRequired(produce_slot - 1)) ); // On the produce slot... - slot_clock.write().unwrap().set_slot(produce_slot); + slot_clock.set_slot(produce_slot); assert_eq!( block_producer.poll(), Ok(PollOutcome::BlockProduced(produce_slot)) ); // Trying the same produce slot again... - slot_clock.write().unwrap().set_slot(produce_slot); + slot_clock.set_slot(produce_slot); assert_eq!( block_producer.poll(), Ok(PollOutcome::SlotAlreadyProcessed(produce_slot)) ); // One slot after the produce slot... - slot_clock.write().unwrap().set_slot(produce_slot + 1); + slot_clock.set_slot(produce_slot + 1); assert_eq!( block_producer.poll(), Ok(PollOutcome::BlockProductionNotRequired(produce_slot + 1)) @@ -301,7 +301,7 @@ mod tests { // In an epoch without known duties... let slot = (produce_epoch + 1) * spec.epoch_length; - slot_clock.write().unwrap().set_slot(slot); + slot_clock.set_slot(slot); assert_eq!( block_producer.poll(), Ok(PollOutcome::ProducerDutiesUnknown(slot)) diff --git a/eth2/block_producer/src/test_utils/epoch_map.rs b/eth2/block_producer/src/test_utils/epoch_map.rs index 4cb1fb9fb..848ae252e 100644 --- a/eth2/block_producer/src/test_utils/epoch_map.rs +++ b/eth2/block_producer/src/test_utils/epoch_map.rs @@ -3,11 +3,11 @@ use std::collections::HashMap; pub struct TestEpochMap { epoch_length: u64, - map: HashMap, + pub map: HashMap, } impl TestEpochMap { - fn new(epoch_length: u64) -> Self { + pub fn new(epoch_length: u64) -> Self { Self { epoch_length, map: HashMap::new(), diff --git a/eth2/block_producer/src/traits.rs b/eth2/block_producer/src/traits.rs index dada6ce3a..451ce6cc7 100644 --- a/eth2/block_producer/src/traits.rs +++ b/eth2/block_producer/src/traits.rs @@ -36,6 +36,7 @@ pub trait BeaconNode: Send + Sync { pub enum DutiesReaderError { UnknownValidator, UnknownEpoch, + EpochLengthIsZero, Poisoned, } diff --git a/validator_client/src/block_producer_service/grpc.rs b/validator_client/src/block_producer_service/grpc.rs index 69a146d19..9ac8e779c 100644 --- a/validator_client/src/block_producer_service/grpc.rs +++ b/validator_client/src/block_producer_service/grpc.rs @@ -1,11 +1,11 @@ -use block_producer::{BeaconNode, BeaconNodeError}; +use block_producer::{BeaconNode, BeaconNodeError, PublishOutcome}; use protos::services::{ BeaconBlock as GrpcBeaconBlock, ProduceBeaconBlockRequest, PublishBeaconBlockRequest, }; use protos::services_grpc::BeaconBlockServiceClient; use ssz::{ssz_encode, Decodable}; use std::sync::Arc; -use types::{BeaconBlock, BeaconBlockBody, Eth1Data, Hash256, Signature}; +use types::{BeaconBlock, BeaconBlockBody, Eth1Data, Hash256, PublicKey, Signature}; /// A newtype designed to wrap the gRPC-generated service so the `BeaconNode` trait may be /// implemented upon it. @@ -20,11 +20,21 @@ impl BeaconBlockGrpcClient { } impl BeaconNode for BeaconBlockGrpcClient { + fn proposer_nonce(&self, pubkey: &PublicKey) -> Result { + // TODO: this might not be required. + // + // See: https://github.com/ethereum/eth2.0-specs/pull/496 + panic!("Not implemented.") + } /// Request a Beacon Node (BN) to produce a new block at the supplied slot. /// /// Returns `None` if it is not possible to produce at the supplied slot. For example, if the /// BN is unable to find a parent block. - fn produce_beacon_block(&self, slot: u64) -> Result, BeaconNodeError> { + fn produce_beacon_block( + &self, + slot: u64, + randao_reveal: &Signature, + ) -> Result, BeaconNodeError> { let mut req = ProduceBeaconBlockRequest::new(); req.set_slot(slot); @@ -73,7 +83,7 @@ impl BeaconNode for BeaconBlockGrpcClient { /// /// Generally, this will be called after a `produce_beacon_block` call with a block that has /// been completed (signed) by the validator client. - fn publish_beacon_block(&self, block: BeaconBlock) -> Result { + fn publish_beacon_block(&self, block: BeaconBlock) -> Result { let mut req = PublishBeaconBlockRequest::new(); // TODO: this conversion is incomplete; fix it. @@ -90,6 +100,11 @@ impl BeaconNode for BeaconBlockGrpcClient { .publish_beacon_block(&req) .map_err(|err| BeaconNodeError::RemoteFailure(format!("{:?}", err)))?; - Ok(reply.get_success()) + if reply.get_success() { + Ok(PublishOutcome::ValidBlock) + } else { + // TODO: distinguish between different errors + Ok(PublishOutcome::InvalidBlock("Publish failed".to_string())) + } } } diff --git a/validator_client/src/block_producer_service/service.rs b/validator_client/src/block_producer_service/service.rs index 3966d42f9..5e335e383 100644 --- a/validator_client/src/block_producer_service/service.rs +++ b/validator_client/src/block_producer_service/service.rs @@ -42,6 +42,9 @@ impl BlockProducerServi Ok(BlockProducerPollOutcome::SignerRejection(slot)) => { error!(self.log, "The cryptographic signer refused to sign the block"; "slot" => slot) } + Ok(BlockProducerPollOutcome::ValidatorIsUnknown(slot)) => { + error!(self.log, "The Beacon Node does not recognise the validator"; "slot" => slot) + } }; std::thread::sleep(Duration::from_millis(self.poll_interval_millis)); diff --git a/validator_client/src/duties/duties_map.rs b/validator_client/src/duties/duties_map.rs new file mode 100644 index 000000000..ea3c8ae4a --- /dev/null +++ b/validator_client/src/duties/duties_map.rs @@ -0,0 +1,81 @@ +use block_producer::{DutiesReader, DutiesReaderError}; +use std::collections::HashMap; +use std::sync::RwLock; + +/// The information required for a validator to propose and attest during some epoch. +/// +/// Generally obtained from a Beacon Node, this information contains the validators canonical index +/// (thier sequence in the global validator induction process) and the "shuffling" for that index +/// for some epoch. +#[derive(Debug, PartialEq, Clone, Copy, Default)] +pub struct EpochDuties { + pub validator_index: u64, + pub block_production_slot: Option, + // Future shard info +} + +impl EpochDuties { + /// Returns `true` if the supplied `slot` is a slot in which the validator should produce a + /// block. + pub fn is_block_production_slot(&self, slot: u64) -> bool { + match self.block_production_slot { + Some(s) if s == slot => true, + _ => false, + } + } +} + +pub enum EpochDutiesMapError { + Poisoned, +} + +/// Maps an `epoch` to some `EpochDuties` for a single validator. +pub struct EpochDutiesMap { + pub epoch_length: u64, + pub map: RwLock>, +} + +impl EpochDutiesMap { + pub fn new(epoch_length: u64) -> Self { + Self { + epoch_length, + map: RwLock::new(HashMap::new()), + } + } + + pub fn get(&self, epoch: u64) -> Result, EpochDutiesMapError> { + let map = self.map.read().map_err(|_| EpochDutiesMapError::Poisoned)?; + match map.get(&epoch) { + Some(duties) => Ok(Some(duties.clone())), + None => Ok(None), + } + } + + pub fn insert( + &self, + epoch: u64, + epoch_duties: EpochDuties, + ) -> Result, EpochDutiesMapError> { + let mut map = self + .map + .write() + .map_err(|_| EpochDutiesMapError::Poisoned)?; + Ok(map.insert(epoch, epoch_duties)) + } +} + +impl DutiesReader for EpochDutiesMap { + fn is_block_production_slot(&self, slot: u64) -> Result { + let epoch = slot + .checked_div(self.epoch_length) + .ok_or_else(|| DutiesReaderError::EpochLengthIsZero)?; + + let map = self.map.read().map_err(|_| DutiesReaderError::Poisoned)?; + let duties = map + .get(&epoch) + .ok_or_else(|| DutiesReaderError::UnknownEpoch)?; + Ok(duties.is_block_production_slot(slot)) + } +} + +// TODO: add tests. diff --git a/validator_client/src/duties/mod.rs b/validator_client/src/duties/mod.rs index 8484ad80c..78dbfa964 100644 --- a/validator_client/src/duties/mod.rs +++ b/validator_client/src/duties/mod.rs @@ -1,88 +1,18 @@ +mod duties_map; mod grpc; mod service; #[cfg(test)] mod test_node; mod traits; +pub use self::duties_map::EpochDutiesMap; +use self::duties_map::{EpochDuties, EpochDutiesMapError}; +pub use self::service::DutiesManagerService; use self::traits::{BeaconNode, BeaconNodeError}; -use block_producer::{DutiesReader, DutiesReaderError}; use bls::PublicKey; use slot_clock::SlotClock; use spec::ChainSpec; -use std::collections::HashMap; -use std::sync::{Arc, RwLock}; - -pub use self::service::DutiesManagerService; - -/// The information required for a validator to propose and attest during some epoch. -/// -/// Generally obtained from a Beacon Node, this information contains the validators canonical index -/// (thier sequence in the global validator induction process) and the "shuffling" for that index -/// for some epoch. -#[derive(Debug, PartialEq, Clone, Copy, Default)] -pub struct EpochDuties { - pub validator_index: u64, - pub block_production_slot: Option, - // Future shard info -} - -impl EpochDuties { - /// Returns `true` if the supplied `slot` is a slot in which the validator should produce a - /// block. - pub fn is_block_production_slot(&self, slot: u64) -> bool { - match self.block_production_slot { - Some(s) if s == slot => true, - _ => false, - } - } -} - -pub enum EpochDutiesMapError { - Poisoned, -} - -/// Maps an `epoch` to some `EpochDuties` for a single validator. -pub struct EpochDutiesMap { - pub map: RwLock>, -} - -impl EpochDutiesMap { - pub fn new() -> Self { - Self { - map: RwLock::new(HashMap::new()), - } - } - - pub fn get(&self, epoch: u64) -> Result, EpochDutiesMapError> { - let map = self.map.read().map_err(|_| EpochDutiesMapError::Poisoned)?; - match map.get(&epoch) { - Some(duties) => Ok(Some(duties.clone())), - None => Ok(None), - } - } - - pub fn insert( - &self, - epoch: u64, - epoch_duties: EpochDuties, - ) -> Result, EpochDutiesMapError> { - let mut map = self - .map - .write() - .map_err(|_| EpochDutiesMapError::Poisoned)?; - Ok(map.insert(epoch, epoch_duties)) - } -} - -impl DutiesReader for EpochDutiesMap { - fn is_block_production_slot(&self, epoch: u64, slot: u64) -> Result { - let map = self.map.read().map_err(|_| DutiesReaderError::Poisoned)?; - let duties = map - .get(&epoch) - .ok_or_else(|| DutiesReaderError::UnknownEpoch)?; - Ok(duties.is_block_production_slot(slot)) - } -} +use std::sync::Arc; #[derive(Debug, PartialEq, Clone, Copy)] pub enum PollOutcome { @@ -117,7 +47,7 @@ pub struct DutiesManager { /// The validator's public key. pub pubkey: PublicKey, pub spec: Arc, - pub slot_clock: Arc>, + pub slot_clock: Arc, pub beacon_node: Arc, } @@ -129,8 +59,6 @@ impl DutiesManager { pub fn poll(&self) -> Result { let slot = self .slot_clock - .read() - .map_err(|_| Error::SlotClockPoisoned)? .present_slot() .map_err(|_| Error::SlotClockError)? .ok_or(Error::SlotUnknowable)?; @@ -187,9 +115,9 @@ mod tests { #[test] pub fn polling() { let spec = Arc::new(ChainSpec::foundation()); - let duties_map = Arc::new(EpochDutiesMap::new()); + let duties_map = Arc::new(EpochDutiesMap::new(spec.epoch_length)); let keypair = Keypair::random(); - let slot_clock = Arc::new(RwLock::new(TestingSlotClock::new(0))); + let slot_clock = Arc::new(TestingSlotClock::new(0)); let beacon_node = Arc::new(TestBeaconNode::default()); let manager = DutiesManager { diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index cd626b2fc..49927b8c9 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -10,7 +10,7 @@ use slog::{error, info, o, Drain}; use slot_clock::SystemTimeSlotClock; use spec::ChainSpec; use std::path::PathBuf; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use std::thread; mod block_producer_service; @@ -92,7 +92,7 @@ fn main() { info!(log, "Genesis time"; "unix_epoch_seconds" => spec.genesis_time); let clock = SystemTimeSlotClock::new(spec.genesis_time, spec.slot_duration) .expect("Unable to instantiate SystemTimeSlotClock."); - Arc::new(RwLock::new(clock)) + Arc::new(clock) }; let poll_interval_millis = spec.slot_duration * 1000 / 10; // 10% epoch time precision. @@ -108,7 +108,7 @@ fn main() { for keypair in keypairs { info!(log, "Starting validator services"; "validator" => keypair.pk.concatenated_hex_id()); - let duties_map = Arc::new(EpochDutiesMap::new()); + let duties_map = Arc::new(EpochDutiesMap::new(spec.epoch_length)); // Spawn a new thread to maintain the validator's `EpochDuties`. let duties_manager_thread = { @@ -139,6 +139,7 @@ fn main() { // Spawn a new thread to perform block production for the validator. let producer_thread = { let spec = spec.clone(); + let pubkey = keypair.pk.clone(); let signer = Arc::new(TestSigner::new(keypair.clone())); let duties_map = duties_map.clone(); let slot_clock = slot_clock.clone(); @@ -146,7 +147,7 @@ fn main() { let client = Arc::new(BeaconBlockGrpcClient::new(beacon_block_grpc_client.clone())); thread::spawn(move || { let block_producer = - BlockProducer::new(spec, duties_map, slot_clock, client, signer); + BlockProducer::new(spec, pubkey, duties_map, slot_clock, client, signer); let mut block_producer_service = BlockProducerService { block_producer, poll_interval_millis, From 9c86c07eea43570c84d3fbe588a78c10a476fc16 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 25 Jan 2019 13:05:11 +1100 Subject: [PATCH 022/107] Fix unused deps warnings --- beacon_node/beacon_chain/src/attestation_targets.rs | 5 ++--- beacon_node/beacon_chain/src/block_graph.rs | 3 +-- validator_client/src/duties/mod.rs | 1 - 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_targets.rs b/beacon_node/beacon_chain/src/attestation_targets.rs index 6971d5b4e..d968abcf2 100644 --- a/beacon_node/beacon_chain/src/attestation_targets.rs +++ b/beacon_node/beacon_chain/src/attestation_targets.rs @@ -1,7 +1,6 @@ -use crate::{BeaconChain, CheckPoint, ClientDB, SlotClock}; +use crate::{BeaconChain, ClientDB, SlotClock}; use std::collections::HashMap; -use std::sync::RwLockReadGuard; -use types::{BeaconBlock, BeaconState, Hash256}; +use types::Hash256; pub struct AttestationTargets { map: HashMap, diff --git a/beacon_node/beacon_chain/src/block_graph.rs b/beacon_node/beacon_chain/src/block_graph.rs index 8a0f6e61d..b031c2c10 100644 --- a/beacon_node/beacon_chain/src/block_graph.rs +++ b/beacon_node/beacon_chain/src/block_graph.rs @@ -1,7 +1,6 @@ -use crate::{BeaconChain, CheckPoint, ClientDB, SlotClock}; use std::collections::HashSet; use std::sync::{RwLock, RwLockReadGuard}; -use types::{BeaconBlock, BeaconState, Hash256}; +use types::Hash256; pub struct BlockGraph { pub leaves: RwLock>, diff --git a/validator_client/src/duties/mod.rs b/validator_client/src/duties/mod.rs index 78dbfa964..f368d6dca 100644 --- a/validator_client/src/duties/mod.rs +++ b/validator_client/src/duties/mod.rs @@ -33,7 +33,6 @@ pub enum Error { SlotClockError, SlotUnknowable, EpochMapPoisoned, - SlotClockPoisoned, EpochLengthIsZero, BeaconNodeError(BeaconNodeError), } From ea523c86585ba85111322959f9f9a99ae1f94e8a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 25 Jan 2019 13:52:21 +1100 Subject: [PATCH 023/107] Merge `spec` crate into `types` --- Cargo.toml | 1 - beacon_node/Cargo.toml | 1 - beacon_node/beacon_chain/Cargo.toml | 1 - beacon_node/beacon_chain/src/lib.rs | 3 +-- beacon_node/beacon_chain/src/state_transition.rs | 16 ++++++++-------- beacon_node/beacon_chain/tests/chain.rs | 3 +-- beacon_node/beacon_chain/tests/utils/test_rig.rs | 8 +++----- .../beacon_chain/tests/utils/validator.rs | 5 ++--- beacon_node/db/src/stores/mod.rs | 2 -- beacon_node/db/src/stores/validator_store.rs | 4 ++-- beacon_node/src/main.rs | 2 +- eth2/block_producer/Cargo.toml | 1 - eth2/block_producer/src/lib.rs | 3 +-- eth2/genesis/Cargo.toml | 1 - eth2/genesis/src/beacon_block.rs | 3 +-- eth2/genesis/src/beacon_state.rs | 3 +-- eth2/genesis/src/lib.rs | 5 ----- eth2/spec/Cargo.toml | 9 --------- eth2/types/src/attestation.rs | 6 +++--- eth2/types/src/attestation_data.rs | 4 ++-- .../src/attestation_data_and_custody_bit.rs | 4 ++-- eth2/types/src/beacon_block.rs | 4 ++-- eth2/types/src/beacon_block_body.rs | 4 ++-- eth2/types/src/beacon_state.rs | 2 +- eth2/types/src/casper_slashing.rs | 4 ++-- eth2/types/src/crosslink.rs | 4 ++-- eth2/types/src/deposit.rs | 4 ++-- eth2/types/src/deposit_data.rs | 4 ++-- eth2/types/src/deposit_input.rs | 4 ++-- eth2/types/src/eth1_data.rs | 4 ++-- eth2/types/src/eth1_data_vote.rs | 4 ++-- eth2/types/src/exit.rs | 4 ++-- eth2/types/src/fork.rs | 4 ++-- eth2/types/src/lib.rs | 9 +++------ eth2/types/src/pending_attestation.rs | 4 ++-- eth2/types/src/proposal_signed_data.rs | 4 ++-- eth2/types/src/proposer_slashing.rs | 4 ++-- eth2/types/src/shard_committee.rs | 4 ++-- eth2/types/src/shard_reassignment_record.rs | 4 ++-- eth2/types/src/slashable_vote_data.rs | 4 ++-- eth2/{spec/src => types/src/spec}/foundation.rs | 2 +- eth2/{spec/src/lib.rs => types/src/spec/mod.rs} | 5 +---- eth2/types/src/special_record.rs | 2 +- eth2/types/src/validator.rs | 4 ++-- eth2/types/src/validator_registry_delta_block.rs | 2 +- eth2/utils/bls/src/aggregate_signature.rs | 4 ++-- eth2/utils/bls/src/public_key.rs | 2 +- eth2/utils/bls/src/signature.rs | 4 ++-- eth2/validator_induction/Cargo.toml | 1 - eth2/validator_induction/src/inductor.rs | 3 +-- eth2/validator_induction/src/lib.rs | 5 ----- eth2/validator_shuffling/Cargo.toml | 1 - eth2/validator_shuffling/src/lib.rs | 5 ----- eth2/validator_shuffling/src/shuffle.rs | 3 +-- validator_client/Cargo.toml | 1 - validator_client/src/duties/mod.rs | 2 +- validator_client/src/main.rs | 2 +- 57 files changed, 81 insertions(+), 131 deletions(-) delete mode 100644 eth2/spec/Cargo.toml rename eth2/{spec/src => types/src/spec}/foundation.rs (98%) rename eth2/{spec/src/lib.rs => types/src/spec/mod.rs} (95%) diff --git a/Cargo.toml b/Cargo.toml index 5a25e3484..e22d78d3c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,6 @@ members = [ "eth2/block_producer", "eth2/genesis", "eth2/naive_fork_choice", - "eth2/spec", "eth2/types", "eth2/utils/bls", "eth2/utils/boolean-bitfield", diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 7b44bb0f9..e5893195e 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -19,7 +19,6 @@ slog = "^2.2.3" slot_clock = { path = "../eth2/utils/slot_clock" } slog-term = "^2.4.0" slog-async = "^2.3.0" -spec = { path = "../eth2/spec" } types = { path = "../eth2/types" } ssz = { path = "../eth2/utils/ssz" } tokio = "0.1" diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index 7e157dffb..4fba69546 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -14,6 +14,5 @@ failure_derive = "0.1" genesis = { path = "../../eth2/genesis" } serde_derive = "1.0" slot_clock = { path = "../../eth2/utils/slot_clock" } -spec = { path = "../../eth2/spec" } ssz = { path = "../../eth2/utils/ssz" } types = { path = "../../eth2/types" } diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 8ae6d0f90..e820c4b20 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -16,10 +16,9 @@ use db::{ }; use genesis::{genesis_beacon_block, genesis_beacon_state, GenesisError}; use slot_clock::SlotClock; -use spec::ChainSpec; use ssz::ssz_encode; use std::sync::{Arc, RwLock}; -use types::{BeaconBlock, BeaconState, Hash256}; +use types::{BeaconBlock, BeaconState, ChainSpec, Hash256}; pub use self::block_processing::Outcome as BlockProcessingOutcome; diff --git a/beacon_node/beacon_chain/src/state_transition.rs b/beacon_node/beacon_chain/src/state_transition.rs index 4a41d6f2c..95fd9a7ad 100644 --- a/beacon_node/beacon_chain/src/state_transition.rs +++ b/beacon_node/beacon_chain/src/state_transition.rs @@ -430,19 +430,19 @@ fn get_domain(_fork: &Fork, _slot: u64, _domain_type: u64) -> u64 { 0 } -fn bls_verify(_pubkey: &PublicKey, _message: &[u8], _signature: &Signature, _domain: u64) -> bool { - // TODO: stubbed out. - true +fn bls_verify(pubkey: &PublicKey, message: &[u8], signature: &Signature, _domain: u64) -> bool { + // TODO: add domain + signature.verify(message, pubkey) } fn bls_verify_aggregate( - _pubkey: &AggregatePublicKey, - _message: &[u8], - _signature: &AggregateSignature, + pubkey: &AggregatePublicKey, + message: &[u8], + signature: &AggregateSignature, _domain: u64, ) -> bool { - // TODO: stubbed out. - true + // TODO: add domain + signature.verify(message, pubkey) } fn hash_tree_root(_input: &T) -> Hash256 { diff --git a/beacon_node/beacon_chain/tests/chain.rs b/beacon_node/beacon_chain/tests/chain.rs index 52d001eed..9bf8959c5 100644 --- a/beacon_node/beacon_chain/tests/chain.rs +++ b/beacon_node/beacon_chain/tests/chain.rs @@ -7,9 +7,8 @@ use db::{ MemoryDB, }; use slot_clock::TestingSlotClock; -use spec::ChainSpec; use std::sync::{Arc, RwLock}; -use types::{Keypair, Validator}; +use types::{ChainSpec, Keypair, Validator}; mod utils; diff --git a/beacon_node/beacon_chain/tests/utils/test_rig.rs b/beacon_node/beacon_chain/tests/utils/test_rig.rs index 7bd494b23..b6aef3290 100644 --- a/beacon_node/beacon_chain/tests/utils/test_rig.rs +++ b/beacon_node/beacon_chain/tests/utils/test_rig.rs @@ -1,15 +1,13 @@ -use super::{DirectBeaconNode, DirectDuties, TestValidator}; +use super::TestValidator; use beacon_chain::BeaconChain; #[cfg(test)] -use block_producer::{test_utils::TestSigner, BlockProducer}; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, MemoryDB, }; use slot_clock::TestingSlotClock; -use spec::ChainSpec; -use std::sync::{Arc, RwLock}; -use types::{Keypair, Validator}; +use std::sync::Arc; +use types::{ChainSpec, Keypair, Validator}; pub struct TestRig { db: Arc, diff --git a/beacon_node/beacon_chain/tests/utils/validator.rs b/beacon_node/beacon_chain/tests/utils/validator.rs index 755ea51d1..7a3b3404d 100644 --- a/beacon_node/beacon_chain/tests/utils/validator.rs +++ b/beacon_node/beacon_chain/tests/utils/validator.rs @@ -4,9 +4,8 @@ use beacon_chain::BeaconChain; use block_producer::{test_utils::TestSigner, BlockProducer, Error as PollError}; use db::MemoryDB; use slot_clock::TestingSlotClock; -use spec::ChainSpec; -use std::sync::{Arc, RwLock}; -use types::{Keypair, Validator}; +use std::sync::Arc; +use types::{ChainSpec, Keypair}; pub use block_producer::PollOutcome; diff --git a/beacon_node/db/src/stores/mod.rs b/beacon_node/db/src/stores/mod.rs index c78d10dbf..44de7eed1 100644 --- a/beacon_node/db/src/stores/mod.rs +++ b/beacon_node/db/src/stores/mod.rs @@ -12,8 +12,6 @@ pub use self::beacon_state_store::BeaconStateStore; pub use self::pow_chain_store::PoWChainStore; pub use self::validator_store::{ValidatorStore, ValidatorStoreError}; -use super::bls; - pub const BLOCKS_DB_COLUMN: &str = "blocks"; pub const STATES_DB_COLUMN: &str = "states"; pub const POW_CHAIN_DB_COLUMN: &str = "powchain"; diff --git a/beacon_node/db/src/stores/validator_store.rs b/beacon_node/db/src/stores/validator_store.rs index 500bb50af..02e90dc5c 100644 --- a/beacon_node/db/src/stores/validator_store.rs +++ b/beacon_node/db/src/stores/validator_store.rs @@ -1,9 +1,9 @@ extern crate bytes; use self::bytes::{BufMut, BytesMut}; -use super::bls::PublicKey; use super::VALIDATOR_DB_COLUMN as DB_COLUMN; use super::{ClientDB, DBError}; +use bls::PublicKey; use ssz::{ssz_encode, Decodable}; use std::sync::Arc; @@ -80,8 +80,8 @@ impl ValidatorStore { #[cfg(test)] mod tests { use super::super::super::MemoryDB; - use super::super::bls::Keypair; use super::*; + use bls::Keypair; #[test] fn test_prefix_bytes() { diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index f9e77723a..25239a9f6 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -15,8 +15,8 @@ use db::{ }; use slog::{error, info, o, Drain}; use slot_clock::SystemTimeSlotClock; -use spec::ChainSpec; use std::sync::Arc; +use types::ChainSpec; fn main() { let decorator = slog_term::TermDecorator::new().build(); diff --git a/eth2/block_producer/Cargo.toml b/eth2/block_producer/Cargo.toml index 7203dbe31..86dde92f7 100644 --- a/eth2/block_producer/Cargo.toml +++ b/eth2/block_producer/Cargo.toml @@ -6,6 +6,5 @@ edition = "2018" [dependencies] slot_clock = { path = "../../eth2/utils/slot_clock" } -spec = { path = "../../eth2/spec" } ssz = { path = "../../eth2/utils/ssz" } types = { path = "../../eth2/types" } diff --git a/eth2/block_producer/src/lib.rs b/eth2/block_producer/src/lib.rs index 74788eb60..8ce6da859 100644 --- a/eth2/block_producer/src/lib.rs +++ b/eth2/block_producer/src/lib.rs @@ -2,10 +2,9 @@ pub mod test_utils; mod traits; use slot_clock::SlotClock; -use spec::ChainSpec; use ssz::ssz_encode; use std::sync::Arc; -use types::{BeaconBlock, Hash256, ProposalSignedData, PublicKey}; +use types::{BeaconBlock, ChainSpec, Hash256, ProposalSignedData, PublicKey}; pub use self::traits::{ BeaconNode, BeaconNodeError, DutiesReader, DutiesReaderError, PublishOutcome, Signer, diff --git a/eth2/genesis/Cargo.toml b/eth2/genesis/Cargo.toml index 499333979..d56b3f929 100644 --- a/eth2/genesis/Cargo.toml +++ b/eth2/genesis/Cargo.toml @@ -6,7 +6,6 @@ edition = "2018" [dependencies] bls = { path = "../utils/bls" } -spec = { path = "../spec" } ssz = { path = "../utils/ssz" } types = { path = "../types" } validator_induction = { path = "../validator_induction" } diff --git a/eth2/genesis/src/beacon_block.rs b/eth2/genesis/src/beacon_block.rs index 478d76ded..8b78f9e2d 100644 --- a/eth2/genesis/src/beacon_block.rs +++ b/eth2/genesis/src/beacon_block.rs @@ -1,5 +1,4 @@ -use spec::ChainSpec; -use types::{BeaconBlock, BeaconBlockBody, Eth1Data, Hash256}; +use types::{BeaconBlock, BeaconBlockBody, ChainSpec, Eth1Data, Hash256}; /// Generate a genesis BeaconBlock. pub fn genesis_beacon_block(state_root: Hash256, spec: &ChainSpec) -> BeaconBlock { diff --git a/eth2/genesis/src/beacon_state.rs b/eth2/genesis/src/beacon_state.rs index 7644e782b..788af7c82 100644 --- a/eth2/genesis/src/beacon_state.rs +++ b/eth2/genesis/src/beacon_state.rs @@ -1,5 +1,4 @@ -use spec::ChainSpec; -use types::{BeaconState, Crosslink, Fork}; +use types::{BeaconState, ChainSpec, Crosslink, Fork}; use validator_shuffling::{shard_and_committees_for_cycle, ValidatorAssignmentError}; #[derive(Debug, PartialEq)] diff --git a/eth2/genesis/src/lib.rs b/eth2/genesis/src/lib.rs index 6f45863aa..003e66959 100644 --- a/eth2/genesis/src/lib.rs +++ b/eth2/genesis/src/lib.rs @@ -1,8 +1,3 @@ -extern crate spec; -extern crate types; -extern crate validator_induction; -extern crate validator_shuffling; - mod beacon_block; mod beacon_state; diff --git a/eth2/spec/Cargo.toml b/eth2/spec/Cargo.toml deleted file mode 100644 index 5c535ab41..000000000 --- a/eth2/spec/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "spec" -version = "0.1.0" -authors = ["Paul Hauner "] -edition = "2018" - -[dependencies] -bls = { path = "../utils/bls" } -types = { path = "../types" } diff --git a/eth2/types/src/attestation.rs b/eth2/types/src/attestation.rs index bef35bc78..911464ba0 100644 --- a/eth2/types/src/attestation.rs +++ b/eth2/types/src/attestation.rs @@ -1,8 +1,8 @@ -use super::bls::AggregateSignature; -use super::ssz::{Decodable, DecodeError, Encodable, SszStream}; use super::{AttestationData, Bitfield}; use crate::test_utils::TestRandom; +use bls::AggregateSignature; use rand::RngCore; +use ssz::{Decodable, DecodeError, Encodable, SszStream}; #[derive(Debug, Clone, PartialEq)] pub struct Attestation { @@ -62,7 +62,7 @@ impl TestRandom for Attestation { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; + use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; diff --git a/eth2/types/src/attestation_data.rs b/eth2/types/src/attestation_data.rs index c1e895d6c..682838075 100644 --- a/eth2/types/src/attestation_data.rs +++ b/eth2/types/src/attestation_data.rs @@ -1,7 +1,7 @@ -use super::ssz::{Decodable, DecodeError, Encodable, SszStream}; use super::Hash256; use crate::test_utils::TestRandom; use rand::RngCore; +use ssz::{Decodable, DecodeError, Encodable, SszStream}; pub const SSZ_ATTESTION_DATA_LENGTH: usize = { 8 + // slot @@ -102,7 +102,7 @@ impl TestRandom for AttestationData { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; + use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; diff --git a/eth2/types/src/attestation_data_and_custody_bit.rs b/eth2/types/src/attestation_data_and_custody_bit.rs index f53614d6b..5d5747f94 100644 --- a/eth2/types/src/attestation_data_and_custody_bit.rs +++ b/eth2/types/src/attestation_data_and_custody_bit.rs @@ -1,7 +1,7 @@ -use super::ssz::{Decodable, DecodeError, Encodable, SszStream}; use super::AttestationData; use crate::test_utils::TestRandom; use rand::RngCore; +use ssz::{Decodable, DecodeError, Encodable, SszStream}; #[derive(Debug, Clone, PartialEq, Default)] pub struct AttestationDataAndCustodyBit { @@ -39,7 +39,7 @@ impl TestRandom for AttestationDataAndCustodyBit { #[cfg(test)] mod test { - use super::super::ssz::ssz_encode; + use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; diff --git a/eth2/types/src/beacon_block.rs b/eth2/types/src/beacon_block.rs index 4dff199f3..7d80cd398 100644 --- a/eth2/types/src/beacon_block.rs +++ b/eth2/types/src/beacon_block.rs @@ -1,9 +1,9 @@ -use super::ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream}; use super::{BeaconBlockBody, Eth1Data, Hash256}; use crate::test_utils::TestRandom; use bls::Signature; use hashing::canonical_hash; use rand::RngCore; +use ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream}; #[derive(Debug, PartialEq, Clone)] pub struct BeaconBlock { @@ -77,7 +77,7 @@ impl TestRandom for BeaconBlock { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; + use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; diff --git a/eth2/types/src/beacon_block_body.rs b/eth2/types/src/beacon_block_body.rs index 67fa34d91..94a7cf0d7 100644 --- a/eth2/types/src/beacon_block_body.rs +++ b/eth2/types/src/beacon_block_body.rs @@ -1,7 +1,7 @@ -use super::ssz::{Decodable, DecodeError, Encodable, SszStream}; use super::{Attestation, CasperSlashing, Deposit, Exit, ProposerSlashing}; use crate::test_utils::TestRandom; use rand::RngCore; +use ssz::{Decodable, DecodeError, Encodable, SszStream}; // The following types are just dummy classes as they will not be defined until // Phase 1 (Sharding phase) @@ -78,7 +78,7 @@ impl TestRandom for BeaconBlockBody { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; + use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 34afeaa83..921d49360 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -203,7 +203,7 @@ impl TestRandom for BeaconState { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; + use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; diff --git a/eth2/types/src/casper_slashing.rs b/eth2/types/src/casper_slashing.rs index 08dbd9ff3..0bdfab511 100644 --- a/eth2/types/src/casper_slashing.rs +++ b/eth2/types/src/casper_slashing.rs @@ -1,7 +1,7 @@ -use super::ssz::{Decodable, DecodeError, Encodable, SszStream}; use super::SlashableVoteData; use crate::test_utils::TestRandom; use rand::RngCore; +use ssz::{Decodable, DecodeError, Encodable, SszStream}; #[derive(Debug, PartialEq, Clone)] pub struct CasperSlashing { @@ -42,7 +42,7 @@ impl TestRandom for CasperSlashing { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; + use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; diff --git a/eth2/types/src/crosslink.rs b/eth2/types/src/crosslink.rs index 69f94662a..50cc50305 100644 --- a/eth2/types/src/crosslink.rs +++ b/eth2/types/src/crosslink.rs @@ -1,7 +1,7 @@ -use super::ssz::{Decodable, DecodeError, Encodable, SszStream}; use super::Hash256; use crate::test_utils::TestRandom; use rand::RngCore; +use ssz::{Decodable, DecodeError, Encodable, SszStream}; #[derive(Clone, Debug, PartialEq)] pub struct Crosslink { @@ -52,7 +52,7 @@ impl TestRandom for Crosslink { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; + use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; diff --git a/eth2/types/src/deposit.rs b/eth2/types/src/deposit.rs index 9d84bc278..f5e716f9e 100644 --- a/eth2/types/src/deposit.rs +++ b/eth2/types/src/deposit.rs @@ -1,7 +1,7 @@ -use super::ssz::{Decodable, DecodeError, Encodable, SszStream}; use super::{DepositData, Hash256}; use crate::test_utils::TestRandom; use rand::RngCore; +use ssz::{Decodable, DecodeError, Encodable, SszStream}; #[derive(Debug, PartialEq, Clone)] pub struct Deposit { @@ -47,7 +47,7 @@ impl TestRandom for Deposit { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; + use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; diff --git a/eth2/types/src/deposit_data.rs b/eth2/types/src/deposit_data.rs index b85a95708..984a3f0f0 100644 --- a/eth2/types/src/deposit_data.rs +++ b/eth2/types/src/deposit_data.rs @@ -1,7 +1,7 @@ -use super::ssz::{Decodable, DecodeError, Encodable, SszStream}; use super::DepositInput; use crate::test_utils::TestRandom; use rand::RngCore; +use ssz::{Decodable, DecodeError, Encodable, SszStream}; #[derive(Debug, PartialEq, Clone)] pub struct DepositData { @@ -47,7 +47,7 @@ impl TestRandom for DepositData { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; + use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; diff --git a/eth2/types/src/deposit_input.rs b/eth2/types/src/deposit_input.rs index 4f2d71096..f02db2640 100644 --- a/eth2/types/src/deposit_input.rs +++ b/eth2/types/src/deposit_input.rs @@ -1,8 +1,8 @@ -use super::ssz::{Decodable, DecodeError, Encodable, SszStream}; use super::Hash256; use crate::test_utils::TestRandom; use bls::{PublicKey, Signature}; use rand::RngCore; +use ssz::{Decodable, DecodeError, Encodable, SszStream}; #[derive(Debug, PartialEq, Clone)] pub struct DepositInput { @@ -48,7 +48,7 @@ impl TestRandom for DepositInput { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; + use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; diff --git a/eth2/types/src/eth1_data.rs b/eth2/types/src/eth1_data.rs index a20559e18..0f1645242 100644 --- a/eth2/types/src/eth1_data.rs +++ b/eth2/types/src/eth1_data.rs @@ -1,7 +1,7 @@ -use super::ssz::{Decodable, DecodeError, Encodable, SszStream}; use super::Hash256; use crate::test_utils::TestRandom; use rand::RngCore; +use ssz::{Decodable, DecodeError, Encodable, SszStream}; // Note: this is refer to as DepositRootVote in specs #[derive(Debug, PartialEq, Clone, Default)] @@ -43,7 +43,7 @@ impl TestRandom for Eth1Data { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; + use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; diff --git a/eth2/types/src/eth1_data_vote.rs b/eth2/types/src/eth1_data_vote.rs index c4d9d01af..3353e4068 100644 --- a/eth2/types/src/eth1_data_vote.rs +++ b/eth2/types/src/eth1_data_vote.rs @@ -1,7 +1,7 @@ -use super::ssz::{Decodable, DecodeError, Encodable, SszStream}; use super::Eth1Data; use crate::test_utils::TestRandom; use rand::RngCore; +use ssz::{Decodable, DecodeError, Encodable, SszStream}; // Note: this is refer to as DepositRootVote in specs #[derive(Debug, PartialEq, Clone, Default)] @@ -43,7 +43,7 @@ impl TestRandom for Eth1DataVote { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; + use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; diff --git a/eth2/types/src/exit.rs b/eth2/types/src/exit.rs index eeac11ce7..a780e1966 100644 --- a/eth2/types/src/exit.rs +++ b/eth2/types/src/exit.rs @@ -1,7 +1,7 @@ -use super::ssz::{Decodable, DecodeError, Encodable, SszStream}; use crate::test_utils::TestRandom; use bls::Signature; use rand::RngCore; +use ssz::{Decodable, DecodeError, Encodable, SszStream}; #[derive(Debug, PartialEq, Clone)] pub struct Exit { @@ -47,7 +47,7 @@ impl TestRandom for Exit { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; + use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; diff --git a/eth2/types/src/fork.rs b/eth2/types/src/fork.rs index b70dafccc..f7e3135d6 100644 --- a/eth2/types/src/fork.rs +++ b/eth2/types/src/fork.rs @@ -1,6 +1,6 @@ -use super::ssz::{Decodable, DecodeError, Encodable, SszStream}; use crate::test_utils::TestRandom; use rand::RngCore; +use ssz::{Decodable, DecodeError, Encodable, SszStream}; #[derive(Debug, Clone, PartialEq, Default)] pub struct Fork { @@ -46,7 +46,7 @@ impl TestRandom for Fork { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; + use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index b4b43695d..3bf13e2d4 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -1,8 +1,3 @@ -extern crate bls; -extern crate boolean_bitfield; -extern crate ethereum_types; -extern crate ssz; - pub mod test_utils; pub mod attestation; @@ -26,6 +21,7 @@ pub mod proposer_slashing; pub mod shard_committee; pub mod shard_reassignment_record; pub mod slashable_vote_data; +pub mod spec; pub mod special_record; pub mod validator; pub mod validator_registry; @@ -33,7 +29,7 @@ pub mod validator_registry_delta_block; pub mod readers; -use self::ethereum_types::{H160, H256, U256}; +use ethereum_types::{H160, H256, U256}; use std::collections::HashMap; pub use crate::attestation::Attestation; @@ -56,6 +52,7 @@ pub use crate::proposal_signed_data::ProposalSignedData; pub use crate::proposer_slashing::ProposerSlashing; pub use crate::shard_committee::ShardCommittee; pub use crate::slashable_vote_data::SlashableVoteData; +pub use crate::spec::ChainSpec; pub use crate::special_record::{SpecialRecord, SpecialRecordKind}; pub use crate::validator::{StatusFlags as ValidatorStatusFlags, Validator}; pub use crate::validator_registry_delta_block::ValidatorRegistryDeltaBlock; diff --git a/eth2/types/src/pending_attestation.rs b/eth2/types/src/pending_attestation.rs index ad3dbb782..0453b11ed 100644 --- a/eth2/types/src/pending_attestation.rs +++ b/eth2/types/src/pending_attestation.rs @@ -1,7 +1,7 @@ -use super::ssz::{Decodable, DecodeError, Encodable, SszStream}; use super::{AttestationData, Bitfield}; use crate::test_utils::TestRandom; use rand::RngCore; +use ssz::{Decodable, DecodeError, Encodable, SszStream}; #[derive(Debug, Clone, PartialEq)] pub struct PendingAttestation { @@ -52,7 +52,7 @@ impl TestRandom for PendingAttestation { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; + use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; diff --git a/eth2/types/src/proposal_signed_data.rs b/eth2/types/src/proposal_signed_data.rs index e38a9cadb..9a1f2f0b2 100644 --- a/eth2/types/src/proposal_signed_data.rs +++ b/eth2/types/src/proposal_signed_data.rs @@ -1,7 +1,7 @@ -use super::ssz::{Decodable, DecodeError, Encodable, SszStream}; use super::Hash256; use crate::test_utils::TestRandom; use rand::RngCore; +use ssz::{Decodable, DecodeError, Encodable, SszStream}; #[derive(Debug, PartialEq, Clone, Default)] pub struct ProposalSignedData { @@ -47,7 +47,7 @@ impl TestRandom for ProposalSignedData { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; + use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; diff --git a/eth2/types/src/proposer_slashing.rs b/eth2/types/src/proposer_slashing.rs index 3754c3b32..c02dc8c8e 100644 --- a/eth2/types/src/proposer_slashing.rs +++ b/eth2/types/src/proposer_slashing.rs @@ -1,8 +1,8 @@ -use super::ssz::{Decodable, DecodeError, Encodable, SszStream}; use super::ProposalSignedData; use crate::test_utils::TestRandom; use bls::Signature; use rand::RngCore; +use ssz::{Decodable, DecodeError, Encodable, SszStream}; #[derive(Debug, PartialEq, Clone)] pub struct ProposerSlashing { @@ -58,7 +58,7 @@ impl TestRandom for ProposerSlashing { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; + use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; diff --git a/eth2/types/src/shard_committee.rs b/eth2/types/src/shard_committee.rs index d920f7db3..961140fbd 100644 --- a/eth2/types/src/shard_committee.rs +++ b/eth2/types/src/shard_committee.rs @@ -1,6 +1,6 @@ -use super::ssz::{Decodable, DecodeError, Encodable, SszStream}; use crate::test_utils::TestRandom; use rand::RngCore; +use ssz::{Decodable, DecodeError, Encodable, SszStream}; #[derive(Clone, Debug, PartialEq)] pub struct ShardCommittee { @@ -35,7 +35,7 @@ impl TestRandom for ShardCommittee { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; + use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; diff --git a/eth2/types/src/shard_reassignment_record.rs b/eth2/types/src/shard_reassignment_record.rs index 9bc014689..f4c976bda 100644 --- a/eth2/types/src/shard_reassignment_record.rs +++ b/eth2/types/src/shard_reassignment_record.rs @@ -1,6 +1,6 @@ -use super::ssz::{Decodable, DecodeError, Encodable, SszStream}; use crate::test_utils::TestRandom; use rand::RngCore; +use ssz::{Decodable, DecodeError, Encodable, SszStream}; #[derive(Debug, PartialEq, Clone)] pub struct ShardReassignmentRecord { @@ -46,9 +46,9 @@ impl TestRandom for ShardReassignmentRecord { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/slashable_vote_data.rs b/eth2/types/src/slashable_vote_data.rs index e6fa36bc1..2984c9298 100644 --- a/eth2/types/src/slashable_vote_data.rs +++ b/eth2/types/src/slashable_vote_data.rs @@ -1,8 +1,8 @@ -use super::ssz::{Decodable, DecodeError, Encodable, SszStream}; use super::AttestationData; use crate::test_utils::TestRandom; use bls::AggregateSignature; use rand::RngCore; +use ssz::{Decodable, DecodeError, Encodable, SszStream}; #[derive(Debug, PartialEq, Clone)] pub struct SlashableVoteData { @@ -53,7 +53,7 @@ impl TestRandom for SlashableVoteData { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; + use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; diff --git a/eth2/spec/src/foundation.rs b/eth2/types/src/spec/foundation.rs similarity index 98% rename from eth2/spec/src/foundation.rs rename to eth2/types/src/spec/foundation.rs index 8508a74e1..f24a55b2a 100644 --- a/eth2/spec/src/foundation.rs +++ b/eth2/types/src/spec/foundation.rs @@ -1,7 +1,7 @@ use super::ChainSpec; use bls::{Keypair, PublicKey, SecretKey, Signature}; -use types::{Address, Eth1Data, Hash256, Validator}; +use crate::{Address, Eth1Data, Hash256, Validator}; /// The size of a validators deposit in GWei. pub const DEPOSIT_GWEI: u64 = 32_000_000_000; diff --git a/eth2/spec/src/lib.rs b/eth2/types/src/spec/mod.rs similarity index 95% rename from eth2/spec/src/lib.rs rename to eth2/types/src/spec/mod.rs index aa3906076..077b6bef5 100644 --- a/eth2/spec/src/lib.rs +++ b/eth2/types/src/spec/mod.rs @@ -1,10 +1,7 @@ -extern crate bls; -extern crate types; - mod foundation; +use crate::{Address, Eth1Data, Hash256, Validator}; use bls::Signature; -use types::{Address, Eth1Data, Hash256, Validator}; #[derive(PartialEq, Debug, Clone)] pub struct ChainSpec { diff --git a/eth2/types/src/special_record.rs b/eth2/types/src/special_record.rs index 0a0a6978a..47ee3de36 100644 --- a/eth2/types/src/special_record.rs +++ b/eth2/types/src/special_record.rs @@ -1,4 +1,4 @@ -use super::ssz::{Decodable, DecodeError, Encodable, SszStream}; +use ssz::{Decodable, DecodeError, Encodable, SszStream}; /// The value of the "type" field of SpecialRecord. /// diff --git a/eth2/types/src/validator.rs b/eth2/types/src/validator.rs index 8408e7423..1015f70b2 100644 --- a/eth2/types/src/validator.rs +++ b/eth2/types/src/validator.rs @@ -1,6 +1,6 @@ -use super::bls::PublicKey; use super::Hash256; use crate::test_utils::TestRandom; +use bls::PublicKey; use rand::RngCore; use ssz::{Decodable, DecodeError, Encodable, SszStream}; @@ -162,7 +162,7 @@ impl TestRandom for Validator { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; + use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; diff --git a/eth2/types/src/validator_registry_delta_block.rs b/eth2/types/src/validator_registry_delta_block.rs index 57aa469df..253671cda 100644 --- a/eth2/types/src/validator_registry_delta_block.rs +++ b/eth2/types/src/validator_registry_delta_block.rs @@ -72,7 +72,7 @@ impl TestRandom for ValidatorRegistryDeltaBlock { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; + use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; diff --git a/eth2/utils/bls/src/aggregate_signature.rs b/eth2/utils/bls/src/aggregate_signature.rs index 6012f78c1..5f02d93f8 100644 --- a/eth2/utils/bls/src/aggregate_signature.rs +++ b/eth2/utils/bls/src/aggregate_signature.rs @@ -1,6 +1,6 @@ -use super::ssz::{decode_ssz_list, Decodable, DecodeError, Encodable, SszStream}; use super::{AggregatePublicKey, Signature}; use bls_aggregates::AggregateSignature as RawAggregateSignature; +use ssz::{decode_ssz_list, Decodable, DecodeError, Encodable, SszStream}; /// A BLS aggregate signature. /// @@ -46,8 +46,8 @@ impl Decodable for AggregateSignature { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; use super::super::{Keypair, Signature}; + use super::ssz::ssz_encode; use super::*; #[test] diff --git a/eth2/utils/bls/src/public_key.rs b/eth2/utils/bls/src/public_key.rs index 4a53893f0..981f25a75 100644 --- a/eth2/utils/bls/src/public_key.rs +++ b/eth2/utils/bls/src/public_key.rs @@ -67,7 +67,7 @@ impl Hash for PublicKey { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; + use super::ssz::ssz_encode; use super::*; #[test] diff --git a/eth2/utils/bls/src/signature.rs b/eth2/utils/bls/src/signature.rs index 7f9e61718..59fcda725 100644 --- a/eth2/utils/bls/src/signature.rs +++ b/eth2/utils/bls/src/signature.rs @@ -1,6 +1,6 @@ -use super::ssz::{decode_ssz_list, Decodable, DecodeError, Encodable, SszStream}; use super::{PublicKey, SecretKey}; use bls_aggregates::Signature as RawSignature; +use ssz::{decode_ssz_list, Decodable, DecodeError, Encodable, SszStream}; /// A single BLS signature. /// @@ -59,8 +59,8 @@ impl Decodable for Signature { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; use super::super::Keypair; + use super::ssz::ssz_encode; use super::*; #[test] diff --git a/eth2/validator_induction/Cargo.toml b/eth2/validator_induction/Cargo.toml index 3530bbbf1..5907014df 100644 --- a/eth2/validator_induction/Cargo.toml +++ b/eth2/validator_induction/Cargo.toml @@ -8,4 +8,3 @@ edition = "2018" bls = { path = "../utils/bls" } hashing = { path = "../utils/hashing" } types = { path = "../types" } -spec = { path = "../spec" } diff --git a/eth2/validator_induction/src/inductor.rs b/eth2/validator_induction/src/inductor.rs index 6efc9af2e..6b3cfdf16 100644 --- a/eth2/validator_induction/src/inductor.rs +++ b/eth2/validator_induction/src/inductor.rs @@ -1,6 +1,5 @@ use bls::verify_proof_of_possession; -use spec::ChainSpec; -use types::{BeaconState, Deposit, Validator}; +use types::{BeaconState, ChainSpec, Deposit, Validator}; #[derive(Debug, PartialEq, Clone)] pub enum ValidatorInductionError { diff --git a/eth2/validator_induction/src/lib.rs b/eth2/validator_induction/src/lib.rs index 7119d6c2e..3a4fa9a14 100644 --- a/eth2/validator_induction/src/lib.rs +++ b/eth2/validator_induction/src/lib.rs @@ -1,8 +1,3 @@ -extern crate bls; -extern crate hashing; -extern crate spec; -extern crate types; - mod inductor; pub use crate::inductor::{process_deposit, ValidatorInductionError}; diff --git a/eth2/validator_shuffling/Cargo.toml b/eth2/validator_shuffling/Cargo.toml index 99a97c5ec..ae2babf1a 100644 --- a/eth2/validator_shuffling/Cargo.toml +++ b/eth2/validator_shuffling/Cargo.toml @@ -6,6 +6,5 @@ edition = "2018" [dependencies] honey-badger-split = { path = "../utils/honey-badger-split" } -spec = { path = "../spec" } types = { path = "../types" } vec_shuffle = { path = "../utils/vec_shuffle" } diff --git a/eth2/validator_shuffling/src/lib.rs b/eth2/validator_shuffling/src/lib.rs index ee59ffecc..2307dd301 100644 --- a/eth2/validator_shuffling/src/lib.rs +++ b/eth2/validator_shuffling/src/lib.rs @@ -1,8 +1,3 @@ -extern crate honey_badger_split; -extern crate spec; -extern crate types; -extern crate vec_shuffle; - mod shuffle; pub use crate::shuffle::{shard_and_committees_for_cycle, ValidatorAssignmentError}; diff --git a/eth2/validator_shuffling/src/shuffle.rs b/eth2/validator_shuffling/src/shuffle.rs index 734f5995c..48d16fddc 100644 --- a/eth2/validator_shuffling/src/shuffle.rs +++ b/eth2/validator_shuffling/src/shuffle.rs @@ -1,9 +1,8 @@ use std::cmp::min; use honey_badger_split::SplitExt; -use spec::ChainSpec; use types::validator_registry::get_active_validator_indices; -use types::{ShardCommittee, Validator}; +use types::{ChainSpec, ShardCommittee, Validator}; use vec_shuffle::{shuffle, ShuffleErr}; type DelegatedCycle = Vec>; diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index c4f8b8f4a..8ab515e15 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -13,7 +13,6 @@ grpcio = { version = "0.4", default-features = false, features = ["protobuf-code protobuf = "2.0.2" protos = { path = "../protos" } slot_clock = { path = "../eth2/utils/slot_clock" } -spec = { path = "../eth2/spec" } types = { path = "../eth2/types" } slog = "^2.2.3" slog-term = "^2.4.0" diff --git a/validator_client/src/duties/mod.rs b/validator_client/src/duties/mod.rs index f368d6dca..c9ebda249 100644 --- a/validator_client/src/duties/mod.rs +++ b/validator_client/src/duties/mod.rs @@ -11,8 +11,8 @@ pub use self::service::DutiesManagerService; use self::traits::{BeaconNode, BeaconNodeError}; use bls::PublicKey; use slot_clock::SlotClock; -use spec::ChainSpec; use std::sync::Arc; +use types::ChainSpec; #[derive(Debug, PartialEq, Clone, Copy)] pub enum PollOutcome { diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index 49927b8c9..3b516870a 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -8,10 +8,10 @@ use grpcio::{ChannelBuilder, EnvBuilder}; use protos::services_grpc::{BeaconBlockServiceClient, ValidatorServiceClient}; use slog::{error, info, o, Drain}; use slot_clock::SystemTimeSlotClock; -use spec::ChainSpec; use std::path::PathBuf; use std::sync::Arc; use std::thread; +use types::ChainSpec; mod block_producer_service; mod config; From 5ef02688d5fbd9d02dd45328ff8a479c150a0e7e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 25 Jan 2019 16:46:16 +1100 Subject: [PATCH 024/107] Remove pre-2018 extern crate uses --- eth2/types/src/attestation.rs | 2 +- eth2/types/src/attestation_data.rs | 2 +- eth2/types/src/attestation_data_and_custody_bit.rs | 2 +- eth2/types/src/{beacon_block.rs => beacon_block/mod.rs} | 4 +++- eth2/types/src/beacon_block_body.rs | 2 +- eth2/types/src/casper_slashing.rs | 2 +- eth2/types/src/crosslink.rs | 2 +- eth2/types/src/deposit.rs | 2 +- eth2/types/src/deposit_data.rs | 2 +- eth2/types/src/deposit_input.rs | 2 +- eth2/types/src/eth1_data.rs | 2 +- eth2/types/src/eth1_data_vote.rs | 2 +- eth2/types/src/exit.rs | 2 +- eth2/types/src/fork.rs | 2 +- eth2/types/src/pending_attestation.rs | 2 +- eth2/types/src/proposal_signed_data.rs | 2 +- eth2/types/src/proposer_slashing.rs | 2 +- eth2/types/src/shard_committee.rs | 2 +- eth2/types/src/slashable_vote_data.rs | 2 +- eth2/types/src/validator.rs | 2 +- eth2/types/src/validator_registry_delta_block.rs | 2 +- eth2/utils/bls/src/aggregate_signature.rs | 2 +- eth2/utils/bls/src/public_key.rs | 2 +- eth2/utils/bls/src/secret_key.rs | 2 +- eth2/utils/bls/src/signature.rs | 2 +- 25 files changed, 27 insertions(+), 25 deletions(-) rename eth2/types/src/{beacon_block.rs => beacon_block/mod.rs} (98%) diff --git a/eth2/types/src/attestation.rs b/eth2/types/src/attestation.rs index 911464ba0..e53c91fbd 100644 --- a/eth2/types/src/attestation.rs +++ b/eth2/types/src/attestation.rs @@ -62,9 +62,9 @@ impl TestRandom for Attestation { #[cfg(test)] mod tests { - use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/attestation_data.rs b/eth2/types/src/attestation_data.rs index 682838075..9a035472a 100644 --- a/eth2/types/src/attestation_data.rs +++ b/eth2/types/src/attestation_data.rs @@ -102,9 +102,9 @@ impl TestRandom for AttestationData { #[cfg(test)] mod tests { - use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/attestation_data_and_custody_bit.rs b/eth2/types/src/attestation_data_and_custody_bit.rs index 5d5747f94..b6a160189 100644 --- a/eth2/types/src/attestation_data_and_custody_bit.rs +++ b/eth2/types/src/attestation_data_and_custody_bit.rs @@ -39,9 +39,9 @@ impl TestRandom for AttestationDataAndCustodyBit { #[cfg(test)] mod test { - use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/beacon_block.rs b/eth2/types/src/beacon_block/mod.rs similarity index 98% rename from eth2/types/src/beacon_block.rs rename to eth2/types/src/beacon_block/mod.rs index 7d80cd398..685658c07 100644 --- a/eth2/types/src/beacon_block.rs +++ b/eth2/types/src/beacon_block/mod.rs @@ -5,6 +5,8 @@ use hashing::canonical_hash; use rand::RngCore; use ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream}; +mod signing; + #[derive(Debug, PartialEq, Clone)] pub struct BeaconBlock { pub slot: u64, @@ -77,9 +79,9 @@ impl TestRandom for BeaconBlock { #[cfg(test)] mod tests { - use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/beacon_block_body.rs b/eth2/types/src/beacon_block_body.rs index 94a7cf0d7..722264b12 100644 --- a/eth2/types/src/beacon_block_body.rs +++ b/eth2/types/src/beacon_block_body.rs @@ -78,9 +78,9 @@ impl TestRandom for BeaconBlockBody { #[cfg(test)] mod tests { - use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/casper_slashing.rs b/eth2/types/src/casper_slashing.rs index 0bdfab511..8ea0cf5f1 100644 --- a/eth2/types/src/casper_slashing.rs +++ b/eth2/types/src/casper_slashing.rs @@ -42,9 +42,9 @@ impl TestRandom for CasperSlashing { #[cfg(test)] mod tests { - use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/crosslink.rs b/eth2/types/src/crosslink.rs index 50cc50305..f7ba66d5e 100644 --- a/eth2/types/src/crosslink.rs +++ b/eth2/types/src/crosslink.rs @@ -52,9 +52,9 @@ impl TestRandom for Crosslink { #[cfg(test)] mod tests { - use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/deposit.rs b/eth2/types/src/deposit.rs index f5e716f9e..3fbbd817e 100644 --- a/eth2/types/src/deposit.rs +++ b/eth2/types/src/deposit.rs @@ -47,9 +47,9 @@ impl TestRandom for Deposit { #[cfg(test)] mod tests { - use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/deposit_data.rs b/eth2/types/src/deposit_data.rs index 984a3f0f0..00e553a46 100644 --- a/eth2/types/src/deposit_data.rs +++ b/eth2/types/src/deposit_data.rs @@ -47,9 +47,9 @@ impl TestRandom for DepositData { #[cfg(test)] mod tests { - use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/deposit_input.rs b/eth2/types/src/deposit_input.rs index f02db2640..47018419e 100644 --- a/eth2/types/src/deposit_input.rs +++ b/eth2/types/src/deposit_input.rs @@ -48,9 +48,9 @@ impl TestRandom for DepositInput { #[cfg(test)] mod tests { - use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/eth1_data.rs b/eth2/types/src/eth1_data.rs index 0f1645242..d239372d8 100644 --- a/eth2/types/src/eth1_data.rs +++ b/eth2/types/src/eth1_data.rs @@ -43,9 +43,9 @@ impl TestRandom for Eth1Data { #[cfg(test)] mod tests { - use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/eth1_data_vote.rs b/eth2/types/src/eth1_data_vote.rs index 3353e4068..0dc6b90dd 100644 --- a/eth2/types/src/eth1_data_vote.rs +++ b/eth2/types/src/eth1_data_vote.rs @@ -43,9 +43,9 @@ impl TestRandom for Eth1DataVote { #[cfg(test)] mod tests { - use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/exit.rs b/eth2/types/src/exit.rs index a780e1966..cc99718b2 100644 --- a/eth2/types/src/exit.rs +++ b/eth2/types/src/exit.rs @@ -47,9 +47,9 @@ impl TestRandom for Exit { #[cfg(test)] mod tests { - use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/fork.rs b/eth2/types/src/fork.rs index f7e3135d6..5026dc621 100644 --- a/eth2/types/src/fork.rs +++ b/eth2/types/src/fork.rs @@ -46,9 +46,9 @@ impl TestRandom for Fork { #[cfg(test)] mod tests { - use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/pending_attestation.rs b/eth2/types/src/pending_attestation.rs index 0453b11ed..3cb6110de 100644 --- a/eth2/types/src/pending_attestation.rs +++ b/eth2/types/src/pending_attestation.rs @@ -52,9 +52,9 @@ impl TestRandom for PendingAttestation { #[cfg(test)] mod tests { - use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/proposal_signed_data.rs b/eth2/types/src/proposal_signed_data.rs index 9a1f2f0b2..500c0079f 100644 --- a/eth2/types/src/proposal_signed_data.rs +++ b/eth2/types/src/proposal_signed_data.rs @@ -47,9 +47,9 @@ impl TestRandom for ProposalSignedData { #[cfg(test)] mod tests { - use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/proposer_slashing.rs b/eth2/types/src/proposer_slashing.rs index c02dc8c8e..eb72ef0b2 100644 --- a/eth2/types/src/proposer_slashing.rs +++ b/eth2/types/src/proposer_slashing.rs @@ -58,9 +58,9 @@ impl TestRandom for ProposerSlashing { #[cfg(test)] mod tests { - use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/shard_committee.rs b/eth2/types/src/shard_committee.rs index 961140fbd..5d37ffe4c 100644 --- a/eth2/types/src/shard_committee.rs +++ b/eth2/types/src/shard_committee.rs @@ -35,9 +35,9 @@ impl TestRandom for ShardCommittee { #[cfg(test)] mod tests { - use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/slashable_vote_data.rs b/eth2/types/src/slashable_vote_data.rs index 2984c9298..ba5258d11 100644 --- a/eth2/types/src/slashable_vote_data.rs +++ b/eth2/types/src/slashable_vote_data.rs @@ -53,9 +53,9 @@ impl TestRandom for SlashableVoteData { #[cfg(test)] mod tests { - use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/validator.rs b/eth2/types/src/validator.rs index 1015f70b2..446b4d58c 100644 --- a/eth2/types/src/validator.rs +++ b/eth2/types/src/validator.rs @@ -162,9 +162,9 @@ impl TestRandom for Validator { #[cfg(test)] mod tests { - use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/validator_registry_delta_block.rs b/eth2/types/src/validator_registry_delta_block.rs index 253671cda..796b2fc9e 100644 --- a/eth2/types/src/validator_registry_delta_block.rs +++ b/eth2/types/src/validator_registry_delta_block.rs @@ -72,9 +72,9 @@ impl TestRandom for ValidatorRegistryDeltaBlock { #[cfg(test)] mod tests { - use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/utils/bls/src/aggregate_signature.rs b/eth2/utils/bls/src/aggregate_signature.rs index 5f02d93f8..5d5d92251 100644 --- a/eth2/utils/bls/src/aggregate_signature.rs +++ b/eth2/utils/bls/src/aggregate_signature.rs @@ -47,8 +47,8 @@ impl Decodable for AggregateSignature { #[cfg(test)] mod tests { use super::super::{Keypair, Signature}; - use super::ssz::ssz_encode; use super::*; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/utils/bls/src/public_key.rs b/eth2/utils/bls/src/public_key.rs index 981f25a75..eb8419dd9 100644 --- a/eth2/utils/bls/src/public_key.rs +++ b/eth2/utils/bls/src/public_key.rs @@ -67,8 +67,8 @@ impl Hash for PublicKey { #[cfg(test)] mod tests { - use super::ssz::ssz_encode; use super::*; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/utils/bls/src/secret_key.rs b/eth2/utils/bls/src/secret_key.rs index e86e8dec0..54dfa2f24 100644 --- a/eth2/utils/bls/src/secret_key.rs +++ b/eth2/utils/bls/src/secret_key.rs @@ -42,8 +42,8 @@ impl Decodable for SecretKey { #[cfg(test)] mod tests { - use super::super::ssz::ssz_encode; use super::*; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/utils/bls/src/signature.rs b/eth2/utils/bls/src/signature.rs index 59fcda725..99790fa37 100644 --- a/eth2/utils/bls/src/signature.rs +++ b/eth2/utils/bls/src/signature.rs @@ -60,8 +60,8 @@ impl Decodable for Signature { #[cfg(test)] mod tests { use super::super::Keypair; - use super::ssz::ssz_encode; use super::*; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { From 138fcd6275e0755ee120f40eef017ab38802d2e2 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 25 Jan 2019 16:47:24 +1100 Subject: [PATCH 025/107] Fix issues with old state information --- .../beacon_chain/src/block_production.rs | 3 +- .../beacon_chain/src/canonical_head.rs | 40 +++++- beacon_node/beacon_chain/src/info.rs | 16 +-- .../beacon_chain/src/state_transition.rs | 122 +++++++----------- beacon_node/beacon_chain/src/transition.rs | 29 ----- beacon_node/beacon_chain/tests/chain.rs | 41 +----- eth2/block_producer/src/lib.rs | 30 ++--- eth2/types/src/beacon_block/signing.rs | 19 +++ .../{beacon_state.rs => beacon_state/mod.rs} | 6 +- eth2/types/src/beacon_state/slot_advance.rs | 45 +++++++ eth2/utils/hashing/Cargo.toml | 1 + eth2/utils/hashing/src/lib.rs | 7 +- 12 files changed, 188 insertions(+), 171 deletions(-) delete mode 100644 beacon_node/beacon_chain/src/transition.rs create mode 100644 eth2/types/src/beacon_block/signing.rs rename eth2/types/src/{beacon_state.rs => beacon_state/mod.rs} (98%) create mode 100644 eth2/types/src/beacon_state/slot_advance.rs diff --git a/beacon_node/beacon_chain/src/block_production.rs b/beacon_node/beacon_chain/src/block_production.rs index c0a057ad2..d633b3490 100644 --- a/beacon_node/beacon_chain/src/block_production.rs +++ b/beacon_node/beacon_chain/src/block_production.rs @@ -68,7 +68,8 @@ where }, }; - let state = self.state_transition(parent_state, &block)?; + let state = + self.state_transition_without_verifying_block_signature(parent_state, &block)?; let state_root = state.canonical_root(); diff --git a/beacon_node/beacon_chain/src/canonical_head.rs b/beacon_node/beacon_chain/src/canonical_head.rs index acccb5de7..f6ab646a8 100644 --- a/beacon_node/beacon_chain/src/canonical_head.rs +++ b/beacon_node/beacon_chain/src/canonical_head.rs @@ -1,6 +1,11 @@ use crate::{BeaconChain, CheckPoint, ClientDB, SlotClock}; use std::sync::RwLockReadGuard; -use types::{BeaconBlock, BeaconState, Hash256}; +use types::{beacon_state::SlotProcessingError, BeaconBlock, BeaconState, Hash256}; + +pub enum Error { + PastSlot, + UnableToDetermineProducer, +} impl BeaconChain where @@ -31,4 +36,37 @@ where .read() .expect("CRITICAL: CanonicalHead poisioned.") } + + pub fn state(&self, slot: u64) -> Result { + let mut state = self + .canonical_head + .read() + .expect("CRITICAL: CanonicalHead poisioned.") + .beacon_state + .clone(); + let previous_block_root = self + .canonical_head + .read() + .expect("CRITICAL: CanonicalHead poisioned.") + .beacon_block_root + .clone(); + + match slot.checked_sub(state.slot) { + None => Err(Error::PastSlot), + Some(distance) => { + for _ in 0..distance { + state.per_slot_processing(previous_block_root.clone(), &self.spec)? + } + Ok(state) + } + } + } +} + +impl From for Error { + fn from(e: SlotProcessingError) -> Error { + match e { + SlotProcessingError::UnableToDetermineProducer => Error::UnableToDetermineProducer, + } + } } diff --git a/beacon_node/beacon_chain/src/info.rs b/beacon_node/beacon_chain/src/info.rs index 2025ba27a..eddb2fdb9 100644 --- a/beacon_node/beacon_chain/src/info.rs +++ b/beacon_node/beacon_chain/src/info.rs @@ -22,12 +22,10 @@ where } pub fn proposer_slots(&self, validator_index: usize) -> Option { - if let Some(validator) = self - .canonical_head() - .beacon_state - .validator_registry - .get(validator_index) - { + let slot = self.present_slot()?; + let state = self.state(slot).ok()?; + + if let Some(validator) = state.validator_registry.get(validator_index) { Some(validator.proposer_slots) } else { None @@ -42,8 +40,8 @@ where } pub fn block_proposer(&self, slot: u64) -> Option { - //TODO: this is a stub; fix. - let validator_count = self.canonical_head().beacon_state.validator_registry.len(); - Some((slot as usize) % validator_count) + let present_slot = self.present_slot()?; + let state = self.state(present_slot).ok()?; + state.get_beacon_proposer_index(slot, &self.spec) } } diff --git a/beacon_node/beacon_chain/src/state_transition.rs b/beacon_node/beacon_chain/src/state_transition.rs index 95fd9a7ad..fe2b0e7b1 100644 --- a/beacon_node/beacon_chain/src/state_transition.rs +++ b/beacon_node/beacon_chain/src/state_transition.rs @@ -4,8 +4,9 @@ use boolean_bitfield::BooleanBitfield; use slot_clock::{SystemTimeSlotClockError, TestingSlotClockError}; use ssz::ssz_encode; use types::{ - readers::BeaconBlockReader, AttestationData, AttestationDataAndCustodyBit, BeaconBlock, - BeaconState, Exit, Fork, Hash256, PendingAttestation, ProposalSignedData, + beacon_state::SlotProcessingError, readers::BeaconBlockReader, AttestationData, + AttestationDataAndCustodyBit, BeaconBlock, BeaconState, Exit, Fork, Hash256, + PendingAttestation, }; // TODO: define elsehwere. @@ -58,14 +59,31 @@ where U: SlotClock, { pub fn state_transition( + &self, + state: BeaconState, + block: &BeaconBlock, + ) -> Result { + self.internal_state_transition(state, block, true) + } + + pub fn state_transition_without_verifying_block_signature( + &self, + state: BeaconState, + block: &BeaconBlock, + ) -> Result { + self.internal_state_transition(state, block, false) + } + + fn internal_state_transition( &self, mut state: BeaconState, block: &BeaconBlock, + verify_block_signature: bool, ) -> Result { ensure!(state.slot < block.slot, Error::StateAlreadyTransitioned); for _ in state.slot..block.slot { - self.per_slot_processing(&mut state, &block.parent_root)?; + state.per_slot_processing(block.parent_root.clone(), &self.spec)?; } /* @@ -78,44 +96,35 @@ where * Proposer Signature */ - let block_without_signature_root = { - let mut block_without_signature = block.clone(); - block_without_signature.signature = self.spec.empty_signature.clone(); - block_without_signature.canonical_root() - }; - - let proposal_root = { - let proposal = ProposalSignedData { - slot: state.slot, - shard: self.spec.beacon_chain_shard_number, - block_root: block_without_signature_root, - }; - hash_tree_root(&proposal) - }; - - let block_proposer_index = - get_beacon_proposer_index(&state, block.slot, self.spec.epoch_length) - .ok_or(Error::NoBlockProducer)?; + let block_proposer_index = state + .get_beacon_proposer_index(block.slot, &self.spec) + .ok_or(Error::NoBlockProducer)?; let block_proposer = &state.validator_registry[block_proposer_index]; - ensure!( - bls_verify( - &block_proposer.pubkey, - &proposal_root, - &block.signature, - get_domain(&state.fork_data, state.slot, DOMAIN_PROPOSAL) - ), - Error::BadBlockSignature - ); + if verify_block_signature { + ensure!( + bls_verify( + &block_proposer.pubkey, + &block.proposal_root(&self.spec)[..], + &block.signature, + get_domain(&state.fork_data, state.slot, DOMAIN_PROPOSAL) + ), + Error::BadBlockSignature + ); + } /* * RANDAO */ + println!("proposer pubkey: {:?}", &block_proposer.pubkey); ensure!( bls_verify( &block_proposer.pubkey, - &ssz_encode(&block_proposer.proposer_slots), + // TODO: https://github.com/ethereum/eth2.0-specs/pull/496 + // + // &ssz_encode(&block_proposer.proposer_slots), + &ssz_encode(&block.slot), &block.randao_reveal, get_domain(&state.fork_data, state.slot, DOMAIN_RANDAO) ), @@ -358,35 +367,6 @@ where Ok(state) } - - fn per_slot_processing( - &self, - state: &mut BeaconState, - previous_block_root: &Hash256, - ) -> Result<(), Error> { - let epoch_length = self.spec.epoch_length; - let latest_randao_mixes_length = self.spec.latest_randao_mixes_length; - let latest_block_roots_length = self.spec.latest_block_roots_length; - - // Misc counters. - state.slot += 1; - let block_proposer = get_beacon_proposer_index(&state, state.slot, epoch_length) - .ok_or(Error::NoBlockProducer)?; - state.validator_registry[block_proposer].proposer_slots += 1; - state.latest_randao_mixes[(state.slot % latest_randao_mixes_length) as usize] = - state.latest_randao_mixes[((state.slot - 1) % latest_randao_mixes_length) as usize]; - - // Block roots. - state.latest_block_roots - [((state.slot - 1) % self.spec.latest_block_roots_length) as usize] = - *previous_block_root; - - if state.slot % latest_block_roots_length == 0 { - let root = merkle_root(&state.latest_block_roots[..]); - state.batched_block_roots.push(root); - } - Ok(()) - } } fn initiate_validator_exit(_state: &BeaconState, _index: u32) { @@ -450,20 +430,6 @@ fn hash_tree_root(_input: &T) -> Hash256 { Hash256::zero() } -fn merkle_root(_input: &[Hash256]) -> Hash256 { - // TODO: stubbed out. - Hash256::zero() -} - -fn get_beacon_proposer_index( - _state: &BeaconState, - _slot: u64, - _epoch_length: u64, -) -> Option { - // TODO: stubbed out. - Some(0) -} - impl From for Error { fn from(e: DBError) -> Error { Error::DBError(e.message) @@ -481,3 +447,11 @@ impl From for Error { Error::SlotClockError(e) } } + +impl From for Error { + fn from(e: SlotProcessingError) -> Error { + match e { + SlotProcessingError::UnableToDetermineProducer => Error::NoBlockProducer, + } + } +} diff --git a/beacon_node/beacon_chain/src/transition.rs b/beacon_node/beacon_chain/src/transition.rs deleted file mode 100644 index df434cc0c..000000000 --- a/beacon_node/beacon_chain/src/transition.rs +++ /dev/null @@ -1,29 +0,0 @@ -use super::BeaconChain; -use db::ClientDB; -use state_transition::{extend_active_state, StateTransitionError}; -use types::{ActiveState, BeaconBlock, CrystallizedState, Hash256}; - -impl BeaconChain -where - T: ClientDB + Sized, -{ - pub(crate) fn transition_states( - &self, - act_state: &ActiveState, - cry_state: &CrystallizedState, - block: &BeaconBlock, - block_hash: &Hash256, - ) -> Result<(ActiveState, Option), StateTransitionError> { - let state_recalc_distance = block - .slot - .checked_sub(cry_state.last_state_recalculation_slot) - .ok_or(StateTransitionError::BlockSlotBeforeRecalcSlot)?; - - if state_recalc_distance >= u64::from(self.spec.epoch_length) { - panic!("Not implemented!") - } else { - let new_act_state = extend_active_state(act_state, block, block_hash)?; - Ok((new_act_state, None)) - } - } -} diff --git a/beacon_node/beacon_chain/tests/chain.rs b/beacon_node/beacon_chain/tests/chain.rs index 9bf8959c5..af1310cd6 100644 --- a/beacon_node/beacon_chain/tests/chain.rs +++ b/beacon_node/beacon_chain/tests/chain.rs @@ -1,42 +1,15 @@ use self::utils::TestRig; -use beacon_chain::BeaconChain; -#[cfg(test)] -use block_producer::{test_utils::TestSigner, BlockProducer}; -use db::{ - stores::{BeaconBlockStore, BeaconStateStore}, - MemoryDB, -}; -use slot_clock::TestingSlotClock; -use std::sync::{Arc, RwLock}; -use types::{ChainSpec, Keypair, Validator}; +use types::ChainSpec; mod utils; #[test] -fn rig_can_generate_validators() { - /* - let (_db, mut chain) = in_memory_test_chain(ChainSpec::foundation()); - let validators = generate_validators(2, &chain); - chain.spec = inject_validators_into_spec(chain.spec.clone(), &validators[..]); - */ +fn it_can_produce_blocks() { let validator_count = 2; + let blocks = 3; + let mut rig = TestRig::new(ChainSpec::foundation(), validator_count); - rig.produce_next_slot(); + for _ in 0..blocks { + rig.produce_next_slot(); + } } - -/* -#[test] -fn it_produces() { - let (_db, mut chain) = in_memory_test_chain(ChainSpec::foundation()); - let (_block, _state) = chain.produce_block().unwrap(); -} - -#[test] -fn it_processes_a_block_it_produces() { - let (_db, mut chain) = in_memory_test_chain(ChainSpec::foundation()); - let (block, _state) = chain.produce_block().unwrap(); - let (outcome, new_block_hash) = chain.process_block(block).unwrap(); - assert_eq!(outcome, BlockProcessingOutcome::Processed); - assert_eq!(chain.canonical_leaf_block, new_block_hash); -} -*/ diff --git a/eth2/block_producer/src/lib.rs b/eth2/block_producer/src/lib.rs index 8ce6da859..d1b1f4e7c 100644 --- a/eth2/block_producer/src/lib.rs +++ b/eth2/block_producer/src/lib.rs @@ -134,14 +134,23 @@ impl BlockProducer Result { let randao_reveal = { + /* + * TODO: + * https://github.com/ethereum/eth2.0-specs/pull/496 + * let producer_nonce = self.beacon_node.proposer_nonce(&self.pubkey)?; // TODO: add domain, etc to this message. let message = ssz_encode(&producer_nonce); + */ + // TODO: add domain, etc to this message. + let message = ssz_encode(&slot); + println!("validator randao: {:?}", &message); match self.signer.bls_sign(&message) { None => return Ok(PollOutcome::SignerRejection(slot)), Some(signature) => signature, } }; + println!("validator pubkey: {:?}", &self.pubkey); if let Some(block) = self .beacon_node @@ -169,21 +178,7 @@ impl BlockProducer Option { self.store_produce(&block); - let proposal_root = { - let block_without_signature_root = { - let mut block_without_signature = block.clone(); - block_without_signature.signature = self.spec.empty_signature.clone(); - block_without_signature.canonical_root() - }; - let proposal = ProposalSignedData { - slot: block.slot, - shard: self.spec.beacon_chain_shard_number, - block_root: block_without_signature_root, - }; - hash_tree_root(&proposal) - }; - - match self.signer.bls_sign(&proposal_root[..]) { + match self.signer.bls_sign(&block.proposal_root(&self.spec)[..]) { None => None, Some(signature) => { block.signature = signature; @@ -214,11 +209,6 @@ impl BlockProducer(_input: &T) -> Hash256 { - // TODO: stubbed out. - Hash256::zero() -} - impl From for Error { fn from(e: BeaconNodeError) -> Error { Error::BeaconNodeError(e) diff --git a/eth2/types/src/beacon_block/signing.rs b/eth2/types/src/beacon_block/signing.rs new file mode 100644 index 000000000..c3470217f --- /dev/null +++ b/eth2/types/src/beacon_block/signing.rs @@ -0,0 +1,19 @@ +use crate::{BeaconBlock, ChainSpec, Hash256, ProposalSignedData}; +use hashing::hash_tree_root; + +impl BeaconBlock { + pub fn proposal_root(&self, spec: &ChainSpec) -> Hash256 { + let block_without_signature_root = { + let mut block_without_signature = self.clone(); + block_without_signature.signature = spec.empty_signature.clone(); + block_without_signature.canonical_root() + }; + + let proposal = ProposalSignedData { + slot: self.slot, + shard: spec.beacon_chain_shard_number, + block_root: block_without_signature_root, + }; + Hash256::from_slice(&hash_tree_root(&proposal)[..]) + } +} diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state/mod.rs similarity index 98% rename from eth2/types/src/beacon_state.rs rename to eth2/types/src/beacon_state/mod.rs index 921d49360..57fff2c53 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state/mod.rs @@ -10,6 +10,10 @@ use hashing::canonical_hash; use rand::RngCore; use ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream}; +mod slot_advance; + +pub use self::slot_advance::Error as SlotProcessingError; + // Custody will not be added to the specs until Phase 1 (Sharding Phase) so dummy class used. type CustodyChallenge = usize; @@ -203,9 +207,9 @@ impl TestRandom for BeaconState { #[cfg(test)] mod tests { - use super::ssz::ssz_encode; use super::*; use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; #[test] pub fn test_ssz_round_trip() { diff --git a/eth2/types/src/beacon_state/slot_advance.rs b/eth2/types/src/beacon_state/slot_advance.rs new file mode 100644 index 000000000..e3be9ccee --- /dev/null +++ b/eth2/types/src/beacon_state/slot_advance.rs @@ -0,0 +1,45 @@ +use crate::{BeaconState, ChainSpec, Hash256}; + +pub enum Error { + UnableToDetermineProducer, +} + +impl BeaconState { + pub fn per_slot_processing( + &mut self, + previous_block_root: Hash256, + spec: &ChainSpec, + ) -> Result<(), Error> { + self.slot += 1; + + let block_proposer = self + .get_beacon_proposer_index(self.slot, spec) + .ok_or_else(|| Error::UnableToDetermineProducer)?; + + self.validator_registry[block_proposer].proposer_slots += 1; + self.latest_randao_mixes[(self.slot % spec.latest_randao_mixes_length) as usize] = + self.latest_randao_mixes[((self.slot - 1) % spec.latest_randao_mixes_length) as usize]; + + // Block roots. + self.latest_block_roots[((self.slot - 1) % spec.latest_block_roots_length) as usize] = + previous_block_root; + + if self.slot % spec.latest_block_roots_length == 0 { + let root = merkle_root(&self.latest_block_roots[..]); + self.batched_block_roots.push(root); + } + Ok(()) + } + + pub fn get_beacon_proposer_index(&self, slot: u64, spec: &ChainSpec) -> Option { + // TODO: this is a stub; implement it properly. + // + // https://github.com/sigp/lighthouse/pull/148/files + let validator_count = self.validator_registry.len(); + Some((slot as usize) % validator_count) + } +} + +fn merkle_root(_input: &[Hash256]) -> Hash256 { + Hash256::zero() +} diff --git a/eth2/utils/hashing/Cargo.toml b/eth2/utils/hashing/Cargo.toml index 1527bceba..6abd9cb0c 100644 --- a/eth2/utils/hashing/Cargo.toml +++ b/eth2/utils/hashing/Cargo.toml @@ -6,3 +6,4 @@ edition = "2018" [dependencies] tiny-keccak = "1.4.2" +ssz = { path = "../ssz" } diff --git a/eth2/utils/hashing/src/lib.rs b/eth2/utils/hashing/src/lib.rs index 02203dc16..3d388fd92 100644 --- a/eth2/utils/hashing/src/lib.rs +++ b/eth2/utils/hashing/src/lib.rs @@ -1,5 +1,4 @@ -extern crate tiny_keccak; - +use ssz::{ssz_encode, Encodable as SszEncodable}; use tiny_keccak::Keccak; pub fn canonical_hash(input: &[u8]) -> Vec { @@ -10,6 +9,10 @@ pub fn canonical_hash(input: &[u8]) -> Vec { result } +pub fn hash_tree_root(input: &T) -> Vec { + canonical_hash(&ssz_encode(input)) +} + #[cfg(test)] mod tests { use super::*; From 78c00d336105f028879e916d3b4439b12e6fad08 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 25 Jan 2019 16:54:19 +1100 Subject: [PATCH 026/107] Move back to using `proposer_slots` --- beacon_node/beacon_chain/src/state_transition.rs | 6 +----- eth2/block_producer/src/lib.rs | 11 ++--------- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/beacon_node/beacon_chain/src/state_transition.rs b/beacon_node/beacon_chain/src/state_transition.rs index fe2b0e7b1..1c4ca8370 100644 --- a/beacon_node/beacon_chain/src/state_transition.rs +++ b/beacon_node/beacon_chain/src/state_transition.rs @@ -117,14 +117,10 @@ where * RANDAO */ - println!("proposer pubkey: {:?}", &block_proposer.pubkey); ensure!( bls_verify( &block_proposer.pubkey, - // TODO: https://github.com/ethereum/eth2.0-specs/pull/496 - // - // &ssz_encode(&block_proposer.proposer_slots), - &ssz_encode(&block.slot), + &ssz_encode(&block_proposer.proposer_slots), &block.randao_reveal, get_domain(&state.fork_data, state.slot, DOMAIN_RANDAO) ), diff --git a/eth2/block_producer/src/lib.rs b/eth2/block_producer/src/lib.rs index d1b1f4e7c..bb62be960 100644 --- a/eth2/block_producer/src/lib.rs +++ b/eth2/block_producer/src/lib.rs @@ -134,23 +134,16 @@ impl BlockProducer Result { let randao_reveal = { - /* - * TODO: - * https://github.com/ethereum/eth2.0-specs/pull/496 - * let producer_nonce = self.beacon_node.proposer_nonce(&self.pubkey)?; + // TODO: add domain, etc to this message. let message = ssz_encode(&producer_nonce); - */ - // TODO: add domain, etc to this message. - let message = ssz_encode(&slot); - println!("validator randao: {:?}", &message); + match self.signer.bls_sign(&message) { None => return Ok(PollOutcome::SignerRejection(slot)), Some(signature) => signature, } }; - println!("validator pubkey: {:?}", &self.pubkey); if let Some(block) = self .beacon_node From ad62e1e35c73e8f4caa37279a7befca1f935394c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 25 Jan 2019 17:23:54 +1100 Subject: [PATCH 027/107] Rust cargo fmt --- beacon_node/beacon_chain/src/block_processing.rs | 2 +- eth2/block_producer/src/lib.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/beacon_node/beacon_chain/src/block_processing.rs b/beacon_node/beacon_chain/src/block_processing.rs index 32670f51d..41e1fb3d3 100644 --- a/beacon_node/beacon_chain/src/block_processing.rs +++ b/beacon_node/beacon_chain/src/block_processing.rs @@ -101,7 +101,7 @@ where Err(error) => { return Ok(Outcome::InvalidBlock(InvalidBlock::StateTransitionFailed( error, - ))) + ))); } }; diff --git a/eth2/block_producer/src/lib.rs b/eth2/block_producer/src/lib.rs index bb62be960..4ea08647d 100644 --- a/eth2/block_producer/src/lib.rs +++ b/eth2/block_producer/src/lib.rs @@ -94,10 +94,10 @@ impl BlockProducer result, Err(DutiesReaderError::UnknownEpoch) => { - return Ok(PollOutcome::ProducerDutiesUnknown(slot)) + return Ok(PollOutcome::ProducerDutiesUnknown(slot)); } Err(DutiesReaderError::UnknownValidator) => { - return Ok(PollOutcome::ValidatorIsUnknown(slot)) + return Ok(PollOutcome::ValidatorIsUnknown(slot)); } Err(DutiesReaderError::EpochLengthIsZero) => return Err(Error::EpochLengthIsZero), Err(DutiesReaderError::Poisoned) => return Err(Error::EpochMapPoisoned), From 73d86bcc3b2a25000b949f12cf1aa336a3536d05 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 25 Jan 2019 17:39:46 +1100 Subject: [PATCH 028/107] Fix canonical_root fns Fixes #92 --- eth2/types/src/beacon_block/mod.rs | 4 +--- eth2/types/src/beacon_state/mod.rs | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/eth2/types/src/beacon_block/mod.rs b/eth2/types/src/beacon_block/mod.rs index 9b58b27ee..c736c0735 100644 --- a/eth2/types/src/beacon_block/mod.rs +++ b/eth2/types/src/beacon_block/mod.rs @@ -20,9 +20,7 @@ pub struct BeaconBlock { impl BeaconBlock { pub fn canonical_root(&self) -> Hash256 { - // TODO: implement tree hashing. - // https://github.com/sigp/lighthouse/issues/70 - Hash256::from(&canonical_hash(&ssz_encode(self))[..]) + Hash256::from(&self.hash_tree_root()[..]) } } diff --git a/eth2/types/src/beacon_state/mod.rs b/eth2/types/src/beacon_state/mod.rs index 32cc7d829..f8f458ae1 100644 --- a/eth2/types/src/beacon_state/mod.rs +++ b/eth2/types/src/beacon_state/mod.rs @@ -64,9 +64,7 @@ pub struct BeaconState { impl BeaconState { pub fn canonical_root(&self) -> Hash256 { - // TODO: implement tree hashing. - // https://github.com/sigp/lighthouse/issues/70 - Hash256::from(&canonical_hash(&ssz_encode(self))[..]) + Hash256::from(&self.hash_tree_root()[..]) } } From f9acc42acaed50f30a32689b9f6cb30972f06da7 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 25 Jan 2019 17:40:22 +1100 Subject: [PATCH 029/107] Rename `canonical_hash` to `hash --- beacon_node/beacon_chain/Cargo.toml | 1 + beacon_node/beacon_chain/src/state_transition.rs | 9 +++------ eth2/types/src/beacon_block/mod.rs | 3 +-- eth2/types/src/beacon_state/mod.rs | 3 +-- eth2/utils/bls/src/lib.rs | 6 +++--- eth2/utils/hashing/src/lib.rs | 4 ++-- eth2/utils/ssz/src/impl_tree_hash.rs | 3 ++- eth2/utils/ssz/src/lib.rs | 4 +++- eth2/utils/ssz/src/tree_hash.rs | 6 +----- eth2/utils/vec_shuffle/src/lib.rs | 4 ++-- eth2/utils/vec_shuffle/src/rng.rs | 6 +++--- 11 files changed, 22 insertions(+), 27 deletions(-) diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index 4fba69546..54902ddd6 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -12,6 +12,7 @@ db = { path = "../db" } failure = "0.1" failure_derive = "0.1" genesis = { path = "../../eth2/genesis" } +hashing = { path = "../../eth2/utils/hashing" } serde_derive = "1.0" slot_clock = { path = "../../eth2/utils/slot_clock" } ssz = { path = "../../eth2/utils/ssz" } diff --git a/beacon_node/beacon_chain/src/state_transition.rs b/beacon_node/beacon_chain/src/state_transition.rs index 1c4ca8370..f6555942c 100644 --- a/beacon_node/beacon_chain/src/state_transition.rs +++ b/beacon_node/beacon_chain/src/state_transition.rs @@ -1,6 +1,7 @@ use super::{BeaconChain, ClientDB, DBError, SlotClock}; use bls::{AggregatePublicKey, AggregateSignature, PublicKey, Signature}; use boolean_bitfield::BooleanBitfield; +use hashing::hash; use slot_clock::{SystemTimeSlotClockError, TestingSlotClockError}; use ssz::ssz_encode; use types::{ @@ -127,12 +128,13 @@ where Error::BadRandaoSignature ); + // TODO: check this is correct. let new_mix = { let mut mix = state.latest_randao_mixes [(state.slot % self.spec.latest_randao_mixes_length) as usize] .to_vec(); mix.append(&mut ssz_encode(&block.randao_reveal)); - hash(&mix) + Hash256::from(&hash(&mix)[..]) }; state.latest_randao_mixes[(state.slot % self.spec.latest_randao_mixes_length) as usize] = @@ -396,11 +398,6 @@ fn penalize_validator(_state: &BeaconState, _proposer_index: usize) { // TODO: stubbed out. } -fn hash(_input: &T) -> Hash256 { - // TODO: stubbed out. - Hash256::zero() -} - fn get_domain(_fork: &Fork, _slot: u64, _domain_type: u64) -> u64 { // TODO: stubbed out. 0 diff --git a/eth2/types/src/beacon_block/mod.rs b/eth2/types/src/beacon_block/mod.rs index c736c0735..b65f1b053 100644 --- a/eth2/types/src/beacon_block/mod.rs +++ b/eth2/types/src/beacon_block/mod.rs @@ -1,9 +1,8 @@ use super::{BeaconBlockBody, Eth1Data, Hash256}; use crate::test_utils::TestRandom; use bls::Signature; -use hashing::canonical_hash; use rand::RngCore; -use ssz::{hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; mod signing; diff --git a/eth2/types/src/beacon_state/mod.rs b/eth2/types/src/beacon_state/mod.rs index f8f458ae1..bedd12233 100644 --- a/eth2/types/src/beacon_state/mod.rs +++ b/eth2/types/src/beacon_state/mod.rs @@ -6,9 +6,8 @@ use super::pending_attestation::PendingAttestation; use super::validator::Validator; use super::Hash256; use crate::test_utils::TestRandom; -use hashing::canonical_hash; use rand::RngCore; -use ssz::{hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; mod slot_advance; diff --git a/eth2/utils/bls/src/lib.rs b/eth2/utils/bls/src/lib.rs index d7a3ff15d..fe2f3a17e 100644 --- a/eth2/utils/bls/src/lib.rs +++ b/eth2/utils/bls/src/lib.rs @@ -18,7 +18,7 @@ pub use self::bls_aggregates::AggregatePublicKey; pub const BLS_AGG_SIG_BYTE_SIZE: usize = 97; -use hashing::canonical_hash; +use hashing::hash; use ssz::ssz_encode; use std::default::Default; @@ -30,13 +30,13 @@ fn extend_if_needed(hash: &mut Vec) { /// For some signature and public key, ensure that the signature message was the public key and it /// was signed by the secret key that corresponds to that public key. pub fn verify_proof_of_possession(sig: &Signature, pubkey: &PublicKey) -> bool { - let mut hash = canonical_hash(&ssz_encode(pubkey)); + let mut hash = hash(&ssz_encode(pubkey)); extend_if_needed(&mut hash); sig.verify_hashed(&hash, &pubkey) } pub fn create_proof_of_possession(keypair: &Keypair) -> Signature { - let mut hash = canonical_hash(&ssz_encode(&keypair.pk)); + let mut hash = hash(&ssz_encode(&keypair.pk)); extend_if_needed(&mut hash); Signature::new_hashed(&hash, &keypair.sk) } diff --git a/eth2/utils/hashing/src/lib.rs b/eth2/utils/hashing/src/lib.rs index c263c91e0..b2bd5a279 100644 --- a/eth2/utils/hashing/src/lib.rs +++ b/eth2/utils/hashing/src/lib.rs @@ -1,6 +1,6 @@ use tiny_keccak::Keccak; -pub fn canonical_hash(input: &[u8]) -> Vec { +pub fn hash(input: &[u8]) -> Vec { let mut keccak = Keccak::new_keccak256(); keccak.update(input); let mut result = vec![0; 32]; @@ -17,7 +17,7 @@ mod tests { fn test_hashing() { let input: Vec = From::from("hello"); - let output = canonical_hash(input.as_ref()); + let output = hash(input.as_ref()); let expected = &[ 0x1c, 0x8a, 0xff, 0x95, 0x06, 0x85, 0xc2, 0xed, 0x4b, 0xc3, 0x17, 0x4f, 0x34, 0x72, 0x28, 0x7b, 0x56, 0xd9, 0x51, 0x7b, 0x9c, 0x94, 0x81, 0x27, 0x31, 0x9a, 0x09, 0xa7, diff --git a/eth2/utils/ssz/src/impl_tree_hash.rs b/eth2/utils/ssz/src/impl_tree_hash.rs index 9463283cb..578977eec 100644 --- a/eth2/utils/ssz/src/impl_tree_hash.rs +++ b/eth2/utils/ssz/src/impl_tree_hash.rs @@ -1,5 +1,6 @@ use super::ethereum_types::{Address, H256}; -use super::{hash, merkle_hash, ssz_encode, TreeHash}; +use super::{merkle_hash, ssz_encode, TreeHash}; +use hashing::hash; impl TreeHash for u8 { fn hash_tree_root(&self) -> Vec { diff --git a/eth2/utils/ssz/src/lib.rs b/eth2/utils/ssz/src/lib.rs index 206040c2d..a6baa35a7 100644 --- a/eth2/utils/ssz/src/lib.rs +++ b/eth2/utils/ssz/src/lib.rs @@ -20,7 +20,9 @@ mod impl_tree_hash; pub use crate::decode::{decode_ssz, decode_ssz_list, Decodable, DecodeError}; pub use crate::encode::{Encodable, SszStream}; -pub use crate::tree_hash::{hash, merkle_hash, TreeHash}; +pub use crate::tree_hash::{merkle_hash, TreeHash}; + +pub use hashing::hash; pub const LENGTH_BYTES: usize = 4; pub const MAX_LIST_SIZE: usize = 1 << (4 * 8); diff --git a/eth2/utils/ssz/src/tree_hash.rs b/eth2/utils/ssz/src/tree_hash.rs index 497d62629..fe5b1dfc3 100644 --- a/eth2/utils/ssz/src/tree_hash.rs +++ b/eth2/utils/ssz/src/tree_hash.rs @@ -1,4 +1,4 @@ -use hashing::canonical_hash; +use hashing::hash; const SSZ_CHUNK_SIZE: usize = 128; const HASHSIZE: usize = 32; @@ -64,10 +64,6 @@ fn list_to_blob(list: &mut Vec>) -> (usize, Vec) { (chunk_size, data) } -pub fn hash(data: &[u8]) -> Vec { - canonical_hash(data) -} - #[cfg(test)] mod tests { use super::*; diff --git a/eth2/utils/vec_shuffle/src/lib.rs b/eth2/utils/vec_shuffle/src/lib.rs index f5c2b7ebd..78bb8aa10 100644 --- a/eth2/utils/vec_shuffle/src/lib.rs +++ b/eth2/utils/vec_shuffle/src/lib.rs @@ -45,7 +45,7 @@ mod tests { use std::{fs::File, io::prelude::*, path::PathBuf}; - use super::{hashing::canonical_hash, *}; + use super::{hashing::hash, *}; #[test] fn test_shuffling() { @@ -70,7 +70,7 @@ mod tests { let seed_bytes = test_case["seed"].as_str().unwrap().as_bytes(); let seed = if seed_bytes.len() > 0 { - canonical_hash(seed_bytes) + hash(seed_bytes) } else { vec![] }; diff --git a/eth2/utils/vec_shuffle/src/rng.rs b/eth2/utils/vec_shuffle/src/rng.rs index e338647de..7a4a785ff 100644 --- a/eth2/utils/vec_shuffle/src/rng.rs +++ b/eth2/utils/vec_shuffle/src/rng.rs @@ -1,4 +1,4 @@ -use super::hashing::canonical_hash; +use super::hashing::hash; const SEED_SIZE_BYTES: usize = 32; const RAND_BYTES: usize = 3; // 24 / 8 @@ -16,7 +16,7 @@ impl ShuffleRng { /// Create a new instance given some "seed" bytes. pub fn new(initial_seed: &[u8]) -> Self { Self { - seed: canonical_hash(initial_seed), + seed: hash(initial_seed), idx: 0, rand_max: RAND_MAX, } @@ -24,7 +24,7 @@ impl ShuffleRng { /// "Regenerates" the seed by hashing it. fn rehash_seed(&mut self) { - self.seed = canonical_hash(&self.seed); + self.seed = hash(&self.seed); self.idx = 0; } From 4d3889d83876bd7d5c866d996284115748a361b3 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 25 Jan 2019 17:47:33 +1100 Subject: [PATCH 030/107] Remove old hash_tree_root stub from state trans --- beacon_node/beacon_chain/src/state_transition.rs | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/beacon_node/beacon_chain/src/state_transition.rs b/beacon_node/beacon_chain/src/state_transition.rs index f6555942c..3e694d331 100644 --- a/beacon_node/beacon_chain/src/state_transition.rs +++ b/beacon_node/beacon_chain/src/state_transition.rs @@ -3,7 +3,7 @@ use bls::{AggregatePublicKey, AggregateSignature, PublicKey, Signature}; use boolean_bitfield::BooleanBitfield; use hashing::hash; use slot_clock::{SystemTimeSlotClockError, TestingSlotClockError}; -use ssz::ssz_encode; +use ssz::{ssz_encode, TreeHash}; use types::{ beacon_state::SlotProcessingError, readers::BeaconBlockReader, AttestationData, AttestationDataAndCustodyBit, BeaconBlock, BeaconState, Exit, Fork, Hash256, @@ -183,7 +183,7 @@ where ensure!( bls_verify( &proposer.pubkey, - &hash_tree_root(&proposer_slashing.proposal_data_1), + &proposer_slashing.proposal_data_1.hash_tree_root(), &proposer_slashing.proposal_signature_1, get_domain( &state.fork_data, @@ -196,7 +196,7 @@ where ensure!( bls_verify( &proposer.pubkey, - &hash_tree_root(&proposer_slashing.proposal_data_2), + &proposer_slashing.proposal_data_2.hash_tree_root(), &proposer_slashing.proposal_signature_2, get_domain( &state.fork_data, @@ -273,7 +273,7 @@ where data: attestation.data.clone(), custody_bit: false, }; - hash_tree_root(&attestation_data_and_custody_bit).to_vec() + &attestation_data_and_custody_bit.hash_tree_root() }; // Signature verification. ensure!( @@ -333,7 +333,7 @@ where validator_index: exit.validator_index, signature: self.spec.empty_signature.clone(), }; - hash_tree_root(&exit_struct) + exit_struct.hash_tree_root() }; ensure!( bls_verify( @@ -418,11 +418,6 @@ fn bls_verify_aggregate( signature.verify(message, pubkey) } -fn hash_tree_root(_input: &T) -> Hash256 { - // TODO: stubbed out. - Hash256::zero() -} - impl From for Error { fn from(e: DBError) -> Error { Error::DBError(e.message) From dbd5e850fe3a729c012078c45b65338ec3cd76c9 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 26 Jan 2019 07:20:58 +1100 Subject: [PATCH 031/107] Add `chain_dump` fn to beacon_chain --- beacon_node/beacon_chain/src/dump.rs | 72 +++++++++++++++++++ beacon_node/beacon_chain/src/lib.rs | 1 + beacon_node/beacon_chain/tests/chain.rs | 3 + .../beacon_chain/tests/utils/test_rig.rs | 5 ++ 4 files changed, 81 insertions(+) create mode 100644 beacon_node/beacon_chain/src/dump.rs diff --git a/beacon_node/beacon_chain/src/dump.rs b/beacon_node/beacon_chain/src/dump.rs new file mode 100644 index 000000000..e4cb0844a --- /dev/null +++ b/beacon_node/beacon_chain/src/dump.rs @@ -0,0 +1,72 @@ +use super::{BeaconChain, ClientDB, DBError, SlotClock}; +use types::{BeaconBlock, BeaconState, Hash256}; + +#[derive(Debug, Clone)] +pub struct SlotDump { + pub beacon_block: BeaconBlock, + pub beacon_block_root: Hash256, + pub beacon_state: BeaconState, + pub beacon_state_root: Hash256, +} + +#[derive(Debug, Clone)] +pub enum Error { + DBError(String), + MissingBlock(Hash256), +} + +impl BeaconChain +where + T: ClientDB, + U: SlotClock, +{ + pub fn chain_dump(&self) -> Result, Error> { + let mut dump = vec![]; + + let mut last_slot = SlotDump { + beacon_block: self.canonical_head().beacon_block.clone(), + beacon_block_root: self.canonical_head().beacon_block_root, + beacon_state: self.canonical_head().beacon_state.clone(), + beacon_state_root: self.canonical_head().beacon_state_root, + }; + + dump.push(last_slot.clone()); + + loop { + let beacon_block_root = last_slot.beacon_block.parent_root; + + if beacon_block_root == self.spec.zero_hash { + // Genesis has been reached. + break; + } + + let beacon_block = self + .block_store + .get_deserialized(&beacon_block_root)? + .ok_or_else(|| Error::MissingBlock(beacon_block_root))?; + let beacon_state_root = beacon_block.state_root; + let beacon_state = self + .state_store + .get_deserialized(&beacon_state_root)? + .ok_or_else(|| Error::MissingBlock(beacon_state_root))?; + + let slot = SlotDump { + beacon_block, + beacon_block_root, + beacon_state, + beacon_state_root, + }; + + dump.push(slot.clone()); + last_slot = slot; + } + + Ok(dump) + } +} + +impl From for Error { + fn from(e: DBError) -> Error { + Error::DBError(e.message) + } +} diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index e820c4b20..d6058eab7 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -3,6 +3,7 @@ mod block_graph; pub mod block_processing; pub mod block_production; mod canonical_head; +pub mod dump; mod finalized_head; mod info; mod lmd_ghost; diff --git a/beacon_node/beacon_chain/tests/chain.rs b/beacon_node/beacon_chain/tests/chain.rs index af1310cd6..c4d806265 100644 --- a/beacon_node/beacon_chain/tests/chain.rs +++ b/beacon_node/beacon_chain/tests/chain.rs @@ -12,4 +12,7 @@ fn it_can_produce_blocks() { for _ in 0..blocks { rig.produce_next_slot(); } + let dump = rig.chain_dump().expect("Chain dump failed."); + + assert_eq!(dump.len(), blocks + 1); // + 1 for genesis block. } diff --git a/beacon_node/beacon_chain/tests/utils/test_rig.rs b/beacon_node/beacon_chain/tests/utils/test_rig.rs index b6aef3290..a3a641715 100644 --- a/beacon_node/beacon_chain/tests/utils/test_rig.rs +++ b/beacon_node/beacon_chain/tests/utils/test_rig.rs @@ -1,4 +1,5 @@ use super::TestValidator; +pub use beacon_chain::dump::{Error as DumpError, SlotDump}; use beacon_chain::BeaconChain; #[cfg(test)] use db::{ @@ -82,4 +83,8 @@ impl TestRig { self.validators[proposer].set_slot(slot); self.validators[proposer].produce_block().unwrap(); } + + pub fn chain_dump(&self) -> Result, DumpError> { + self.beacon_chain.chain_dump() + } } From 90ae2298ab48b4e411a9f466f3cb71950fbd7b0d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 26 Jan 2019 08:25:31 +1100 Subject: [PATCH 032/107] Impl serde::Serialize for all types --- eth2/types/Cargo.toml | 3 +++ eth2/types/src/attestation.rs | 3 ++- eth2/types/src/attestation_data.rs | 3 ++- eth2/types/src/attestation_data_and_custody_bit.rs | 3 ++- eth2/types/src/beacon_block/mod.rs | 3 ++- eth2/types/src/beacon_block_body.rs | 3 ++- eth2/types/src/beacon_state/mod.rs | 3 ++- eth2/types/src/casper_slashing.rs | 3 ++- eth2/types/src/crosslink.rs | 3 ++- eth2/types/src/deposit.rs | 3 ++- eth2/types/src/deposit_data.rs | 3 ++- eth2/types/src/deposit_input.rs | 3 ++- eth2/types/src/eth1_data.rs | 3 ++- eth2/types/src/eth1_data_vote.rs | 3 ++- eth2/types/src/exit.rs | 3 ++- eth2/types/src/fork.rs | 3 ++- eth2/types/src/pending_attestation.rs | 3 ++- eth2/types/src/proposal_signed_data.rs | 3 ++- eth2/types/src/proposer_slashing.rs | 3 ++- eth2/types/src/shard_committee.rs | 3 ++- eth2/types/src/shard_reassignment_record.rs | 3 ++- eth2/types/src/slashable_vote_data.rs | 3 ++- eth2/types/src/special_record.rs | 3 ++- eth2/types/src/validator.rs | 5 +++-- eth2/types/src/validator_registry_delta_block.rs | 3 ++- eth2/utils/bls/Cargo.toml | 1 + eth2/utils/bls/src/aggregate_signature.rs | 14 +++++++++++++- eth2/utils/bls/src/public_key.rs | 10 ++++++++++ eth2/utils/bls/src/signature.rs | 14 +++++++++++++- eth2/utils/boolean-bitfield/Cargo.toml | 2 ++ eth2/utils/boolean-bitfield/src/lib.rs | 10 ++++++++++ 31 files changed, 101 insertions(+), 27 deletions(-) diff --git a/eth2/types/Cargo.toml b/eth2/types/Cargo.toml index af53fa597..b3c3f7462 100644 --- a/eth2/types/Cargo.toml +++ b/eth2/types/Cargo.toml @@ -10,4 +10,7 @@ boolean-bitfield = { path = "../utils/boolean-bitfield" } ethereum-types = "0.4.0" hashing = { path = "../utils/hashing" } rand = "0.5.5" +serde = "1.0" +serde_derive = "1.0" +serde_json = "1.0" ssz = { path = "../utils/ssz" } diff --git a/eth2/types/src/attestation.rs b/eth2/types/src/attestation.rs index 9720472b8..d6738bd6e 100644 --- a/eth2/types/src/attestation.rs +++ b/eth2/types/src/attestation.rs @@ -2,9 +2,10 @@ use super::{AttestationData, Bitfield}; use crate::test_utils::TestRandom; use bls::AggregateSignature; use rand::RngCore; +use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize)] pub struct Attestation { pub data: AttestationData, pub aggregation_bitfield: Bitfield, diff --git a/eth2/types/src/attestation_data.rs b/eth2/types/src/attestation_data.rs index 5639099f0..aab5f7563 100644 --- a/eth2/types/src/attestation_data.rs +++ b/eth2/types/src/attestation_data.rs @@ -1,6 +1,7 @@ use super::Hash256; use crate::test_utils::TestRandom; use rand::RngCore; +use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; pub const SSZ_ATTESTION_DATA_LENGTH: usize = { @@ -14,7 +15,7 @@ pub const SSZ_ATTESTION_DATA_LENGTH: usize = { 32 // justified_block_root }; -#[derive(Debug, Clone, PartialEq, Default)] +#[derive(Debug, Clone, PartialEq, Default, Serialize)] pub struct AttestationData { pub slot: u64, pub shard: u64, diff --git a/eth2/types/src/attestation_data_and_custody_bit.rs b/eth2/types/src/attestation_data_and_custody_bit.rs index 1b3b2ea3b..8200abf30 100644 --- a/eth2/types/src/attestation_data_and_custody_bit.rs +++ b/eth2/types/src/attestation_data_and_custody_bit.rs @@ -1,9 +1,10 @@ use super::AttestationData; use crate::test_utils::TestRandom; use rand::RngCore; +use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; -#[derive(Debug, Clone, PartialEq, Default)] +#[derive(Debug, Clone, PartialEq, Default, Serialize)] pub struct AttestationDataAndCustodyBit { pub data: AttestationData, pub custody_bit: bool, diff --git a/eth2/types/src/beacon_block/mod.rs b/eth2/types/src/beacon_block/mod.rs index b65f1b053..3e07572d1 100644 --- a/eth2/types/src/beacon_block/mod.rs +++ b/eth2/types/src/beacon_block/mod.rs @@ -2,11 +2,12 @@ use super::{BeaconBlockBody, Eth1Data, Hash256}; use crate::test_utils::TestRandom; use bls::Signature; use rand::RngCore; +use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; mod signing; -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, PartialEq, Clone, Serialize)] pub struct BeaconBlock { pub slot: u64, pub parent_root: Hash256, diff --git a/eth2/types/src/beacon_block_body.rs b/eth2/types/src/beacon_block_body.rs index 8084c38d9..ad9ec7ea6 100644 --- a/eth2/types/src/beacon_block_body.rs +++ b/eth2/types/src/beacon_block_body.rs @@ -1,6 +1,7 @@ use super::{Attestation, CasperSlashing, Deposit, Exit, ProposerSlashing}; use crate::test_utils::TestRandom; use rand::RngCore; +use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; // The following types are just dummy classes as they will not be defined until @@ -9,7 +10,7 @@ type CustodyReseed = usize; type CustodyChallenge = usize; type CustodyResponse = usize; -#[derive(Debug, PartialEq, Clone, Default)] +#[derive(Debug, PartialEq, Clone, Default, Serialize)] pub struct BeaconBlockBody { pub proposer_slashings: Vec, pub casper_slashings: Vec, diff --git a/eth2/types/src/beacon_state/mod.rs b/eth2/types/src/beacon_state/mod.rs index bedd12233..71c4d1e87 100644 --- a/eth2/types/src/beacon_state/mod.rs +++ b/eth2/types/src/beacon_state/mod.rs @@ -7,6 +7,7 @@ use super::validator::Validator; use super::Hash256; use crate::test_utils::TestRandom; use rand::RngCore; +use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; mod slot_advance; @@ -16,7 +17,7 @@ pub use self::slot_advance::Error as SlotProcessingError; // Custody will not be added to the specs until Phase 1 (Sharding Phase) so dummy class used. type CustodyChallenge = usize; -#[derive(Debug, PartialEq, Clone, Default)] +#[derive(Debug, PartialEq, Clone, Default, Serialize)] pub struct BeaconState { // Misc pub slot: u64, diff --git a/eth2/types/src/casper_slashing.rs b/eth2/types/src/casper_slashing.rs index 6b10988d6..0eab069b4 100644 --- a/eth2/types/src/casper_slashing.rs +++ b/eth2/types/src/casper_slashing.rs @@ -1,9 +1,10 @@ use super::SlashableVoteData; use crate::test_utils::TestRandom; use rand::RngCore; +use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, PartialEq, Clone, Serialize)] pub struct CasperSlashing { pub slashable_vote_data_1: SlashableVoteData, pub slashable_vote_data_2: SlashableVoteData, diff --git a/eth2/types/src/crosslink.rs b/eth2/types/src/crosslink.rs index 5fbaa3342..f727d7b43 100644 --- a/eth2/types/src/crosslink.rs +++ b/eth2/types/src/crosslink.rs @@ -1,9 +1,10 @@ use super::Hash256; use crate::test_utils::TestRandom; use rand::RngCore; +use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, Serialize)] pub struct Crosslink { pub slot: u64, pub shard_block_root: Hash256, diff --git a/eth2/types/src/deposit.rs b/eth2/types/src/deposit.rs index cc0c3c524..85b002101 100644 --- a/eth2/types/src/deposit.rs +++ b/eth2/types/src/deposit.rs @@ -1,9 +1,10 @@ use super::{DepositData, Hash256}; use crate::test_utils::TestRandom; use rand::RngCore; +use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, PartialEq, Clone, Serialize)] pub struct Deposit { pub merkle_branch: Vec, pub merkle_tree_index: u64, diff --git a/eth2/types/src/deposit_data.rs b/eth2/types/src/deposit_data.rs index bdc068db3..5c8c302f4 100644 --- a/eth2/types/src/deposit_data.rs +++ b/eth2/types/src/deposit_data.rs @@ -1,9 +1,10 @@ use super::DepositInput; use crate::test_utils::TestRandom; use rand::RngCore; +use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, PartialEq, Clone, Serialize)] pub struct DepositData { pub amount: u64, pub timestamp: u64, diff --git a/eth2/types/src/deposit_input.rs b/eth2/types/src/deposit_input.rs index bb7bdf5eb..fc53baae9 100644 --- a/eth2/types/src/deposit_input.rs +++ b/eth2/types/src/deposit_input.rs @@ -2,9 +2,10 @@ use super::Hash256; use crate::test_utils::TestRandom; use bls::{PublicKey, Signature}; use rand::RngCore; +use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, PartialEq, Clone, Serialize)] pub struct DepositInput { pub pubkey: PublicKey, pub withdrawal_credentials: Hash256, diff --git a/eth2/types/src/eth1_data.rs b/eth2/types/src/eth1_data.rs index e8d30109e..6e9bb7d26 100644 --- a/eth2/types/src/eth1_data.rs +++ b/eth2/types/src/eth1_data.rs @@ -1,10 +1,11 @@ use super::Hash256; use crate::test_utils::TestRandom; use rand::RngCore; +use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; // Note: this is refer to as DepositRootVote in specs -#[derive(Debug, PartialEq, Clone, Default)] +#[derive(Debug, PartialEq, Clone, Default, Serialize)] pub struct Eth1Data { pub deposit_root: Hash256, pub block_hash: Hash256, diff --git a/eth2/types/src/eth1_data_vote.rs b/eth2/types/src/eth1_data_vote.rs index b6917d516..2bfee4d02 100644 --- a/eth2/types/src/eth1_data_vote.rs +++ b/eth2/types/src/eth1_data_vote.rs @@ -1,10 +1,11 @@ use super::Eth1Data; use crate::test_utils::TestRandom; use rand::RngCore; +use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; // Note: this is refer to as DepositRootVote in specs -#[derive(Debug, PartialEq, Clone, Default)] +#[derive(Debug, PartialEq, Clone, Default, Serialize)] pub struct Eth1DataVote { pub eth1_data: Eth1Data, pub vote_count: u64, diff --git a/eth2/types/src/exit.rs b/eth2/types/src/exit.rs index 284a180e7..97f1fd286 100644 --- a/eth2/types/src/exit.rs +++ b/eth2/types/src/exit.rs @@ -1,9 +1,10 @@ use crate::test_utils::TestRandom; use bls::Signature; use rand::RngCore; +use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, PartialEq, Clone, Serialize)] pub struct Exit { pub slot: u64, pub validator_index: u32, diff --git a/eth2/types/src/fork.rs b/eth2/types/src/fork.rs index 4c6a7c6a4..c5a06caea 100644 --- a/eth2/types/src/fork.rs +++ b/eth2/types/src/fork.rs @@ -1,8 +1,9 @@ use crate::test_utils::TestRandom; use rand::RngCore; +use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; -#[derive(Debug, Clone, PartialEq, Default)] +#[derive(Debug, Clone, PartialEq, Default, Serialize)] pub struct Fork { pub pre_fork_version: u64, pub post_fork_version: u64, diff --git a/eth2/types/src/pending_attestation.rs b/eth2/types/src/pending_attestation.rs index 13990a014..d2af52826 100644 --- a/eth2/types/src/pending_attestation.rs +++ b/eth2/types/src/pending_attestation.rs @@ -1,9 +1,10 @@ use super::{AttestationData, Bitfield}; use crate::test_utils::TestRandom; use rand::RngCore; +use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize)] pub struct PendingAttestation { pub data: AttestationData, pub aggregation_bitfield: Bitfield, diff --git a/eth2/types/src/proposal_signed_data.rs b/eth2/types/src/proposal_signed_data.rs index 1a433fd5c..829a16987 100644 --- a/eth2/types/src/proposal_signed_data.rs +++ b/eth2/types/src/proposal_signed_data.rs @@ -1,9 +1,10 @@ use super::Hash256; use crate::test_utils::TestRandom; use rand::RngCore; +use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; -#[derive(Debug, PartialEq, Clone, Default)] +#[derive(Debug, PartialEq, Clone, Default, Serialize)] pub struct ProposalSignedData { pub slot: u64, pub shard: u64, diff --git a/eth2/types/src/proposer_slashing.rs b/eth2/types/src/proposer_slashing.rs index 08a165665..a82a37074 100644 --- a/eth2/types/src/proposer_slashing.rs +++ b/eth2/types/src/proposer_slashing.rs @@ -2,9 +2,10 @@ use super::ProposalSignedData; use crate::test_utils::TestRandom; use bls::Signature; use rand::RngCore; +use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, PartialEq, Clone, Serialize)] pub struct ProposerSlashing { pub proposer_index: u32, pub proposal_data_1: ProposalSignedData, diff --git a/eth2/types/src/shard_committee.rs b/eth2/types/src/shard_committee.rs index 943149230..3632cb0c1 100644 --- a/eth2/types/src/shard_committee.rs +++ b/eth2/types/src/shard_committee.rs @@ -1,8 +1,9 @@ use crate::test_utils::TestRandom; use rand::RngCore; +use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, Serialize)] pub struct ShardCommittee { pub shard: u64, pub committee: Vec, diff --git a/eth2/types/src/shard_reassignment_record.rs b/eth2/types/src/shard_reassignment_record.rs index 5528a92f1..a239233df 100644 --- a/eth2/types/src/shard_reassignment_record.rs +++ b/eth2/types/src/shard_reassignment_record.rs @@ -1,8 +1,9 @@ use crate::test_utils::TestRandom; use rand::RngCore; +use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, PartialEq, Clone, Serialize)] pub struct ShardReassignmentRecord { pub validator_index: u64, pub shard: u64, diff --git a/eth2/types/src/slashable_vote_data.rs b/eth2/types/src/slashable_vote_data.rs index d153da2b9..acffca26d 100644 --- a/eth2/types/src/slashable_vote_data.rs +++ b/eth2/types/src/slashable_vote_data.rs @@ -2,9 +2,10 @@ use super::AttestationData; use crate::test_utils::TestRandom; use bls::AggregateSignature; use rand::RngCore; +use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, PartialEq, Clone, Serialize)] pub struct SlashableVoteData { pub custody_bit_0_indices: Vec, pub custody_bit_1_indices: Vec, diff --git a/eth2/types/src/special_record.rs b/eth2/types/src/special_record.rs index 21e9c88f6..2ab6f2b5b 100644 --- a/eth2/types/src/special_record.rs +++ b/eth2/types/src/special_record.rs @@ -1,9 +1,10 @@ +use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; /// The value of the "type" field of SpecialRecord. /// /// Note: this value must serialize to a u8 and therefore must not be greater than 255. -#[derive(Debug, PartialEq, Clone, Copy)] +#[derive(Debug, PartialEq, Clone, Copy, Serialize)] pub enum SpecialRecordKind { Logout = 0, CasperSlashing = 1, diff --git a/eth2/types/src/validator.rs b/eth2/types/src/validator.rs index eaf3308b9..708300d9d 100644 --- a/eth2/types/src/validator.rs +++ b/eth2/types/src/validator.rs @@ -2,12 +2,13 @@ use super::Hash256; use crate::test_utils::TestRandom; use bls::PublicKey; use rand::RngCore; +use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; const STATUS_FLAG_INITIATED_EXIT: u8 = 1; const STATUS_FLAG_WITHDRAWABLE: u8 = 2; -#[derive(Debug, PartialEq, Clone, Copy)] +#[derive(Debug, PartialEq, Clone, Copy, Serialize)] pub enum StatusFlags { InitiatedExit, Withdrawable, @@ -43,7 +44,7 @@ fn status_flag_from_byte(flag: u8) -> Result, StatusFlagsDec } } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize)] pub struct Validator { pub pubkey: PublicKey, pub withdrawal_credentials: Hash256, diff --git a/eth2/types/src/validator_registry_delta_block.rs b/eth2/types/src/validator_registry_delta_block.rs index f673f2a51..196da754d 100644 --- a/eth2/types/src/validator_registry_delta_block.rs +++ b/eth2/types/src/validator_registry_delta_block.rs @@ -2,10 +2,11 @@ use super::Hash256; use crate::test_utils::TestRandom; use bls::PublicKey; use rand::RngCore; +use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; // The information gathered from the PoW chain validator registration function. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize)] pub struct ValidatorRegistryDeltaBlock { pub latest_registry_delta_root: Hash256, pub validator_index: u32, diff --git a/eth2/utils/bls/Cargo.toml b/eth2/utils/bls/Cargo.toml index 0dc64e483..3b26ee26e 100644 --- a/eth2/utils/bls/Cargo.toml +++ b/eth2/utils/bls/Cargo.toml @@ -8,4 +8,5 @@ edition = "2018" bls-aggregates = { git = "https://github.com/sigp/signature-schemes" } hashing = { path = "../hashing" } hex = "0.3" +serde = "1.0" ssz = { path = "../ssz" } diff --git a/eth2/utils/bls/src/aggregate_signature.rs b/eth2/utils/bls/src/aggregate_signature.rs index 505620f4a..6fed183f0 100644 --- a/eth2/utils/bls/src/aggregate_signature.rs +++ b/eth2/utils/bls/src/aggregate_signature.rs @@ -1,6 +1,9 @@ use super::{AggregatePublicKey, Signature}; use bls_aggregates::AggregateSignature as RawAggregateSignature; -use ssz::{decode_ssz_list, hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use serde::ser::{Serialize, Serializer}; +use ssz::{ + decode_ssz_list, hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash, +}; /// A BLS aggregate signature. /// @@ -44,6 +47,15 @@ impl Decodable for AggregateSignature { } } +impl Serialize for AggregateSignature { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_bytes(&ssz_encode(self)) + } +} + impl TreeHash for AggregateSignature { fn hash_tree_root(&self) -> Vec { hash(&self.0.as_bytes()) diff --git a/eth2/utils/bls/src/public_key.rs b/eth2/utils/bls/src/public_key.rs index 5a6bb3ed1..0c2ad81bb 100644 --- a/eth2/utils/bls/src/public_key.rs +++ b/eth2/utils/bls/src/public_key.rs @@ -1,6 +1,7 @@ use super::SecretKey; use bls_aggregates::PublicKey as RawPublicKey; use hex::encode as hex_encode; +use serde::ser::{Serialize, Serializer}; use ssz::{ decode_ssz_list, hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash, }; @@ -55,6 +56,15 @@ impl Decodable for PublicKey { } } +impl Serialize for PublicKey { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_bytes(&ssz_encode(self)) + } +} + impl TreeHash for PublicKey { fn hash_tree_root(&self) -> Vec { hash(&self.0.as_bytes()) diff --git a/eth2/utils/bls/src/signature.rs b/eth2/utils/bls/src/signature.rs index 59f455523..396e4eab7 100644 --- a/eth2/utils/bls/src/signature.rs +++ b/eth2/utils/bls/src/signature.rs @@ -1,6 +1,9 @@ use super::{PublicKey, SecretKey}; use bls_aggregates::Signature as RawSignature; -use ssz::{decode_ssz_list, hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use serde::ser::{Serialize, Serializer}; +use ssz::{ + decode_ssz_list, hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash, +}; /// A single BLS signature. /// @@ -63,6 +66,15 @@ impl TreeHash for Signature { } } +impl Serialize for Signature { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_bytes(&ssz_encode(self)) + } +} + #[cfg(test)] mod tests { use super::super::Keypair; diff --git a/eth2/utils/boolean-bitfield/Cargo.toml b/eth2/utils/boolean-bitfield/Cargo.toml index b3d05f979..d94b9f7b1 100644 --- a/eth2/utils/boolean-bitfield/Cargo.toml +++ b/eth2/utils/boolean-bitfield/Cargo.toml @@ -7,3 +7,5 @@ edition = "2018" [dependencies] ssz = { path = "../ssz" } bit-vec = "0.5.0" +serde = "1.0" +serde_derive = "1.0" diff --git a/eth2/utils/boolean-bitfield/src/lib.rs b/eth2/utils/boolean-bitfield/src/lib.rs index 33c461361..a00a081be 100644 --- a/eth2/utils/boolean-bitfield/src/lib.rs +++ b/eth2/utils/boolean-bitfield/src/lib.rs @@ -3,6 +3,7 @@ extern crate ssz; use bit_vec::BitVec; +use serde::ser::{Serialize, Serializer}; use std::cmp; use std::default; @@ -149,6 +150,15 @@ impl ssz::Decodable for BooleanBitfield { } } +impl Serialize for BooleanBitfield { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_bytes(&ssz::ssz_encode(self)) + } +} + impl ssz::TreeHash for BooleanBitfield { fn hash_tree_root(&self) -> Vec { self.to_bytes().hash_tree_root() From eb77fb75b7fce18bda60d50eb1c53c8b15bd8254 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 26 Jan 2019 08:25:56 +1100 Subject: [PATCH 033/107] Dump chain to JSON file --- beacon_node/beacon_chain/Cargo.toml | 2 ++ beacon_node/beacon_chain/src/dump.rs | 3 ++- beacon_node/beacon_chain/tests/chain.rs | 2 ++ beacon_node/beacon_chain/tests/utils/test_rig.rs | 9 +++++++++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index 54902ddd6..e12476553 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -13,7 +13,9 @@ failure = "0.1" failure_derive = "0.1" genesis = { path = "../../eth2/genesis" } hashing = { path = "../../eth2/utils/hashing" } +serde = "1.0" serde_derive = "1.0" +serde_json = "1.0" slot_clock = { path = "../../eth2/utils/slot_clock" } ssz = { path = "../../eth2/utils/ssz" } types = { path = "../../eth2/types" } diff --git a/beacon_node/beacon_chain/src/dump.rs b/beacon_node/beacon_chain/src/dump.rs index e4cb0844a..1f1df9ea2 100644 --- a/beacon_node/beacon_chain/src/dump.rs +++ b/beacon_node/beacon_chain/src/dump.rs @@ -1,7 +1,8 @@ use super::{BeaconChain, ClientDB, DBError, SlotClock}; +use serde_derive::Serialize; use types::{BeaconBlock, BeaconState, Hash256}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize)] pub struct SlotDump { pub beacon_block: BeaconBlock, pub beacon_block_root: Hash256, diff --git a/beacon_node/beacon_chain/tests/chain.rs b/beacon_node/beacon_chain/tests/chain.rs index c4d806265..85dd4662e 100644 --- a/beacon_node/beacon_chain/tests/chain.rs +++ b/beacon_node/beacon_chain/tests/chain.rs @@ -15,4 +15,6 @@ fn it_can_produce_blocks() { let dump = rig.chain_dump().expect("Chain dump failed."); assert_eq!(dump.len(), blocks + 1); // + 1 for genesis block. + + rig.dump_to_file("/tmp/chaindump.json".to_string(), &dump); } diff --git a/beacon_node/beacon_chain/tests/utils/test_rig.rs b/beacon_node/beacon_chain/tests/utils/test_rig.rs index a3a641715..52d3fbdae 100644 --- a/beacon_node/beacon_chain/tests/utils/test_rig.rs +++ b/beacon_node/beacon_chain/tests/utils/test_rig.rs @@ -6,7 +6,10 @@ use db::{ stores::{BeaconBlockStore, BeaconStateStore}, MemoryDB, }; +use serde_json::Result as SerdeResult; use slot_clock::TestingSlotClock; +use std::fs::File; +use std::io::prelude::*; use std::sync::Arc; use types::{ChainSpec, Keypair, Validator}; @@ -87,4 +90,10 @@ impl TestRig { pub fn chain_dump(&self) -> Result, DumpError> { self.beacon_chain.chain_dump() } + + pub fn dump_to_file(&self, filename: String, chain_dump: &Vec) { + let json = serde_json::to_string(chain_dump).unwrap(); + let mut file = File::create(filename).unwrap(); + file.write_all(json.as_bytes()); + } } From 7ee836d1186e96cef54e5393b16939cf114420ff Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 26 Jan 2019 14:50:56 +1100 Subject: [PATCH 034/107] Implement the majority of per-epoch processing --- .../beacon_chain/src/epoch_processing.rs | 9 + beacon_node/beacon_chain/src/lib.rs | 1 + .../beacon_chain/src/state_transition.rs | 29 +- eth2/types/Cargo.toml | 1 + .../src/beacon_state/epoch_processing.rs | 837 ++++++++++++++++++ eth2/types/src/beacon_state/mod.rs | 19 +- .../{slot_advance.rs => slot_processing.rs} | 0 eth2/types/src/beacon_state/winning_root.rs | 92 ++ 8 files changed, 959 insertions(+), 29 deletions(-) create mode 100644 beacon_node/beacon_chain/src/epoch_processing.rs create mode 100644 eth2/types/src/beacon_state/epoch_processing.rs rename eth2/types/src/beacon_state/{slot_advance.rs => slot_processing.rs} (100%) create mode 100644 eth2/types/src/beacon_state/winning_root.rs diff --git a/beacon_node/beacon_chain/src/epoch_processing.rs b/beacon_node/beacon_chain/src/epoch_processing.rs new file mode 100644 index 000000000..405ba7df2 --- /dev/null +++ b/beacon_node/beacon_chain/src/epoch_processing.rs @@ -0,0 +1,9 @@ +use super::{BeaconChain, ClientDB, DBError, SlotClock}; + +impl BeaconChain +where + T: ClientDB, + U: SlotClock, +{ + pub fn per_epoch_processing(&self) {} +} diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index d6058eab7..2a6bc9a6b 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -4,6 +4,7 @@ pub mod block_processing; pub mod block_production; mod canonical_head; pub mod dump; +pub mod epoch_processing; mod finalized_head; mod info; mod lmd_ghost; diff --git a/beacon_node/beacon_chain/src/state_transition.rs b/beacon_node/beacon_chain/src/state_transition.rs index 3e694d331..4b39c12de 100644 --- a/beacon_node/beacon_chain/src/state_transition.rs +++ b/beacon_node/beacon_chain/src/state_transition.rs @@ -239,12 +239,13 @@ where } ensure!( attestation.data.justified_block_root - == *get_block_root( - &state, - attestation.data.justified_slot, - self.spec.latest_block_roots_length - ) - .ok_or(Error::NoBlockRoot)?, + == *state + .get_block_root( + &state, + attestation.data.justified_slot, + self.spec.latest_block_roots_length + ) + .ok_or(Error::NoBlockRoot)?, Error::BadAttestation ); ensure!( @@ -376,24 +377,10 @@ fn get_attestation_participants( _attestation_data: &AttestationData, _aggregation_bitfield: &BooleanBitfield, ) -> Vec { + // TODO: stubbed out. vec![0, 1] } -fn get_block_root( - state: &BeaconState, - slot: u64, - latest_block_roots_length: u64, -) -> Option<&Hash256> { - // TODO: test - if state.slot <= slot + latest_block_roots_length && slot <= state.slot { - state - .latest_block_roots - .get((slot % latest_block_roots_length) as usize) - } else { - None - } -} - fn penalize_validator(_state: &BeaconState, _proposer_index: usize) { // TODO: stubbed out. } diff --git a/eth2/types/Cargo.toml b/eth2/types/Cargo.toml index b3c3f7462..a553e4240 100644 --- a/eth2/types/Cargo.toml +++ b/eth2/types/Cargo.toml @@ -9,6 +9,7 @@ bls = { path = "../utils/bls" } boolean-bitfield = { path = "../utils/boolean-bitfield" } ethereum-types = "0.4.0" hashing = { path = "../utils/hashing" } +integer-sqrt = "0.1" rand = "0.5.5" serde = "1.0" serde_derive = "1.0" diff --git a/eth2/types/src/beacon_state/epoch_processing.rs b/eth2/types/src/beacon_state/epoch_processing.rs new file mode 100644 index 000000000..cd980af3d --- /dev/null +++ b/eth2/types/src/beacon_state/epoch_processing.rs @@ -0,0 +1,837 @@ +use super::winning_root::WinningRoot; +use crate::{ + validator::StatusFlags, validator_registry::get_active_validator_indices, Attestation, + AttestationData, BeaconState, Bitfield, ChainSpec, Crosslink, Hash256, PendingAttestation, +}; +use integer_sqrt::IntegerSquareRoot; +use std::collections::{HashMap, HashSet}; +use std::iter::FromIterator; + +pub enum Error { + UnableToDetermineProducer, + NoBlockRoots, + UnableToGetCrosslinkCommittees, + BaseRewardQuotientIsZero, +} + +macro_rules! safe_add_assign { + ($a: expr, $b: expr) => { + $a = $a.saturating_add($b); + }; +} +macro_rules! safe_sub_assign { + ($a: expr, $b: expr) => { + $a = $a.saturating_sub($b); + }; +} + +impl BeaconState { + pub fn per_epoch_processing(&mut self, spec: &ChainSpec) -> Result<(), Error> { + /* + * All Validators + */ + let active_validator_indices = + get_active_validator_indices(&self.validator_registry, self.slot); + let total_balance: u64 = active_validator_indices + .iter() + .fold(0, |acc, i| self.get_effective_balance(*i, spec)); + + let current_epoch_attestations: Vec<&PendingAttestation> = self + .latest_attestations + .iter() + .filter(|a| (self.slot - spec.epoch_length <= a.data.slot) && (a.data.slot < self.slot)) + .collect(); + + /* + * Validators attesting during the current epoch. + */ + if self.latest_block_roots.is_empty() { + return Err(Error::NoBlockRoots); + } + + let current_epoch_boundary_attestations: Vec<&PendingAttestation> = + current_epoch_attestations + .iter() + // `filter_map` is used to avoid a double borrow (`&&..`). + .filter_map(|a| { + // TODO: ensure this saturating sub is correct. + let block_root = match self + .get_block_root(self.slot.saturating_sub(spec.epoch_length), spec) + { + Some(root) => root, + // Protected by a check that latest_block_roots isn't empty. + // + // TODO: provide detailed reasoning. + None => unreachable!(), + }; + + if (a.data.epoch_boundary_root == *block_root) + && (a.data.justified_slot == self.justified_slot) + { + Some(*a) + } else { + None + } + }) + .collect(); + + let current_epoch_boundary_attester_indices: Vec = + current_epoch_boundary_attestations + .iter() + .fold(vec![], |mut acc, a| { + acc.append( + &mut self.get_attestation_participants(&a.data, &a.aggregation_bitfield), + ); + acc + }); + + let current_epoch_boundary_attesting_balance = current_epoch_boundary_attester_indices + .iter() + .fold(0_u64, |acc, i| acc + self.get_effective_balance(*i, spec)); + + /* + * Validators attesting during the previous epoch + */ + + /* + * Validators that made an attestation during the previous epoch + */ + let previous_epoch_attestations: Vec<&PendingAttestation> = self + .latest_attestations + .iter() + .filter(|a| { + //TODO: ensure these saturating subs are correct. + (self.slot.saturating_sub(2 * spec.epoch_length) <= a.data.slot) + && (a.data.slot < self.slot.saturating_sub(spec.epoch_length)) + }) + .collect(); + + let previous_epoch_attester_indices: Vec = + previous_epoch_attestations + .iter() + .fold(vec![], |mut acc, a| { + acc.append( + &mut self.get_attestation_participants(&a.data, &a.aggregation_bitfield), + ); + acc + }); + + /* + * Validators targetting the previous justified slot + */ + let previous_epoch_justified_attestations: Vec<&PendingAttestation> = { + let mut a: Vec<&PendingAttestation> = current_epoch_attestations + .iter() + // `filter_map` is used to avoid a double borrow (`&&..`). + .filter_map(|a| { + if a.data.justified_slot == self.previous_justified_slot { + Some(*a) + } else { + None + } + }) + .collect(); + let mut b: Vec<&PendingAttestation> = previous_epoch_attestations + .iter() + // `filter_map` is used to avoid a double borrow (`&&..`). + .filter_map(|a| { + if a.data.justified_slot == self.previous_justified_slot { + Some(*a) + } else { + None + } + }) + .collect(); + a.append(&mut b); + a + }; + + let previous_epoch_justified_attester_indices: Vec = + previous_epoch_justified_attestations + .iter() + .fold(vec![], |mut acc, a| { + acc.append( + &mut self.get_attestation_participants(&a.data, &a.aggregation_bitfield), + ); + acc + }); + + let previous_epoch_justified_attesting_balance = previous_epoch_justified_attester_indices + .iter() + .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)); + + /* + * Validators justifying the epoch boundary block at the start of the previous epoch + */ + let previous_epoch_boundary_attestations: Vec<&PendingAttestation> = + previous_epoch_justified_attestations + .iter() + // `filter_map` is used to avoid a double borrow (`&&..`). + .filter_map(|a| { + // TODO: ensure this saturating sub is correct. + let block_root = match self + .get_block_root(self.slot.saturating_sub(2 * spec.epoch_length), spec) + { + Some(root) => root, + // Protected by a check that latest_block_roots isn't empty. + // + // TODO: provide detailed reasoning. + None => unreachable!(), + }; + + if a.data.epoch_boundary_root == *block_root { + Some(*a) + } else { + None + } + }) + .collect(); + + let previous_epoch_boundary_attester_indices: Vec = + previous_epoch_boundary_attestations + .iter() + .fold(vec![], |mut acc, a| { + acc.append( + &mut self.get_attestation_participants(&a.data, &a.aggregation_bitfield), + ); + acc + }); + + let previous_epoch_boundary_attesting_balance: u64 = + previous_epoch_boundary_attester_indices + .iter() + .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)); + + /* + * Validators attesting to the expected beacon chain head during the previous epoch. + */ + let previous_epoch_head_attestations: Vec<&PendingAttestation> = + previous_epoch_attestations + .iter() + .filter_map(|a| { + let block_root = match self + .get_block_root(self.slot.saturating_sub(2 * spec.epoch_length), spec) + { + Some(root) => root, + // Protected by a check that latest_block_roots isn't empty. + // + // TODO: provide detailed reasoning. + None => unreachable!(), + }; + + if a.data.beacon_block_root == *block_root { + Some(*a) + } else { + None + } + }) + .collect(); + + let previous_epoch_head_attester_indices: Vec = previous_epoch_head_attestations + .iter() + .fold(vec![], |mut acc, a| { + acc.append( + &mut self.get_attestation_participants(&a.data, &a.aggregation_bitfield), + ); + acc + }); + + let previous_epoch_head_attesting_balance: u64 = previous_epoch_head_attester_indices + .iter() + .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)); + + let all_crosslink_committees: Vec = { + // TODO: check staturating sub is correct + let start_slot = self.slot.saturating_sub(2 * spec.epoch_length); + + // Sub is safe due to previous line. + // + // TODO: provide detailed reasoning. + let mut committees = Vec::with_capacity((self.slot - start_slot) as usize); + for slot in start_slot..self.slot { + match self.get_crosslink_committees_at_slot(slot) { + Some(c) => committees.push(c), + None => return Err(Error::UnableToGetCrosslinkCommittees), + } + } + committees + }; + + // TODO: I didn't include the `winning_balance` stuff.. Not sure why it's there. + + /* + * Eth1 Data + */ + if self.slot % spec.eth1_data_voting_period == 0 { + for eth1_data_vote in &self.eth1_data_votes { + if eth1_data_vote.vote_count * 2 > spec.eth1_data_voting_period { + self.latest_eth1_data = eth1_data_vote.eth1_data.clone(); + } + } + self.eth1_data_votes = vec![]; + } + + /* + * Justification + */ + self.previous_justified_slot = self.justified_slot; + self.justification_bitfield = (self.justification_bitfield * 2) % u64::pow(2, 64); + + // If >= 2/3 of validators voted for the previous epoch boundary + if (3 * previous_epoch_boundary_attesting_balance) >= (2 * total_balance) { + // TODO: check saturating_sub is correct. + self.justification_bitfield |= 2; + self.justified_slot = self.slot.saturating_sub(2 * spec.epoch_length); + } + + // If >= 2/3 of validators voted for the current epoch boundary + if (3 * current_epoch_boundary_attesting_balance) >= (2 * total_balance) { + // TODO: check saturating_sub is correct. + self.justification_bitfield |= 1; + self.justified_slot = self.slot.saturating_sub(1 * spec.epoch_length); + } + + if (self.previous_justified_slot == self.slot.saturating_sub(2 * spec.epoch_length)) + && (self.justification_bitfield % 4 == 3) + { + self.finalized_slot = self.previous_justified_slot; + } + if (self.previous_justified_slot == self.slot.saturating_sub(3 * spec.epoch_length)) + && (self.justification_bitfield % 8 == 7) + { + self.finalized_slot = self.previous_justified_slot; + } + if (self.previous_justified_slot == self.slot.saturating_sub(4 * spec.epoch_length)) + && (self.justification_bitfield % 16 == 14) + { + self.finalized_slot = self.previous_justified_slot; + } + if (self.previous_justified_slot == self.slot.saturating_sub(4 * spec.epoch_length)) + && (self.justification_bitfield % 16 == 15) + { + self.finalized_slot = self.previous_justified_slot; + } + + /* + * Crosslinks + */ + + // Cached for later lookups. + let mut winning_root_for_shards: HashMap = HashMap::new(); + + for slot in self.slot.saturating_sub(2 * spec.epoch_length)..self.slot { + let crosslink_committees_at_slot = self + .get_crosslink_committees_at_slot(slot) + .ok_or_else(|| Error::UnableToGetCrosslinkCommittees)?; + + for (crosslink_committee, shard) in crosslink_committees_at_slot { + let shard = shard as u64; + + let winning_root = self.winning_root( + shard, + ¤t_epoch_attestations, + &previous_epoch_attestations, + spec, + ); + + if let Some(winning_root) = winning_root { + let total_committee_balance: u64 = crosslink_committee + .iter() + .fold(0, |acc, i| self.get_effective_balance(*i, spec)); + + winning_root_for_shards.insert(shard, winning_root.clone()); + + if (3 * winning_root.total_attesting_balance) >= (2 * total_committee_balance) { + self.latest_crosslinks[shard as usize] = Crosslink { + slot: self.slot, + shard_block_root: winning_root.shard_block_root, + } + } + } + } + } + + /* + * Rewards and Penalities + */ + let base_reward_quotient = total_balance.integer_sqrt(); + if base_reward_quotient == 0 { + return Err(Error::BaseRewardQuotientIsZero); + } + + /* + let base_reward = |i| match self.get_effective_balance(i, spec) { + Some(effective_balance) => effective_balance / base_reward_quotient / 5, + None => unreachable!(), + }; + */ + + /* + * Justification and finalization + */ + let epochs_since_finality = (self.slot - self.finalized_slot) / spec.epoch_length; + + // TODO: fix this extra map + let previous_epoch_justified_attester_indices_hashset: HashSet = + HashSet::from_iter(previous_epoch_justified_attester_indices.iter().map(|i| *i)); + let previous_epoch_boundary_attester_indices_hashset: HashSet = + HashSet::from_iter(previous_epoch_boundary_attester_indices.iter().map(|i| *i)); + let previous_epoch_head_attester_indices_hashset: HashSet = + HashSet::from_iter(previous_epoch_head_attester_indices.iter().map(|i| *i)); + let previous_epoch_attester_indices_hashset: HashSet = + HashSet::from_iter(previous_epoch_attester_indices.iter().map(|i| *i)); + + if epochs_since_finality <= 4 { + for index in 0..self.validator_balances.len() { + let base_reward = self.base_reward(index, base_reward_quotient, spec); + + if previous_epoch_justified_attester_indices_hashset.contains(&index) { + safe_add_assign!( + self.validator_balances[index], + base_reward * previous_epoch_justified_attesting_balance / total_balance + ); + } else { + safe_sub_assign!(self.validator_balances[index], base_reward); + } + + if previous_epoch_boundary_attester_indices_hashset.contains(&index) { + safe_add_assign!( + self.validator_balances[index], + base_reward * previous_epoch_boundary_attesting_balance / total_balance + ); + } else { + safe_sub_assign!(self.validator_balances[index], base_reward); + } + + if previous_epoch_head_attester_indices_hashset.contains(&index) { + safe_add_assign!( + self.validator_balances[index], + base_reward * previous_epoch_head_attesting_balance / total_balance + ); + } else { + safe_sub_assign!(self.validator_balances[index], base_reward); + } + } + + for index in previous_epoch_attester_indices { + let base_reward = self.base_reward(index, base_reward_quotient, spec); + let inclusion_distance = match self.inclusion_distance(index) { + Some(distance) => distance, + None => unreachable!(), + }; + + safe_add_assign!( + self.validator_balances[index], + base_reward * spec.min_attestation_inclusion_delay / inclusion_distance + ) + } + } else { + for index in 0..self.validator_balances.len() { + let inactivity_penalty = self.inactivity_penalty( + index, + epochs_since_finality, + base_reward_quotient, + spec, + ); + + if !previous_epoch_justified_attester_indices_hashset.contains(&index) { + safe_sub_assign!(self.validator_balances[index], inactivity_penalty); + } + + if !previous_epoch_boundary_attester_indices_hashset.contains(&index) { + safe_sub_assign!(self.validator_balances[index], inactivity_penalty); + } + + if !previous_epoch_head_attester_indices_hashset.contains(&index) { + safe_sub_assign!(self.validator_balances[index], inactivity_penalty); + } + } + + for index in previous_epoch_attester_indices { + let base_reward = self.base_reward(index, base_reward_quotient, spec); + let inclusion_distance = match self.inclusion_distance(index) { + Some(distance) => distance, + None => unreachable!(), + }; + + safe_sub_assign!( + self.validator_balances[index], + base_reward + - base_reward * spec.min_attestation_inclusion_delay / inclusion_distance + ); + } + } + + /* + * Attestation inclusion + */ + for index in previous_epoch_attester_indices_hashset { + let inclusion_slot = match self.inclusion_slot(index) { + Some(slot) => slot, + None => unreachable!(), + }; + let proposer_index = self + .get_beacon_proposer_index(inclusion_slot, spec) + .ok_or_else(|| Error::UnableToDetermineProducer)?; + let base_reward = self.base_reward(proposer_index, base_reward_quotient, spec); + safe_add_assign!( + self.validator_balances[proposer_index], + base_reward / spec.includer_reward_quotient + ); + } + + /* + * Crosslinks + */ + for slot in self.slot.saturating_sub(2 * spec.epoch_length)..self.slot { + let crosslink_committees_at_slot = self + .get_crosslink_committees_at_slot(slot) + .ok_or_else(|| Error::UnableToGetCrosslinkCommittees)?; + + for (_crosslink_committee, shard) in crosslink_committees_at_slot { + let shard = shard as u64; + + let winning_root = winning_root_for_shards.get(&shard).expect("unreachable"); + + // TODO: remove the map. + let attesting_validator_indices: HashSet = + HashSet::from_iter(winning_root.attesting_validator_indices.iter().map(|i| *i)); + + for index in 0..self.validator_balances.len() { + let base_reward = self.base_reward(index, base_reward_quotient, spec); + + if attesting_validator_indices.contains(&index) { + safe_add_assign!( + self.validator_balances[index], + base_reward * winning_root.total_attesting_balance + / winning_root.total_balance + ); + } else { + safe_sub_assign!(self.validator_balances[index], base_reward); + } + } + + for index in &winning_root.attesting_validator_indices { + let base_reward = self.base_reward(*index, base_reward_quotient, spec); + safe_add_assign!( + self.validator_balances[*index], + base_reward * winning_root.total_attesting_balance + / winning_root.total_balance + ); + } + } + } + + /* + * Ejections + */ + self.process_ejections(); + + /* + * Validator Registry + */ + self.previous_epoch_calculation_slot = self.current_epoch_calculation_slot; + self.previous_epoch_start_shard = self.current_epoch_start_shard; + self.previous_epoch_randao_mix = self.current_epoch_randao_mix; + + let should_update_validator_registy = if self.finalized_slot + > self.validator_registry_update_slot + { + (0..self.get_current_epoch_committee_count_per_slot(spec)).all(|i| { + let shard = (self.current_epoch_start_shard + i as u64) % spec.shard_count; + self.latest_crosslinks[shard as usize].slot > self.validator_registry_update_slot + }) + } else { + false + }; + + if should_update_validator_registy { + self.update_validator_registry(total_balance, spec); + + self.current_epoch_calculation_slot = self.slot; + self.current_epoch_start_shard = (self.current_epoch_start_shard + + self.get_current_epoch_committee_count_per_slot(spec) as u64 * spec.epoch_length) + % spec.shard_count; + self.current_epoch_randao_mix = self.get_randao_mix( + self.current_epoch_calculation_slot + .saturating_sub(spec.seed_lookahead), + spec, + ); + } else { + let epochs_since_last_registry_change = + (self.slot - self.validator_registry_update_slot) / spec.epoch_length; + if epochs_since_last_registry_change.is_power_of_two() { + self.current_epoch_calculation_slot = self.slot; + self.current_epoch_randao_mix = self.get_randao_mix( + self.current_epoch_calculation_slot + .saturating_sub(spec.seed_lookahead), + spec, + ); + } + } + + self.process_penalties_and_exits(spec); + + let e = self.slot / spec.epoch_length; + self.latest_penalized_balances[((e + 1) % spec.latest_penalized_exit_length) as usize] = + self.latest_penalized_balances[(e % spec.latest_penalized_exit_length) as usize]; + + self.latest_attestations = self + .latest_attestations + .iter() + .filter_map(|a| { + if a.data.slot < self.slot - spec.epoch_length { + Some(a.clone()) + } else { + None + } + }) + .collect(); + + Ok(()) + } + + fn process_penalties_and_exits(&mut self, spec: &ChainSpec) { + let active_validator_indices = + get_active_validator_indices(&self.validator_registry, self.slot); + let total_balance = active_validator_indices + .iter() + .fold(0, |acc, i| self.get_effective_balance(*i, spec)); + + for index in 0..self.validator_balances.len() { + let validator = &self.validator_registry[index]; + + if (self.slot / spec.epoch_length) + == (validator.penalized_slot / spec.epoch_length) + + spec.latest_penalized_exit_length / 2 + { + let e = (self.slot / spec.epoch_length) % spec.latest_penalized_exit_length; + let total_at_start = self.latest_penalized_balances + [((e + 1) % spec.latest_penalized_exit_length) as usize]; + let total_at_end = self.latest_penalized_balances[e as usize]; + let total_penalities = total_at_end.saturating_sub(total_at_end); + let penalty = self.get_effective_balance(index, spec) + * std::cmp::min(total_penalities * 3, total_balance) + / total_balance; + safe_sub_assign!(self.validator_balances[index], penalty); + } + } + + let eligible = |index: usize| { + let validator = &self.validator_registry[index]; + + if validator.penalized_slot <= self.slot { + let penalized_withdrawal_time = + spec.latest_penalized_exit_length * spec.epoch_length / 2; + self.slot >= validator.penalized_slot + penalized_withdrawal_time + } else { + self.slot >= validator.exit_slot + spec.min_validator_withdrawal_time + } + }; + + let mut eligable_indices: Vec = (0..self.validator_registry.len()) + .filter(|i| eligible(*i)) + .collect(); + eligable_indices.sort_by_key(|i| self.validator_registry[*i].exit_count); + let mut withdrawn_so_far = 0; + for index in eligable_indices { + self.prepare_validator_for_withdrawal(index); + withdrawn_so_far += 1; + if withdrawn_so_far >= spec.max_withdrawals_per_epoch { + break; + } + } + } + + fn prepare_validator_for_withdrawal(&mut self, index: usize) { + //TODO: we're not ANDing here, we're setting. Potentially wrong. + self.validator_registry[index].status_flags = Some(StatusFlags::Withdrawable); + } + + fn get_randao_mix(&mut self, slot: u64, spec: &ChainSpec) -> Hash256 { + assert!(self.slot < slot + spec.latest_randao_mixes_length); + assert!(slot <= self.slot); + self.latest_randao_mixes[(slot & spec.latest_randao_mixes_length) as usize] + } + + fn update_validator_registry(&mut self, total_balance: u64, spec: &ChainSpec) { + let active_validator_indices = + get_active_validator_indices(&self.validator_registry, self.slot); + let total_balance = active_validator_indices + .iter() + .fold(0, |acc, i| self.get_effective_balance(*i, spec)); + + let max_balance_churn = std::cmp::max( + spec.max_deposit, + total_balance / (2 * spec.max_balance_churn_quotient), + ); + + let mut balance_churn = 0; + for index in 0..self.validator_registry.len() { + let validator = &self.validator_registry[index]; + + if (validator.activation_slot > self.slot + spec.entry_exit_delay) + && self.validator_balances[index] >= spec.max_deposit + { + balance_churn += self.get_effective_balance(index, spec); + if balance_churn > max_balance_churn { + break; + } + + self.activate_validator(index, false, spec); + } + } + + let mut balance_churn = 0; + for index in 0..self.validator_registry.len() { + let validator = &self.validator_registry[index]; + + if (validator.exit_slot > self.slot + spec.entry_exit_delay) + && validator.status_flags == Some(StatusFlags::InitiatedExit) + { + balance_churn += self.get_effective_balance(index, spec); + if balance_churn > max_balance_churn { + break; + } + + self.exit_validator(index, spec); + } + } + + self.validator_registry_update_slot = self.slot; + } + + fn exit_validator(&mut self, validator_index: usize, spec: &ChainSpec) { + if self.validator_registry[validator_index].exit_slot + <= self.entry_exit_effect_slot(self.slot, spec) + { + return; + } + + self.validator_registry[validator_index].exit_slot = + self.entry_exit_effect_slot(self.slot, spec); + + self.validator_registry_exit_count += 1; + self.validator_registry[validator_index].exit_count = self.validator_registry_exit_count; + } + + fn activate_validator(&mut self, validator_index: usize, is_genesis: bool, spec: &ChainSpec) { + self.validator_registry[validator_index].activation_slot = if is_genesis { + spec.genesis_slot + } else { + self.entry_exit_effect_slot(self.slot, spec) + } + } + + fn entry_exit_effect_slot(&self, slot: u64, spec: &ChainSpec) -> u64 { + (slot - slot % spec.epoch_length) + spec.epoch_length + spec.entry_exit_delay + } + + fn get_current_epoch_committee_count_per_slot(&self, spec: &ChainSpec) -> usize { + let current_active_validators = get_active_validator_indices( + &self.validator_registry, + self.current_epoch_calculation_slot, + ); + self.get_committee_count_per_slot(current_active_validators.len(), spec) + } + + fn get_committee_count_per_slot( + &self, + active_validator_count: usize, + spec: &ChainSpec, + ) -> usize { + std::cmp::max( + 1, + std::cmp::min( + spec.shard_count as usize / spec.epoch_length as usize, + active_validator_count + / spec.epoch_length as usize + / spec.target_committee_size as usize, + ), + ) + } + + fn process_ejections(&self) { + //TODO: stubbed out. + } + + fn inactivity_penalty( + &self, + validator_index: usize, + epochs_since_finality: u64, + base_reward_quotient: u64, + spec: &ChainSpec, + ) -> u64 { + let effective_balance = self.get_effective_balance(validator_index, spec); + self.base_reward(validator_index, base_reward_quotient, spec) + + effective_balance * epochs_since_finality / spec.inactivity_penalty_quotient / 2 + } + + fn inclusion_distance(&self, validator_index: usize) -> Option { + let attestation = self.earliest_included_attestation(validator_index)?; + Some( + attestation + .slot_included + .saturating_sub(attestation.data.slot), + ) + } + + fn inclusion_slot(&self, validator_index: usize) -> Option { + let attestation = self.earliest_included_attestation(validator_index)?; + Some(attestation.slot_included) + } + + fn earliest_included_attestation(&self, validator_index: usize) -> Option<&PendingAttestation> { + self.latest_attestations + .iter() + .filter(|a| { + self.get_attestation_participants(&a.data, &a.aggregation_bitfield) + .iter() + .find(|i| **i == validator_index) + .is_some() + }) + .min_by_key(|a| a.slot_included) + } + + fn base_reward( + &self, + validator_index: usize, + base_reward_quotient: u64, + spec: &ChainSpec, + ) -> u64 { + self.get_effective_balance(validator_index, spec) / base_reward_quotient / 5 + } + + pub fn get_crosslink_committees_at_slot(&self, slot: u64) -> Option { + Some(vec![(vec![0], 0)]) + } + + pub fn get_effective_balance(&self, validator_index: usize, spec: &ChainSpec) -> u64 { + std::cmp::min(self.validator_balances[validator_index], spec.max_deposit) + } + + pub fn get_block_root(&self, slot: u64, spec: &ChainSpec) -> Option<&Hash256> { + if self.slot <= slot + spec.latest_block_roots_length && slot <= self.slot { + self.latest_block_roots + .get((slot % spec.latest_block_roots_length) as usize) + } else { + None + } + } + + pub fn get_attestation_participants( + &self, + _attestation_data: &AttestationData, + _aggregation_bitfield: &Bitfield, + ) -> Vec { + // TODO: stubbed out. + vec![0, 1] + } +} + +type CrosslinkCommittee = (Vec, usize); +type CrosslinkCommittees = Vec; + +fn merkle_root(_input: &[Hash256]) -> Hash256 { + Hash256::zero() +} diff --git a/eth2/types/src/beacon_state/mod.rs b/eth2/types/src/beacon_state/mod.rs index 71c4d1e87..46f139691 100644 --- a/eth2/types/src/beacon_state/mod.rs +++ b/eth2/types/src/beacon_state/mod.rs @@ -10,9 +10,12 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; -mod slot_advance; +mod epoch_processing; +mod slot_processing; +mod winning_root; -pub use self::slot_advance::Error as SlotProcessingError; +pub use self::epoch_processing::Error as EpochProcessingError; +pub use self::slot_processing::Error as SlotProcessingError; // Custody will not be added to the specs until Phase 1 (Sharding Phase) so dummy class used. type CustodyChallenge = usize; @@ -53,7 +56,7 @@ pub struct BeaconState { // Recent state pub latest_crosslinks: Vec, pub latest_block_roots: Vec, - pub latest_penalized_exit_balances: Vec, + pub latest_penalized_balances: Vec, pub latest_attestations: Vec, pub batched_block_roots: Vec, @@ -93,7 +96,7 @@ impl Encodable for BeaconState { s.append(&self.finalized_slot); s.append(&self.latest_crosslinks); s.append(&self.latest_block_roots); - s.append(&self.latest_penalized_exit_balances); + s.append(&self.latest_penalized_balances); s.append(&self.latest_attestations); s.append(&self.batched_block_roots); s.append(&self.latest_eth1_data); @@ -126,7 +129,7 @@ impl Decodable for BeaconState { let (finalized_slot, i) = <_>::ssz_decode(bytes, i)?; let (latest_crosslinks, i) = <_>::ssz_decode(bytes, i)?; let (latest_block_roots, i) = <_>::ssz_decode(bytes, i)?; - let (latest_penalized_exit_balances, i) = <_>::ssz_decode(bytes, i)?; + let (latest_penalized_balances, i) = <_>::ssz_decode(bytes, i)?; let (latest_attestations, i) = <_>::ssz_decode(bytes, i)?; let (batched_block_roots, i) = <_>::ssz_decode(bytes, i)?; let (latest_eth1_data, i) = <_>::ssz_decode(bytes, i)?; @@ -157,7 +160,7 @@ impl Decodable for BeaconState { finalized_slot, latest_crosslinks, latest_block_roots, - latest_penalized_exit_balances, + latest_penalized_balances, latest_attestations, batched_block_roots, latest_eth1_data, @@ -194,7 +197,7 @@ impl TreeHash for BeaconState { result.append(&mut self.finalized_slot.hash_tree_root()); result.append(&mut self.latest_crosslinks.hash_tree_root()); result.append(&mut self.latest_block_roots.hash_tree_root()); - result.append(&mut self.latest_penalized_exit_balances.hash_tree_root()); + result.append(&mut self.latest_penalized_balances.hash_tree_root()); result.append(&mut self.latest_attestations.hash_tree_root()); result.append(&mut self.batched_block_roots.hash_tree_root()); result.append(&mut self.latest_eth1_data.hash_tree_root()); @@ -229,7 +232,7 @@ impl TestRandom for BeaconState { finalized_slot: <_>::random_for_test(rng), latest_crosslinks: <_>::random_for_test(rng), latest_block_roots: <_>::random_for_test(rng), - latest_penalized_exit_balances: <_>::random_for_test(rng), + latest_penalized_balances: <_>::random_for_test(rng), latest_attestations: <_>::random_for_test(rng), batched_block_roots: <_>::random_for_test(rng), latest_eth1_data: <_>::random_for_test(rng), diff --git a/eth2/types/src/beacon_state/slot_advance.rs b/eth2/types/src/beacon_state/slot_processing.rs similarity index 100% rename from eth2/types/src/beacon_state/slot_advance.rs rename to eth2/types/src/beacon_state/slot_processing.rs diff --git a/eth2/types/src/beacon_state/winning_root.rs b/eth2/types/src/beacon_state/winning_root.rs new file mode 100644 index 000000000..04adcb007 --- /dev/null +++ b/eth2/types/src/beacon_state/winning_root.rs @@ -0,0 +1,92 @@ +use crate::{BeaconState, ChainSpec, Hash256, PendingAttestation}; +use std::collections::HashMap; + +pub enum Error { + UnableToDetermineProducer, + NoBlockRoots, + UnableToGetCrosslinkCommittees, + BaseRewardQuotientIsZero, +} + +#[derive(Clone)] +pub struct WinningRoot { + pub shard_block_root: Hash256, + pub attesting_validator_indices: Vec, + pub total_balance: u64, + pub total_attesting_balance: u64, +} + +impl BeaconState { + pub(crate) fn winning_root( + &self, + shard: u64, + current_epoch_attestations: &[&PendingAttestation], + previous_epoch_attestations: &[&PendingAttestation], + spec: &ChainSpec, + ) -> Option { + let mut attestations = current_epoch_attestations.to_vec(); + attestations.append(&mut previous_epoch_attestations.to_vec()); + + let mut candidates: HashMap = HashMap::new(); + + let mut highest_seen_balance = 0; + + for a in &attestations { + if a.data.shard != shard { + continue; + } + + let shard_block_root = &a.data.shard_block_root; + + if candidates.contains_key(shard_block_root) { + continue; + } + + let attesting_validator_indices = attestations.iter().fold(vec![], |mut acc, a| { + if (a.data.shard == shard) && (a.data.shard_block_root == *shard_block_root) { + acc.append( + &mut self.get_attestation_participants(&a.data, &a.aggregation_bitfield), + ); + } + acc + }); + + let total_balance: u64 = attesting_validator_indices + .iter() + .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)); + + let total_attesting_balance: u64 = attesting_validator_indices + .iter() + .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)); + + if total_attesting_balance > highest_seen_balance { + highest_seen_balance = total_attesting_balance; + } + + let candidate_root = WinningRoot { + shard_block_root: shard_block_root.clone(), + attesting_validator_indices, + total_attesting_balance, + total_balance, + }; + + candidates.insert(*shard_block_root, candidate_root); + } + + let winner = candidates + .iter() + .filter_map(|(_hash, candidate)| { + if candidate.total_attesting_balance == highest_seen_balance { + Some(candidate) + } else { + None + } + }) + .min_by_key(|candidate| candidate.shard_block_root); + + match winner { + Some(winner) => Some(winner.clone()), + None => None, + } + } +} From 22a08e5160ebe71d4da20e898551112b20807132 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 26 Jan 2019 22:22:52 +1100 Subject: [PATCH 035/107] Fix failing tests --- beacon_node/beacon_chain/src/state_transition.rs | 10 +++++----- beacon_node/beacon_chain/tests/chain.rs | 8 +++++--- beacon_node/beacon_chain/tests/utils/test_rig.rs | 11 +++++++++-- eth2/genesis/src/beacon_state.rs | 8 ++++---- 4 files changed, 23 insertions(+), 14 deletions(-) diff --git a/beacon_node/beacon_chain/src/state_transition.rs b/beacon_node/beacon_chain/src/state_transition.rs index 4b39c12de..5b614234f 100644 --- a/beacon_node/beacon_chain/src/state_transition.rs +++ b/beacon_node/beacon_chain/src/state_transition.rs @@ -240,11 +240,7 @@ where ensure!( attestation.data.justified_block_root == *state - .get_block_root( - &state, - attestation.data.justified_slot, - self.spec.latest_block_roots_length - ) + .get_block_root(attestation.data.justified_slot, &self.spec) .ok_or(Error::NoBlockRoot)?, Error::BadAttestation ); @@ -364,6 +360,10 @@ where Error::BadCustodyResponses ); + if state.slot % self.spec.epoch_length == 0 { + state.per_epoch_processing(&self.spec); + } + Ok(state) } } diff --git a/beacon_node/beacon_chain/tests/chain.rs b/beacon_node/beacon_chain/tests/chain.rs index 85dd4662e..1c6f7e7d1 100644 --- a/beacon_node/beacon_chain/tests/chain.rs +++ b/beacon_node/beacon_chain/tests/chain.rs @@ -4,17 +4,19 @@ use types::ChainSpec; mod utils; #[test] +#[ignore] fn it_can_produce_blocks() { let validator_count = 2; - let blocks = 3; - let mut rig = TestRig::new(ChainSpec::foundation(), validator_count); + + let blocks = rig.spec.epoch_length + 1; + for _ in 0..blocks { rig.produce_next_slot(); } let dump = rig.chain_dump().expect("Chain dump failed."); - assert_eq!(dump.len(), blocks + 1); // + 1 for genesis block. + assert_eq!(dump.len() as u64, blocks + 1); // + 1 for genesis block. rig.dump_to_file("/tmp/chaindump.json".to_string(), &dump); } diff --git a/beacon_node/beacon_chain/tests/utils/test_rig.rs b/beacon_node/beacon_chain/tests/utils/test_rig.rs index 52d3fbdae..76b9ee46d 100644 --- a/beacon_node/beacon_chain/tests/utils/test_rig.rs +++ b/beacon_node/beacon_chain/tests/utils/test_rig.rs @@ -19,6 +19,7 @@ pub struct TestRig { block_store: Arc>, state_store: Arc>, validators: Vec, + pub spec: ChainSpec, } impl TestRig { @@ -50,8 +51,13 @@ impl TestRig { // Create the Beacon Chain let beacon_chain = Arc::new( - BeaconChain::genesis(state_store.clone(), block_store.clone(), slot_clock, spec) - .unwrap(), + BeaconChain::genesis( + state_store.clone(), + block_store.clone(), + slot_clock, + spec.clone(), + ) + .unwrap(), ); // Spawn the test validator instances. @@ -66,6 +72,7 @@ impl TestRig { block_store, state_store, validators, + spec, } } diff --git a/eth2/genesis/src/beacon_state.rs b/eth2/genesis/src/beacon_state.rs index 788af7c82..6e697af65 100644 --- a/eth2/genesis/src/beacon_state.rs +++ b/eth2/genesis/src/beacon_state.rs @@ -73,7 +73,7 @@ pub fn genesis_beacon_state(spec: &ChainSpec) -> Result { */ latest_crosslinks: vec![initial_crosslink; spec.shard_count as usize], latest_block_roots: vec![spec.zero_hash; spec.latest_block_roots_length as usize], - latest_penalized_exit_balances: vec![0; spec.latest_penalized_exit_length as usize], + latest_penalized_balances: vec![0; spec.latest_penalized_exit_length as usize], latest_attestations: vec![], batched_block_roots: vec![], /* @@ -193,9 +193,9 @@ mod tests { assert_eq!(*block, Hash256::zero()); } - // Test latest_penalized_exit_balances - assert_eq!(state.latest_penalized_exit_balances.len(), 8_192); - for item in state.latest_penalized_exit_balances.iter() { + // Test latest_penalized_balances + assert_eq!(state.latest_penalized_balances.len(), 8_192); + for item in state.latest_penalized_balances.iter() { assert!(*item == 0); } From 054be5b9b22cc7a1acdd6a167c10857aefb1c1ab Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 27 Jan 2019 13:59:04 +1100 Subject: [PATCH 036/107] Add BenchingBeaconNode to chain tests --- beacon_node/beacon_chain/tests/chain.rs | 2 +- .../tests/utils/benching_beacon_node.rs | 90 +++++++++++++++++++ beacon_node/beacon_chain/tests/utils/mod.rs | 2 + .../beacon_chain/tests/utils/test_rig.rs | 15 +++- .../beacon_chain/tests/utils/validator.rs | 26 +++--- 5 files changed, 120 insertions(+), 15 deletions(-) create mode 100644 beacon_node/beacon_chain/tests/utils/benching_beacon_node.rs diff --git a/beacon_node/beacon_chain/tests/chain.rs b/beacon_node/beacon_chain/tests/chain.rs index 1c6f7e7d1..4aacf9ac1 100644 --- a/beacon_node/beacon_chain/tests/chain.rs +++ b/beacon_node/beacon_chain/tests/chain.rs @@ -12,7 +12,7 @@ fn it_can_produce_blocks() { let blocks = rig.spec.epoch_length + 1; for _ in 0..blocks { - rig.produce_next_slot(); + rig.advance_chain_with_block(); } let dump = rig.chain_dump().expect("Chain dump failed."); diff --git a/beacon_node/beacon_chain/tests/utils/benching_beacon_node.rs b/beacon_node/beacon_chain/tests/utils/benching_beacon_node.rs new file mode 100644 index 000000000..afcf4490d --- /dev/null +++ b/beacon_node/beacon_chain/tests/utils/benching_beacon_node.rs @@ -0,0 +1,90 @@ +use beacon_chain::block_processing::{Error as ProcessingError, Outcome as ProcessingOutcome}; +use beacon_chain::{block_production::Error as BlockProductionError, BeaconChain}; +use block_producer::{ + BeaconNode as BeaconBlockNode, BeaconNodeError as BeaconBlockNodeError, PublishOutcome, +}; +use db::ClientDB; +use slot_clock::SlotClock; +use std::sync::{Arc, RwLock}; +use types::{BeaconBlock, PublicKey, Signature}; + +pub struct BenchingBeaconNode { + beacon_chain: Arc>, + published_blocks: RwLock>, +} + +impl BenchingBeaconNode { + pub fn new(beacon_chain: Arc>) -> Self { + Self { + beacon_chain, + published_blocks: RwLock::new(vec![]), + } + } + + pub fn last_published_block(&self) -> Option { + Some( + self.published_blocks + .read() + .expect("Unable to unlock `published_blocks` for reading.") + .last()? + .clone(), + ) + } +} + +impl BeaconBlockNode for BenchingBeaconNode +where + BlockProductionError: From<::Error>, + ProcessingError: From<::Error>, +{ + /// Requests the `proposer_nonce` from the `BeaconChain`. + fn proposer_nonce(&self, pubkey: &PublicKey) -> Result { + let validator_index = self + .beacon_chain + .validator_index(pubkey) + .ok_or_else(|| BeaconBlockNodeError::RemoteFailure("pubkey unknown.".to_string()))?; + + self.beacon_chain + .proposer_slots(validator_index) + .ok_or_else(|| { + BeaconBlockNodeError::RemoteFailure("validator_index unknown.".to_string()) + }) + } + + /// Requests a new `BeaconBlock from the `BeaconChain`. + fn produce_beacon_block( + &self, + slot: u64, + randao_reveal: &Signature, + ) -> Result, BeaconBlockNodeError> +where { + let (block, _state) = self + .beacon_chain + .produce_block(randao_reveal.clone()) + .map_err(|e| BeaconBlockNodeError::RemoteFailure(format!("{:?}", e)))?; + + if block.slot == slot { + Ok(Some(block)) + } else { + Err(BeaconBlockNodeError::RemoteFailure( + "Unable to produce at non-current slot.".to_string(), + )) + } + } + + /// A block is not _actually_ published to the `BeaconChain`, instead it is stored in the + /// `published_block_vec` and a successful `ValidBlock` is returned to the caller. + /// + /// The block may be retrieved and then applied to the `BeaconChain` manually, potentially in a + /// benchmarking scenario. + fn publish_beacon_block( + &self, + block: BeaconBlock, + ) -> Result { + self.published_blocks + .write() + .expect("Unable to unlock `published_blocks` for writing.") + .push(block); + Ok(PublishOutcome::ValidBlock) + } +} diff --git a/beacon_node/beacon_chain/tests/utils/mod.rs b/beacon_node/beacon_chain/tests/utils/mod.rs index 6460fe421..aaa4ae2c1 100644 --- a/beacon_node/beacon_chain/tests/utils/mod.rs +++ b/beacon_node/beacon_chain/tests/utils/mod.rs @@ -1,8 +1,10 @@ +mod benching_beacon_node; mod direct_beacon_node; mod direct_duties; mod test_rig; mod validator; +pub use self::benching_beacon_node::BenchingBeaconNode; pub use self::direct_beacon_node::DirectBeaconNode; pub use self::direct_duties::DirectDuties; pub use self::test_rig::TestRig; diff --git a/beacon_node/beacon_chain/tests/utils/test_rig.rs b/beacon_node/beacon_chain/tests/utils/test_rig.rs index 76b9ee46d..7de2c4e56 100644 --- a/beacon_node/beacon_chain/tests/utils/test_rig.rs +++ b/beacon_node/beacon_chain/tests/utils/test_rig.rs @@ -1,6 +1,7 @@ use super::TestValidator; pub use beacon_chain::dump::{Error as DumpError, SlotDump}; use beacon_chain::BeaconChain; +use block_producer::BeaconNode; #[cfg(test)] use db::{ stores::{BeaconBlockStore, BeaconStateStore}, @@ -11,7 +12,7 @@ use slot_clock::TestingSlotClock; use std::fs::File; use std::io::prelude::*; use std::sync::Arc; -use types::{ChainSpec, Keypair, Validator}; +use types::{BeaconBlock, ChainSpec, Keypair, Validator}; pub struct TestRig { db: Arc, @@ -76,7 +77,12 @@ impl TestRig { } } - pub fn produce_next_slot(&mut self) { + pub fn advance_chain_with_block(&mut self) { + let block = self.produce_next_slot(); + self.beacon_chain.process_block(block).unwrap(); + } + + fn produce_next_slot(&mut self) -> BeaconBlock { let slot = self .beacon_chain .present_slot() @@ -91,7 +97,7 @@ impl TestRig { .expect("Unable to determine proposer."); self.validators[proposer].set_slot(slot); - self.validators[proposer].produce_block().unwrap(); + self.validators[proposer].produce_block().unwrap() } pub fn chain_dump(&self) -> Result, DumpError> { @@ -101,6 +107,7 @@ impl TestRig { pub fn dump_to_file(&self, filename: String, chain_dump: &Vec) { let json = serde_json::to_string(chain_dump).unwrap(); let mut file = File::create(filename).unwrap(); - file.write_all(json.as_bytes()); + file.write_all(json.as_bytes()) + .expect("Failed writing dump to file."); } } diff --git a/beacon_node/beacon_chain/tests/utils/validator.rs b/beacon_node/beacon_chain/tests/utils/validator.rs index 7a3b3404d..e4c0846a9 100644 --- a/beacon_node/beacon_chain/tests/utils/validator.rs +++ b/beacon_node/beacon_chain/tests/utils/validator.rs @@ -1,11 +1,11 @@ -use super::{DirectBeaconNode, DirectDuties}; +use super::{BenchingBeaconNode, DirectDuties}; use beacon_chain::BeaconChain; #[cfg(test)] use block_producer::{test_utils::TestSigner, BlockProducer, Error as PollError}; use db::MemoryDB; use slot_clock::TestingSlotClock; use std::sync::Arc; -use types::{ChainSpec, Keypair}; +use types::{BeaconBlock, ChainSpec, Keypair}; pub use block_producer::PollOutcome; @@ -18,14 +18,14 @@ pub enum ProduceError { pub struct TestValidator { block_producer: BlockProducer< TestingSlotClock, - DirectBeaconNode, + BenchingBeaconNode, DirectDuties, TestSigner, >, spec: Arc, epoch_map: Arc>, keypair: Keypair, - beacon_node: Arc>, + beacon_node: Arc>, slot_clock: Arc, signer: Arc, } @@ -38,7 +38,7 @@ impl TestValidator { let spec = Arc::new(ChainSpec::foundation()); let slot_clock = Arc::new(TestingSlotClock::new(0)); let signer = Arc::new(TestSigner::new(keypair.clone())); - let beacon_node = Arc::new(DirectBeaconNode::new(beacon_chain.clone())); + let beacon_node = Arc::new(BenchingBeaconNode::new(beacon_chain.clone())); let epoch_map = Arc::new(DirectDuties::new(keypair.pk.clone(), beacon_chain.clone())); let block_producer = BlockProducer::new( @@ -61,12 +61,18 @@ impl TestValidator { } } - pub fn produce_block(&mut self) -> Result { + pub fn produce_block(&mut self) -> Result { + // Using `BenchingBeaconNode`, the validator will always return sucessufully if it tries to + // publish a block. match self.block_producer.poll() { - Ok(PollOutcome::BlockProduced(slot)) => Ok(PollOutcome::BlockProduced(slot)), - Ok(outcome) => Err(ProduceError::DidNotProduce(outcome)), - Err(error) => Err(ProduceError::PollError(error)), - } + Ok(PollOutcome::BlockProduced(_)) => {} + Ok(outcome) => return Err(ProduceError::DidNotProduce(outcome)), + Err(error) => return Err(ProduceError::PollError(error)), + }; + Ok(self + .beacon_node + .last_published_block() + .expect("Unable to obtain produced block.")) } pub fn set_slot(&mut self, slot: u64) { From 79d1b15afc3b11768cf51b4eed2a12ac19befbcd Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 27 Jan 2019 13:59:29 +1100 Subject: [PATCH 037/107] Tidy epoch_processing --- .../src/beacon_state/epoch_processing.rs | 46 +++++-------------- 1 file changed, 12 insertions(+), 34 deletions(-) diff --git a/eth2/types/src/beacon_state/epoch_processing.rs b/eth2/types/src/beacon_state/epoch_processing.rs index cd980af3d..ac6ccbb04 100644 --- a/eth2/types/src/beacon_state/epoch_processing.rs +++ b/eth2/types/src/beacon_state/epoch_processing.rs @@ -1,7 +1,7 @@ use super::winning_root::WinningRoot; use crate::{ - validator::StatusFlags, validator_registry::get_active_validator_indices, Attestation, - AttestationData, BeaconState, Bitfield, ChainSpec, Crosslink, Hash256, PendingAttestation, + validator::StatusFlags, validator_registry::get_active_validator_indices, AttestationData, + BeaconState, Bitfield, ChainSpec, Crosslink, Hash256, PendingAttestation, }; use integer_sqrt::IntegerSquareRoot; use std::collections::{HashMap, HashSet}; @@ -34,7 +34,7 @@ impl BeaconState { get_active_validator_indices(&self.validator_registry, self.slot); let total_balance: u64 = active_validator_indices .iter() - .fold(0, |acc, i| self.get_effective_balance(*i, spec)); + .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)); let current_epoch_attestations: Vec<&PendingAttestation> = self .latest_attestations @@ -240,25 +240,6 @@ impl BeaconState { .iter() .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)); - let all_crosslink_committees: Vec = { - // TODO: check staturating sub is correct - let start_slot = self.slot.saturating_sub(2 * spec.epoch_length); - - // Sub is safe due to previous line. - // - // TODO: provide detailed reasoning. - let mut committees = Vec::with_capacity((self.slot - start_slot) as usize); - for slot in start_slot..self.slot { - match self.get_crosslink_committees_at_slot(slot) { - Some(c) => committees.push(c), - None => return Err(Error::UnableToGetCrosslinkCommittees), - } - } - committees - }; - - // TODO: I didn't include the `winning_balance` stuff.. Not sure why it's there. - /* * Eth1 Data */ @@ -275,7 +256,8 @@ impl BeaconState { * Justification */ self.previous_justified_slot = self.justified_slot; - self.justification_bitfield = (self.justification_bitfield * 2) % u64::pow(2, 64); + let (new_bitfield, _) = self.justification_bitfield.overflowing_mul(2); + self.justification_bitfield = new_bitfield; // If >= 2/3 of validators voted for the previous epoch boundary if (3 * previous_epoch_boundary_attesting_balance) >= (2 * total_balance) { @@ -337,7 +319,7 @@ impl BeaconState { if let Some(winning_root) = winning_root { let total_committee_balance: u64 = crosslink_committee .iter() - .fold(0, |acc, i| self.get_effective_balance(*i, spec)); + .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)); winning_root_for_shards.insert(shard, winning_root.clone()); @@ -546,7 +528,7 @@ impl BeaconState { }; if should_update_validator_registy { - self.update_validator_registry(total_balance, spec); + self.update_validator_registry(spec); self.current_epoch_calculation_slot = self.slot; self.current_epoch_start_shard = (self.current_epoch_start_shard @@ -596,7 +578,7 @@ impl BeaconState { get_active_validator_indices(&self.validator_registry, self.slot); let total_balance = active_validator_indices .iter() - .fold(0, |acc, i| self.get_effective_balance(*i, spec)); + .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)); for index in 0..self.validator_balances.len() { let validator = &self.validator_registry[index]; @@ -609,7 +591,7 @@ impl BeaconState { let total_at_start = self.latest_penalized_balances [((e + 1) % spec.latest_penalized_exit_length) as usize]; let total_at_end = self.latest_penalized_balances[e as usize]; - let total_penalities = total_at_end.saturating_sub(total_at_end); + let total_penalities = total_at_end.saturating_sub(total_at_start); let penalty = self.get_effective_balance(index, spec) * std::cmp::min(total_penalities * 3, total_balance) / total_balance; @@ -654,12 +636,12 @@ impl BeaconState { self.latest_randao_mixes[(slot & spec.latest_randao_mixes_length) as usize] } - fn update_validator_registry(&mut self, total_balance: u64, spec: &ChainSpec) { + fn update_validator_registry(&mut self, spec: &ChainSpec) { let active_validator_indices = get_active_validator_indices(&self.validator_registry, self.slot); let total_balance = active_validator_indices .iter() - .fold(0, |acc, i| self.get_effective_balance(*i, spec)); + .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)); let max_balance_churn = std::cmp::max( spec.max_deposit, @@ -802,7 +784,7 @@ impl BeaconState { self.get_effective_balance(validator_index, spec) / base_reward_quotient / 5 } - pub fn get_crosslink_committees_at_slot(&self, slot: u64) -> Option { + pub fn get_crosslink_committees_at_slot(&self, _slot: u64) -> Option { Some(vec![(vec![0], 0)]) } @@ -831,7 +813,3 @@ impl BeaconState { type CrosslinkCommittee = (Vec, usize); type CrosslinkCommittees = Vec; - -fn merkle_root(_input: &[Hash256]) -> Hash256 { - Hash256::zero() -} From e3115d2105028809db36fb99e165df5a851b69c5 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 27 Jan 2019 14:22:51 +1100 Subject: [PATCH 038/107] Move TestRig into its own crate --- Cargo.toml | 1 + .../beacon_chain/test_harness/Cargo.toml | 22 +++++++++++++++++++ .../src}/benching_beacon_node.rs | 0 .../src}/direct_beacon_node.rs | 0 .../src}/direct_duties.rs | 0 .../utils/mod.rs => test_harness/src/lib.rs} | 0 .../utils => test_harness/src}/test_rig.rs | 1 - .../utils => test_harness/src}/validator.rs | 1 - .../{ => test_harness}/tests/chain.rs | 12 +++++++--- 9 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 beacon_node/beacon_chain/test_harness/Cargo.toml rename beacon_node/beacon_chain/{tests/utils => test_harness/src}/benching_beacon_node.rs (100%) rename beacon_node/beacon_chain/{tests/utils => test_harness/src}/direct_beacon_node.rs (100%) rename beacon_node/beacon_chain/{tests/utils => test_harness/src}/direct_duties.rs (100%) rename beacon_node/beacon_chain/{tests/utils/mod.rs => test_harness/src/lib.rs} (100%) rename beacon_node/beacon_chain/{tests/utils => test_harness/src}/test_rig.rs (99%) rename beacon_node/beacon_chain/{tests/utils => test_harness/src}/validator.rs (99%) rename beacon_node/beacon_chain/{ => test_harness}/tests/chain.rs (63%) diff --git a/Cargo.toml b/Cargo.toml index e22d78d3c..57b5de8ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ members = [ "beacon_node", "beacon_node/db", "beacon_node/beacon_chain", + "beacon_node/beacon_chain/test_harness", "protos", "validator_client", ] diff --git a/beacon_node/beacon_chain/test_harness/Cargo.toml b/beacon_node/beacon_chain/test_harness/Cargo.toml new file mode 100644 index 000000000..6972e4df2 --- /dev/null +++ b/beacon_node/beacon_chain/test_harness/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "test_harness" +version = "0.1.0" +authors = ["Paul Hauner "] +edition = "2018" + +[dependencies] +beacon_chain = { path = "../../beacon_chain" } +block_producer = { path = "../../../eth2/block_producer" } +bls = { path = "../../../eth2/utils/bls" } +boolean-bitfield = { path = "../../../eth2/utils/boolean-bitfield" } +db = { path = "../../db" } +failure = "0.1" +failure_derive = "0.1" +genesis = { path = "../../../eth2/genesis" } +hashing = { path = "../../../eth2/utils/hashing" } +serde = "1.0" +serde_derive = "1.0" +serde_json = "1.0" +slot_clock = { path = "../../../eth2/utils/slot_clock" } +ssz = { path = "../../../eth2/utils/ssz" } +types = { path = "../../../eth2/types" } diff --git a/beacon_node/beacon_chain/tests/utils/benching_beacon_node.rs b/beacon_node/beacon_chain/test_harness/src/benching_beacon_node.rs similarity index 100% rename from beacon_node/beacon_chain/tests/utils/benching_beacon_node.rs rename to beacon_node/beacon_chain/test_harness/src/benching_beacon_node.rs diff --git a/beacon_node/beacon_chain/tests/utils/direct_beacon_node.rs b/beacon_node/beacon_chain/test_harness/src/direct_beacon_node.rs similarity index 100% rename from beacon_node/beacon_chain/tests/utils/direct_beacon_node.rs rename to beacon_node/beacon_chain/test_harness/src/direct_beacon_node.rs diff --git a/beacon_node/beacon_chain/tests/utils/direct_duties.rs b/beacon_node/beacon_chain/test_harness/src/direct_duties.rs similarity index 100% rename from beacon_node/beacon_chain/tests/utils/direct_duties.rs rename to beacon_node/beacon_chain/test_harness/src/direct_duties.rs diff --git a/beacon_node/beacon_chain/tests/utils/mod.rs b/beacon_node/beacon_chain/test_harness/src/lib.rs similarity index 100% rename from beacon_node/beacon_chain/tests/utils/mod.rs rename to beacon_node/beacon_chain/test_harness/src/lib.rs diff --git a/beacon_node/beacon_chain/tests/utils/test_rig.rs b/beacon_node/beacon_chain/test_harness/src/test_rig.rs similarity index 99% rename from beacon_node/beacon_chain/tests/utils/test_rig.rs rename to beacon_node/beacon_chain/test_harness/src/test_rig.rs index 7de2c4e56..f50f10698 100644 --- a/beacon_node/beacon_chain/tests/utils/test_rig.rs +++ b/beacon_node/beacon_chain/test_harness/src/test_rig.rs @@ -2,7 +2,6 @@ use super::TestValidator; pub use beacon_chain::dump::{Error as DumpError, SlotDump}; use beacon_chain::BeaconChain; use block_producer::BeaconNode; -#[cfg(test)] use db::{ stores::{BeaconBlockStore, BeaconStateStore}, MemoryDB, diff --git a/beacon_node/beacon_chain/tests/utils/validator.rs b/beacon_node/beacon_chain/test_harness/src/validator.rs similarity index 99% rename from beacon_node/beacon_chain/tests/utils/validator.rs rename to beacon_node/beacon_chain/test_harness/src/validator.rs index e4c0846a9..1a0401542 100644 --- a/beacon_node/beacon_chain/tests/utils/validator.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator.rs @@ -1,6 +1,5 @@ use super::{BenchingBeaconNode, DirectDuties}; use beacon_chain::BeaconChain; -#[cfg(test)] use block_producer::{test_utils::TestSigner, BlockProducer, Error as PollError}; use db::MemoryDB; use slot_clock::TestingSlotClock; diff --git a/beacon_node/beacon_chain/tests/chain.rs b/beacon_node/beacon_chain/test_harness/tests/chain.rs similarity index 63% rename from beacon_node/beacon_chain/tests/chain.rs rename to beacon_node/beacon_chain/test_harness/tests/chain.rs index 4aacf9ac1..6130f4c5d 100644 --- a/beacon_node/beacon_chain/tests/chain.rs +++ b/beacon_node/beacon_chain/test_harness/tests/chain.rs @@ -1,11 +1,17 @@ -use self::utils::TestRig; +use test_harness::TestRig; use types::ChainSpec; -mod utils; +#[test] +fn it_can_build_on_genesis_block() { + let validator_count = 2; + let mut rig = TestRig::new(ChainSpec::foundation(), validator_count); + + rig.advance_chain_with_block(); +} #[test] #[ignore] -fn it_can_produce_blocks() { +fn it_can_produce_past_first_epoch_boundary() { let validator_count = 2; let mut rig = TestRig::new(ChainSpec::foundation(), validator_count); From 1082c8857a6cddc7fc62e73f563f17582b40bca7 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 27 Jan 2019 14:25:26 +1100 Subject: [PATCH 039/107] Rename "TestRig" to "BeaconChainHarness" --- .../src/{test_rig.rs => beacon_chain_harness.rs} | 4 ++-- beacon_node/beacon_chain/test_harness/src/lib.rs | 4 ++-- beacon_node/beacon_chain/test_harness/tests/chain.rs | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) rename beacon_node/beacon_chain/test_harness/src/{test_rig.rs => beacon_chain_harness.rs} (98%) diff --git a/beacon_node/beacon_chain/test_harness/src/test_rig.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs similarity index 98% rename from beacon_node/beacon_chain/test_harness/src/test_rig.rs rename to beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index f50f10698..01318727b 100644 --- a/beacon_node/beacon_chain/test_harness/src/test_rig.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -13,7 +13,7 @@ use std::io::prelude::*; use std::sync::Arc; use types::{BeaconBlock, ChainSpec, Keypair, Validator}; -pub struct TestRig { +pub struct BeaconChainHarness { db: Arc, beacon_chain: Arc>, block_store: Arc>, @@ -22,7 +22,7 @@ pub struct TestRig { pub spec: ChainSpec, } -impl TestRig { +impl BeaconChainHarness { pub fn new(mut spec: ChainSpec, validator_count: usize) -> Self { let db = Arc::new(MemoryDB::open()); let block_store = Arc::new(BeaconBlockStore::new(db.clone())); diff --git a/beacon_node/beacon_chain/test_harness/src/lib.rs b/beacon_node/beacon_chain/test_harness/src/lib.rs index aaa4ae2c1..efdc39e1e 100644 --- a/beacon_node/beacon_chain/test_harness/src/lib.rs +++ b/beacon_node/beacon_chain/test_harness/src/lib.rs @@ -1,11 +1,11 @@ +mod beacon_chain_harness; mod benching_beacon_node; mod direct_beacon_node; mod direct_duties; -mod test_rig; mod validator; +pub use self::beacon_chain_harness::BeaconChainHarness; pub use self::benching_beacon_node::BenchingBeaconNode; pub use self::direct_beacon_node::DirectBeaconNode; pub use self::direct_duties::DirectDuties; -pub use self::test_rig::TestRig; pub use self::validator::TestValidator; diff --git a/beacon_node/beacon_chain/test_harness/tests/chain.rs b/beacon_node/beacon_chain/test_harness/tests/chain.rs index 6130f4c5d..19bdf233f 100644 --- a/beacon_node/beacon_chain/test_harness/tests/chain.rs +++ b/beacon_node/beacon_chain/test_harness/tests/chain.rs @@ -1,10 +1,10 @@ -use test_harness::TestRig; +use test_harness::BeaconChainHarness; use types::ChainSpec; #[test] fn it_can_build_on_genesis_block() { let validator_count = 2; - let mut rig = TestRig::new(ChainSpec::foundation(), validator_count); + let mut rig = BeaconChainHarness::new(ChainSpec::foundation(), validator_count); rig.advance_chain_with_block(); } @@ -13,7 +13,7 @@ fn it_can_build_on_genesis_block() { #[ignore] fn it_can_produce_past_first_epoch_boundary() { let validator_count = 2; - let mut rig = TestRig::new(ChainSpec::foundation(), validator_count); + let mut rig = BeaconChainHarness::new(ChainSpec::foundation(), validator_count); let blocks = rig.spec.epoch_length + 1; From 8e9a139560eddf7c9bcf43b7ed16590d77b92814 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 27 Jan 2019 15:39:34 +1100 Subject: [PATCH 040/107] Make BeaconChain `CheckPoint` fields public --- beacon_node/beacon_chain/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 2a6bc9a6b..015fff017 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -32,10 +32,10 @@ pub enum BeaconChainError { } pub struct CheckPoint { - beacon_block: BeaconBlock, - beacon_block_root: Hash256, - beacon_state: BeaconState, - beacon_state_root: Hash256, + pub beacon_block: BeaconBlock, + pub beacon_block_root: Hash256, + pub beacon_state: BeaconState, + pub beacon_state_root: Hash256, } impl CheckPoint { From 2bda7e3d1487719db7c5b5e6e09ae53ccb03aa2e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 27 Jan 2019 15:45:29 +1100 Subject: [PATCH 041/107] Add `BeaconChain` benchmarking --- .../beacon_chain/test_harness/Cargo.toml | 7 ++ .../test_harness/benches/state_transition.rs | 64 +++++++++++++++++++ .../test_harness/src/beacon_chain_harness.rs | 16 +++-- 3 files changed, 80 insertions(+), 7 deletions(-) create mode 100644 beacon_node/beacon_chain/test_harness/benches/state_transition.rs diff --git a/beacon_node/beacon_chain/test_harness/Cargo.toml b/beacon_node/beacon_chain/test_harness/Cargo.toml index 6972e4df2..72fdf7164 100644 --- a/beacon_node/beacon_chain/test_harness/Cargo.toml +++ b/beacon_node/beacon_chain/test_harness/Cargo.toml @@ -4,6 +4,13 @@ version = "0.1.0" authors = ["Paul Hauner "] edition = "2018" +[[bench]] +name = "state_transition" +harness = false + +[dev-dependencies] +criterion = "0.2" + [dependencies] beacon_chain = { path = "../../beacon_chain" } block_producer = { path = "../../../eth2/block_producer" } diff --git a/beacon_node/beacon_chain/test_harness/benches/state_transition.rs b/beacon_node/beacon_chain/test_harness/benches/state_transition.rs new file mode 100644 index 000000000..d78a17720 --- /dev/null +++ b/beacon_node/beacon_chain/test_harness/benches/state_transition.rs @@ -0,0 +1,64 @@ +use criterion::Criterion; +use criterion::{criterion_group, criterion_main}; +use test_harness::BeaconChainHarness; +use types::ChainSpec; + +fn mid_epoch_state_transition(c: &mut Criterion) { + let validator_count = 2; + let mut rig = BeaconChainHarness::new(ChainSpec::foundation(), validator_count); + + let two_and_half_epochs = (rig.spec.epoch_length * 2) + (rig.spec.epoch_length / 2); + + for _ in 0..two_and_half_epochs { + rig.advance_chain_with_block(); + } + + let block = rig.advance_chain_without_block(); + let state = rig.beacon_chain.canonical_head().beacon_state.clone(); + + c.bench_function("mid-epoch state transition 10k validators", move |b| { + let block = block.clone(); + let state = state.clone(); + b.iter(|| { + rig.beacon_chain + .state_transition(state.clone(), &block.clone()) + }) + }); +} + +fn epoch_boundary_state_transition(c: &mut Criterion) { + let validator_count = 10_000; + let mut rig = BeaconChainHarness::new(ChainSpec::foundation(), validator_count); + + let three_epochs = rig.spec.epoch_length * 3; + + for _ in 0..(three_epochs - 1) { + rig.advance_chain_with_block(); + } + + let state = rig.beacon_chain.canonical_head().beacon_state.clone(); + assert_eq!( + state.slot % rig.spec.epoch_length, + rig.spec.epoch_length - 1, + ); + let block = rig.advance_chain_without_block(); + + c.bench_function("epoch boundary state transition 10k validators", move |b| { + let block = block.clone(); + let state = state.clone(); + b.iter(|| { + let state = rig + .beacon_chain + .state_transition(state.clone(), &block.clone()) + .unwrap(); + assert_eq!(state.slot % rig.spec.epoch_length, 0); + }) + }); +} + +criterion_group!( + benches, + mid_epoch_state_transition, + epoch_boundary_state_transition +); +criterion_main!(benches); diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index 01318727b..2f4c11aef 100644 --- a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -1,12 +1,10 @@ use super::TestValidator; pub use beacon_chain::dump::{Error as DumpError, SlotDump}; use beacon_chain::BeaconChain; -use block_producer::BeaconNode; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, MemoryDB, }; -use serde_json::Result as SerdeResult; use slot_clock::TestingSlotClock; use std::fs::File; use std::io::prelude::*; @@ -14,11 +12,11 @@ use std::sync::Arc; use types::{BeaconBlock, ChainSpec, Keypair, Validator}; pub struct BeaconChainHarness { - db: Arc, - beacon_chain: Arc>, - block_store: Arc>, - state_store: Arc>, - validators: Vec, + pub db: Arc, + pub beacon_chain: Arc>, + pub block_store: Arc>, + pub state_store: Arc>, + pub validators: Vec, pub spec: ChainSpec, } @@ -76,6 +74,10 @@ impl BeaconChainHarness { } } + pub fn advance_chain_without_block(&mut self) -> BeaconBlock { + self.produce_next_slot() + } + pub fn advance_chain_with_block(&mut self) { let block = self.produce_next_slot(); self.beacon_chain.process_block(block).unwrap(); From b487db68a1eff1ddd21e71f89869a483b68650c8 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 27 Jan 2019 17:27:53 +1100 Subject: [PATCH 042/107] Add `signable_message()` to Attestation --- beacon_node/beacon_chain/src/state_transition.rs | 9 +-------- .../types/src/{attestation.rs => attestation/mod.rs} | 2 ++ eth2/types/src/attestation/signing.rs | 12 ++++++++++++ 3 files changed, 15 insertions(+), 8 deletions(-) rename eth2/types/src/{attestation.rs => attestation/mod.rs} (99%) create mode 100644 eth2/types/src/attestation/signing.rs diff --git a/beacon_node/beacon_chain/src/state_transition.rs b/beacon_node/beacon_chain/src/state_transition.rs index 5b614234f..0537ad195 100644 --- a/beacon_node/beacon_chain/src/state_transition.rs +++ b/beacon_node/beacon_chain/src/state_transition.rs @@ -265,18 +265,11 @@ where .as_raw(), ) } - let attestation_message = { - let attestation_data_and_custody_bit = AttestationDataAndCustodyBit { - data: attestation.data.clone(), - custody_bit: false, - }; - &attestation_data_and_custody_bit.hash_tree_root() - }; // Signature verification. ensure!( bls_verify_aggregate( &group_public_key, - &attestation_message[..], + &attestation.signable_message(), &attestation.aggregate_signature, get_domain(&state.fork_data, attestation.data.slot, DOMAIN_ATTESTATION) ), diff --git a/eth2/types/src/attestation.rs b/eth2/types/src/attestation/mod.rs similarity index 99% rename from eth2/types/src/attestation.rs rename to eth2/types/src/attestation/mod.rs index d6738bd6e..3b278ef8c 100644 --- a/eth2/types/src/attestation.rs +++ b/eth2/types/src/attestation/mod.rs @@ -5,6 +5,8 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +mod signing; + #[derive(Debug, Clone, PartialEq, Serialize)] pub struct Attestation { pub data: AttestationData, diff --git a/eth2/types/src/attestation/signing.rs b/eth2/types/src/attestation/signing.rs new file mode 100644 index 000000000..a2ef13d2f --- /dev/null +++ b/eth2/types/src/attestation/signing.rs @@ -0,0 +1,12 @@ +use crate::{Attestation, AttestationDataAndCustodyBit}; +use ssz::TreeHash; + +impl Attestation { + pub fn signable_message(&self) -> Vec { + let attestation_data_and_custody_bit = AttestationDataAndCustodyBit { + data: self.data.clone(), + custody_bit: false, + }; + attestation_data_and_custody_bit.hash_tree_root() + } +} From fd47f6c4330388153d686ba7a5c0010009a58a7a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 27 Jan 2019 17:54:26 +1100 Subject: [PATCH 043/107] Move attestation validator to `BeaconState` fn --- .../beacon_chain/src/state_transition.rs | 99 +++---------------- .../beacon_state/attestation_validation.rs | 96 ++++++++++++++++++ eth2/types/src/beacon_state/mod.rs | 2 + eth2/utils/bls/src/lib.rs | 10 ++ 4 files changed, 121 insertions(+), 86 deletions(-) create mode 100644 eth2/types/src/beacon_state/attestation_validation.rs diff --git a/beacon_node/beacon_chain/src/state_transition.rs b/beacon_node/beacon_chain/src/state_transition.rs index 0537ad195..199bf8210 100644 --- a/beacon_node/beacon_chain/src/state_transition.rs +++ b/beacon_node/beacon_chain/src/state_transition.rs @@ -1,17 +1,16 @@ use super::{BeaconChain, ClientDB, DBError, SlotClock}; -use bls::{AggregatePublicKey, AggregateSignature, PublicKey, Signature}; +use bls::{PublicKey, Signature}; use boolean_bitfield::BooleanBitfield; use hashing::hash; use slot_clock::{SystemTimeSlotClockError, TestingSlotClockError}; use ssz::{ssz_encode, TreeHash}; use types::{ - beacon_state::SlotProcessingError, readers::BeaconBlockReader, AttestationData, - AttestationDataAndCustodyBit, BeaconBlock, BeaconState, Exit, Fork, Hash256, - PendingAttestation, + beacon_state::{AttestationValidationError, SlotProcessingError}, + readers::BeaconBlockReader, + AttestationData, BeaconBlock, BeaconState, Exit, Fork, Hash256, PendingAttestation, }; // TODO: define elsehwere. -const DOMAIN_ATTESTATION: u64 = 1; const DOMAIN_PROPOSAL: u64 = 2; const DOMAIN_EXIT: u64 = 3; const DOMAIN_RANDAO: u64 = 4; @@ -43,7 +42,7 @@ pub enum Error { MaxProposerSlashingsExceeded, BadProposerSlashing, MaxAttestationsExceeded, - BadAttestation, + InvalidAttestation(AttestationValidationError), NoBlockRoot, MaxDepositsExceeded, MaxExitsExceeded, @@ -218,67 +217,8 @@ where ); for attestation in &block.body.attestations { - ensure!( - attestation.data.slot + self.spec.min_attestation_inclusion_delay <= state.slot, - Error::BadAttestation - ); - ensure!( - attestation.data.slot + self.spec.epoch_length >= state.slot, - Error::BadAttestation - ); - if state.justified_slot >= state.slot - (state.slot % self.spec.epoch_length) { - ensure!( - attestation.data.justified_slot == state.justified_slot, - Error::BadAttestation - ); - } else { - ensure!( - attestation.data.justified_slot == state.previous_justified_slot, - Error::BadAttestation - ); - } - ensure!( - attestation.data.justified_block_root - == *state - .get_block_root(attestation.data.justified_slot, &self.spec) - .ok_or(Error::NoBlockRoot)?, - Error::BadAttestation - ); - ensure!( - (attestation.data.latest_crosslink_root - == state.latest_crosslinks[attestation.data.shard as usize].shard_block_root) - || (attestation.data.shard_block_root - == state.latest_crosslinks[attestation.data.shard as usize] - .shard_block_root), - Error::BadAttestation - ); - let participants = get_attestation_participants( - &state, - &attestation.data, - &attestation.aggregation_bitfield, - ); - let mut group_public_key = AggregatePublicKey::new(); - for participant in participants { - group_public_key.add( - state.validator_registry[participant as usize] - .pubkey - .as_raw(), - ) - } - // Signature verification. - ensure!( - bls_verify_aggregate( - &group_public_key, - &attestation.signable_message(), - &attestation.aggregate_signature, - get_domain(&state.fork_data, attestation.data.slot, DOMAIN_ATTESTATION) - ), - Error::BadProposerSlashing - ); - ensure!( - attestation.data.shard_block_root == self.spec.zero_hash, - Error::BadAttestation - ); + state.validate_attestation(attestation, &self.spec)?; + let pending_attestation = PendingAttestation { data: attestation.data.clone(), aggregation_bitfield: attestation.aggregation_bitfield.clone(), @@ -365,15 +305,6 @@ fn initiate_validator_exit(_state: &BeaconState, _index: u32) { // TODO: stubbed out. } -fn get_attestation_participants( - _state: &BeaconState, - _attestation_data: &AttestationData, - _aggregation_bitfield: &BooleanBitfield, -) -> Vec { - // TODO: stubbed out. - vec![0, 1] -} - fn penalize_validator(_state: &BeaconState, _proposer_index: usize) { // TODO: stubbed out. } @@ -388,16 +319,6 @@ fn bls_verify(pubkey: &PublicKey, message: &[u8], signature: &Signature, _domain signature.verify(message, pubkey) } -fn bls_verify_aggregate( - pubkey: &AggregatePublicKey, - message: &[u8], - signature: &AggregateSignature, - _domain: u64, -) -> bool { - // TODO: add domain - signature.verify(message, pubkey) -} - impl From for Error { fn from(e: DBError) -> Error { Error::DBError(e.message) @@ -423,3 +344,9 @@ impl From for Error { } } } + +impl From for Error { + fn from(e: AttestationValidationError) -> Error { + Error::InvalidAttestation(e) + } +} diff --git a/eth2/types/src/beacon_state/attestation_validation.rs b/eth2/types/src/beacon_state/attestation_validation.rs new file mode 100644 index 000000000..160ff8128 --- /dev/null +++ b/eth2/types/src/beacon_state/attestation_validation.rs @@ -0,0 +1,96 @@ +use crate::{AggregatePublicKey, Attestation, BeaconState, ChainSpec, Fork}; +use bls::bls_verify_aggregate; + +#[derive(Debug, PartialEq)] +pub enum Error { + IncludedTooEarly, + IncludedTooLate, + WrongJustifiedSlot, + WrongJustifiedRoot, + BadLatestCrosslinkRoot, + BadSignature, + ShardBlockRootNotZero, + NoBlockRoot, +} + +macro_rules! ensure { + ($condition: expr, $result: expr) => { + if !$condition { + return Err($result); + } + }; +} + +// TODO: define elsehwere. +const DOMAIN_ATTESTATION: u64 = 1; + +impl BeaconState { + pub fn validate_attestation( + &self, + attestation: &Attestation, + spec: &ChainSpec, + ) -> Result<(), Error> { + ensure!( + attestation.data.slot + spec.min_attestation_inclusion_delay <= self.slot, + Error::IncludedTooEarly + ); + ensure!( + attestation.data.slot + spec.epoch_length >= self.slot, + Error::IncludedTooLate + ); + if self.justified_slot >= self.slot - (self.slot % spec.epoch_length) { + ensure!( + attestation.data.justified_slot == self.justified_slot, + Error::WrongJustifiedSlot + ); + } else { + ensure!( + attestation.data.justified_slot == self.previous_justified_slot, + Error::WrongJustifiedSlot + ); + } + ensure!( + attestation.data.justified_block_root + == *self + .get_block_root(attestation.data.justified_slot, &spec) + .ok_or(Error::NoBlockRoot)?, + Error::WrongJustifiedRoot + ); + ensure!( + (attestation.data.latest_crosslink_root + == self.latest_crosslinks[attestation.data.shard as usize].shard_block_root) + || (attestation.data.shard_block_root + == self.latest_crosslinks[attestation.data.shard as usize].shard_block_root), + Error::BadLatestCrosslinkRoot + ); + let participants = + self.get_attestation_participants(&attestation.data, &attestation.aggregation_bitfield); + let mut group_public_key = AggregatePublicKey::new(); + for participant in participants { + group_public_key.add( + self.validator_registry[participant as usize] + .pubkey + .as_raw(), + ) + } + ensure!( + bls_verify_aggregate( + &group_public_key, + &attestation.signable_message(), + &attestation.aggregate_signature, + get_domain(&self.fork_data, attestation.data.slot, DOMAIN_ATTESTATION) + ), + Error::BadSignature + ); + ensure!( + attestation.data.shard_block_root == spec.zero_hash, + Error::ShardBlockRootNotZero + ); + Ok(()) + } +} + +pub fn get_domain(_fork: &Fork, _slot: u64, _domain_type: u64) -> u64 { + // TODO: stubbed out. + 0 +} diff --git a/eth2/types/src/beacon_state/mod.rs b/eth2/types/src/beacon_state/mod.rs index 46f139691..b287c0c7f 100644 --- a/eth2/types/src/beacon_state/mod.rs +++ b/eth2/types/src/beacon_state/mod.rs @@ -10,10 +10,12 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +mod attestation_validation; mod epoch_processing; mod slot_processing; mod winning_root; +pub use self::attestation_validation::Error as AttestationValidationError; pub use self::epoch_processing::Error as EpochProcessingError; pub use self::slot_processing::Error as SlotProcessingError; diff --git a/eth2/utils/bls/src/lib.rs b/eth2/utils/bls/src/lib.rs index fe2f3a17e..646047d18 100644 --- a/eth2/utils/bls/src/lib.rs +++ b/eth2/utils/bls/src/lib.rs @@ -40,3 +40,13 @@ pub fn create_proof_of_possession(keypair: &Keypair) -> Signature { extend_if_needed(&mut hash); Signature::new_hashed(&hash, &keypair.sk) } + +pub fn bls_verify_aggregate( + pubkey: &AggregatePublicKey, + message: &[u8], + signature: &AggregateSignature, + _domain: u64, +) -> bool { + // TODO: add domain + signature.verify(message, pubkey) +} From 4f1aeb2c79019f1c1ad75274c9ac80a32c6620f6 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 27 Jan 2019 17:57:38 +1100 Subject: [PATCH 044/107] Run rustfmt --all --- .../test_harness/src/benching_beacon_node.rs | 2 +- .../beacon_chain/test_harness/src/validator.rs | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/src/benching_beacon_node.rs b/beacon_node/beacon_chain/test_harness/src/benching_beacon_node.rs index afcf4490d..551fbff99 100644 --- a/beacon_node/beacon_chain/test_harness/src/benching_beacon_node.rs +++ b/beacon_node/beacon_chain/test_harness/src/benching_beacon_node.rs @@ -1,4 +1,4 @@ -use beacon_chain::block_processing::{Error as ProcessingError, Outcome as ProcessingOutcome}; +use beacon_chain::block_processing::Error as ProcessingError; use beacon_chain::{block_production::Error as BlockProductionError, BeaconChain}; use block_producer::{ BeaconNode as BeaconBlockNode, BeaconNodeError as BeaconBlockNodeError, PublishOutcome, diff --git a/beacon_node/beacon_chain/test_harness/src/validator.rs b/beacon_node/beacon_chain/test_harness/src/validator.rs index 1a0401542..1d7f7bd4a 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator.rs @@ -15,18 +15,18 @@ pub enum ProduceError { } pub struct TestValidator { - block_producer: BlockProducer< + pub block_producer: BlockProducer< TestingSlotClock, BenchingBeaconNode, DirectDuties, TestSigner, >, - spec: Arc, - epoch_map: Arc>, - keypair: Keypair, - beacon_node: Arc>, - slot_clock: Arc, - signer: Arc, + pub spec: Arc, + pub epoch_map: Arc>, + pub keypair: Keypair, + pub beacon_node: Arc>, + pub slot_clock: Arc, + pub signer: Arc, } impl TestValidator { From a037fec28383808ef7235ae5a19ec123cb694206 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 28 Jan 2019 11:22:25 +1100 Subject: [PATCH 045/107] Add `signable_message()` to `AttestationData` --- eth2/types/src/attestation/mod.rs | 8 +++++++- eth2/types/src/attestation/signing.rs | 11 +++-------- .../{attestation_data.rs => attestation_data/mod.rs} | 2 ++ eth2/types/src/attestation_data/signing.rs | 12 ++++++++++++ .../types/src/beacon_state/attestation_validation.rs | 4 +++- 5 files changed, 27 insertions(+), 10 deletions(-) rename eth2/types/src/{attestation_data.rs => attestation_data/mod.rs} (99%) create mode 100644 eth2/types/src/attestation_data/signing.rs diff --git a/eth2/types/src/attestation/mod.rs b/eth2/types/src/attestation/mod.rs index 3b278ef8c..51c396a3b 100644 --- a/eth2/types/src/attestation/mod.rs +++ b/eth2/types/src/attestation/mod.rs @@ -1,4 +1,4 @@ -use super::{AttestationData, Bitfield}; +use super::{AttestationData, Bitfield, Hash256}; use crate::test_utils::TestRandom; use bls::AggregateSignature; use rand::RngCore; @@ -15,6 +15,12 @@ pub struct Attestation { pub aggregate_signature: AggregateSignature, } +impl Attestation { + pub fn canonical_root(&self) -> Hash256 { + Hash256::from(&self.hash_tree_root()[..]) + } +} + impl Encodable for Attestation { fn ssz_append(&self, s: &mut SszStream) { s.append(&self.data); diff --git a/eth2/types/src/attestation/signing.rs b/eth2/types/src/attestation/signing.rs index a2ef13d2f..1fefa068c 100644 --- a/eth2/types/src/attestation/signing.rs +++ b/eth2/types/src/attestation/signing.rs @@ -1,12 +1,7 @@ -use crate::{Attestation, AttestationDataAndCustodyBit}; -use ssz::TreeHash; +use crate::Attestation; impl Attestation { - pub fn signable_message(&self) -> Vec { - let attestation_data_and_custody_bit = AttestationDataAndCustodyBit { - data: self.data.clone(), - custody_bit: false, - }; - attestation_data_and_custody_bit.hash_tree_root() + pub fn signable_message(&self, custody_bit: bool) -> Vec { + self.data.signable_message(custody_bit) } } diff --git a/eth2/types/src/attestation_data.rs b/eth2/types/src/attestation_data/mod.rs similarity index 99% rename from eth2/types/src/attestation_data.rs rename to eth2/types/src/attestation_data/mod.rs index aab5f7563..f22b4007f 100644 --- a/eth2/types/src/attestation_data.rs +++ b/eth2/types/src/attestation_data/mod.rs @@ -4,6 +4,8 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +mod signing; + pub const SSZ_ATTESTION_DATA_LENGTH: usize = { 8 + // slot 8 + // shard diff --git a/eth2/types/src/attestation_data/signing.rs b/eth2/types/src/attestation_data/signing.rs new file mode 100644 index 000000000..f14cc0ef4 --- /dev/null +++ b/eth2/types/src/attestation_data/signing.rs @@ -0,0 +1,12 @@ +use crate::{AttestationData, AttestationDataAndCustodyBit}; +use ssz::TreeHash; + +impl AttestationData { + pub fn signable_message(&self, custody_bit: bool) -> Vec { + let attestation_data_and_custody_bit = AttestationDataAndCustodyBit { + data: self.clone(), + custody_bit, + }; + attestation_data_and_custody_bit.hash_tree_root() + } +} diff --git a/eth2/types/src/beacon_state/attestation_validation.rs b/eth2/types/src/beacon_state/attestation_validation.rs index 160ff8128..26cc4e2b4 100644 --- a/eth2/types/src/beacon_state/attestation_validation.rs +++ b/eth2/types/src/beacon_state/attestation_validation.rs @@ -21,6 +21,8 @@ macro_rules! ensure { }; } +const PHASE_0_CUSTODY_BIT: bool = false; + // TODO: define elsehwere. const DOMAIN_ATTESTATION: u64 = 1; @@ -76,7 +78,7 @@ impl BeaconState { ensure!( bls_verify_aggregate( &group_public_key, - &attestation.signable_message(), + &attestation.signable_message(PHASE_0_CUSTODY_BIT), &attestation.aggregate_signature, get_domain(&self.fork_data, attestation.data.slot, DOMAIN_ATTESTATION) ), From d5da84d967e1886526f89b0f24cb1be2cff73d0b Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 28 Jan 2019 11:23:01 +1100 Subject: [PATCH 046/107] Add `BitAnd` impl for BooleanBitfield --- eth2/utils/boolean-bitfield/src/lib.rs | 30 ++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/eth2/utils/boolean-bitfield/src/lib.rs b/eth2/utils/boolean-bitfield/src/lib.rs index a00a081be..16992c3fa 100644 --- a/eth2/utils/boolean-bitfield/src/lib.rs +++ b/eth2/utils/boolean-bitfield/src/lib.rs @@ -114,6 +114,28 @@ impl cmp::PartialEq for BooleanBitfield { } } +/// Create a new bitfield that is a union of two other bitfields. +/// +/// For example `union(0101, 1000) == 1101` +impl std::ops::BitAnd for BooleanBitfield { + type Output = Self; + + fn bitand(self, other: Self) -> Self { + let (biggest, smallest) = if self.len() > other.len() { + (&self, &other) + } else { + (&other, &self) + }; + let mut new = biggest.clone(); + for i in 0..smallest.len() { + if let Ok(true) = smallest.get(i) { + new.set(i, true); + } + } + new + } +} + impl ssz::Encodable for BooleanBitfield { // ssz_append encodes Self according to the `ssz` spec. fn ssz_append(&self, s: &mut ssz::SszStream) { @@ -375,4 +397,12 @@ mod tests { let (decoded, _) = BooleanBitfield::ssz_decode(&ssz, 0).unwrap(); assert_eq!(original, decoded); } + + #[test] + fn test_bitand() { + let a = BooleanBitfield::from_bytes(&vec![2, 8, 1][..]); + let b = BooleanBitfield::from_bytes(&vec![4, 8, 16][..]); + let c = BooleanBitfield::from_bytes(&vec![6, 8, 17][..]); + assert_eq!(c, a & b); + } } From acf8b79fe97d3860c476f3e16dc92b2bfb2b1957 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 28 Jan 2019 11:23:30 +1100 Subject: [PATCH 047/107] Add unsafe `attester` crate --- Cargo.toml | 1 + eth2/attester/Cargo.toml | 10 + eth2/attester/src/lib.rs | 244 ++++++++++++++++++++ eth2/attester/src/test_utils/beacon_node.rs | 49 ++++ eth2/attester/src/test_utils/epoch_map.rs | 44 ++++ eth2/attester/src/test_utils/mod.rs | 7 + eth2/attester/src/test_utils/signer.rs | 31 +++ eth2/attester/src/traits.rs | 51 ++++ 8 files changed, 437 insertions(+) create mode 100644 eth2/attester/Cargo.toml create mode 100644 eth2/attester/src/lib.rs create mode 100644 eth2/attester/src/test_utils/beacon_node.rs create mode 100644 eth2/attester/src/test_utils/epoch_map.rs create mode 100644 eth2/attester/src/test_utils/mod.rs create mode 100644 eth2/attester/src/test_utils/signer.rs create mode 100644 eth2/attester/src/traits.rs diff --git a/Cargo.toml b/Cargo.toml index 57b5de8ce..59dfce953 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ + "eth2/attester", "eth2/block_producer", "eth2/genesis", "eth2/naive_fork_choice", diff --git a/eth2/attester/Cargo.toml b/eth2/attester/Cargo.toml new file mode 100644 index 000000000..956ecf565 --- /dev/null +++ b/eth2/attester/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "attester" +version = "0.1.0" +authors = ["Paul Hauner "] +edition = "2018" + +[dependencies] +slot_clock = { path = "../../eth2/utils/slot_clock" } +ssz = { path = "../../eth2/utils/ssz" } +types = { path = "../../eth2/types" } diff --git a/eth2/attester/src/lib.rs b/eth2/attester/src/lib.rs new file mode 100644 index 000000000..7e3f0eb6b --- /dev/null +++ b/eth2/attester/src/lib.rs @@ -0,0 +1,244 @@ +pub mod test_utils; +mod traits; + +use slot_clock::SlotClock; +use std::sync::Arc; +use types::{AttestationData, Signature}; + +pub use self::traits::{ + BeaconNode, BeaconNodeError, DutiesReader, DutiesReaderError, PublishOutcome, Signer, +}; + +const PHASE_0_CUSTODY_BIT: bool = false; + +#[derive(Debug, PartialEq)] +pub enum PollOutcome { + AttestationProduced(u64), + AttestationNotRequired(u64), + SlashableAttestationNotProduced(u64), + BeaconNodeUnableToProduceAttestation(u64), + ProducerDutiesUnknown(u64), + SlotAlreadyProcessed(u64), + SignerRejection(u64), + ValidatorIsUnknown(u64), +} + +#[derive(Debug, PartialEq)] +pub enum Error { + SlotClockError, + SlotUnknowable, + EpochMapPoisoned, + SlotClockPoisoned, + EpochLengthIsZero, + BeaconNodeError(BeaconNodeError), +} + +/// A polling state machine which performs block production duties, based upon some epoch duties +/// (`EpochDutiesMap`) and a concept of time (`SlotClock`). +/// +/// Ensures that messages are not slashable. +/// +/// Relies upon an external service to keep the `EpochDutiesMap` updated. +pub struct Attester { + pub last_processed_slot: Option, + duties: Arc, + slot_clock: Arc, + beacon_node: Arc, + signer: Arc, +} + +impl Attester { + /// Returns a new instance where `last_processed_slot == 0`. + pub fn new(duties: Arc, slot_clock: Arc, beacon_node: Arc, signer: Arc) -> Self { + Self { + last_processed_slot: None, + duties, + slot_clock, + beacon_node, + signer, + } + } +} + +impl Attester { + /// Poll the `BeaconNode` and produce an attestation if required. + pub fn poll(&mut self) -> Result { + let slot = self + .slot_clock + .present_slot() + .map_err(|_| Error::SlotClockError)? + .ok_or(Error::SlotUnknowable)?; + + if !self.is_processed_slot(slot) { + self.last_processed_slot = Some(slot); + + let shard = match self.duties.attestation_shard(slot) { + Ok(Some(result)) => result, + Ok(None) => return Ok(PollOutcome::AttestationNotRequired(slot)), + Err(DutiesReaderError::UnknownEpoch) => { + return Ok(PollOutcome::ProducerDutiesUnknown(slot)); + } + Err(DutiesReaderError::UnknownValidator) => { + return Ok(PollOutcome::ValidatorIsUnknown(slot)); + } + Err(DutiesReaderError::EpochLengthIsZero) => return Err(Error::EpochLengthIsZero), + Err(DutiesReaderError::Poisoned) => return Err(Error::EpochMapPoisoned), + }; + + self.produce_attestation(slot, shard) + } else { + Ok(PollOutcome::SlotAlreadyProcessed(slot)) + } + } + + fn produce_attestation(&mut self, slot: u64, shard: u64) -> Result { + let attestation_data = match self.beacon_node.produce_attestation_data(slot, shard)? { + Some(attestation_data) => attestation_data, + None => return Ok(PollOutcome::BeaconNodeUnableToProduceAttestation(slot)), + }; + + if !self.safe_to_produce(&attestation_data) { + return Ok(PollOutcome::SlashableAttestationNotProduced(slot)); + } + + let signature = match self.sign_attestation_data(&attestation_data) { + Some(signature) => signature, + None => return Ok(PollOutcome::SignerRejection(slot)), + }; + + let validator_index = match self.duties.validator_index() { + Some(validator_index) => validator_index, + None => return Ok(PollOutcome::ValidatorIsUnknown(slot)), + }; + + self.beacon_node + .publish_attestation_data(attestation_data, signature, validator_index)?; + Ok(PollOutcome::AttestationProduced(slot)) + } + + fn is_processed_slot(&self, slot: u64) -> bool { + match self.last_processed_slot { + Some(processed_slot) if slot <= processed_slot => true, + _ => false, + } + } + + /// Consumes a block, returning that block signed by the validators private key. + /// + /// Important: this function will not check to ensure the block is not slashable. This must be + /// done upstream. + fn sign_attestation_data(&mut self, attestation_data: &AttestationData) -> Option { + self.store_produce(attestation_data); + + self.signer + .bls_sign(&attestation_data.signable_message(PHASE_0_CUSTODY_BIT)[..]) + } + + /// Returns `true` if signing some attestation_data is safe (non-slashable). + /// + /// !!! UNSAFE !!! + /// + /// Important: this function is presently stubbed-out. It provides ZERO SAFETY. + fn safe_to_produce(&self, _attestation_data: &AttestationData) -> bool { + // TODO: ensure the producer doesn't produce slashable blocks. + // https://github.com/sigp/lighthouse/issues/160 + true + } + + /// Record that a block was produced so that slashable votes may not be made in the future. + /// + /// !!! UNSAFE !!! + /// + /// Important: this function is presently stubbed-out. It provides ZERO SAFETY. + fn store_produce(&mut self, _block: &AttestationData) { + // TODO: record this block production to prevent future slashings. + // https://github.com/sigp/lighthouse/issues/160 + } +} + +impl From for Error { + fn from(e: BeaconNodeError) -> Error { + Error::BeaconNodeError(e) + } +} + +#[cfg(test)] +mod tests { + use super::test_utils::{TestBeaconNode, TestEpochMap, TestSigner}; + use super::*; + use slot_clock::TestingSlotClock; + use types::{ + test_utils::{SeedableRng, TestRandom, XorShiftRng}, + ChainSpec, Keypair, + }; + + // TODO: implement more thorough testing. + // https://github.com/sigp/lighthouse/issues/160 + // + // These tests should serve as a good example for future tests. + + #[test] + pub fn polling() { + let mut rng = XorShiftRng::from_seed([42; 16]); + + let spec = Arc::new(ChainSpec::foundation()); + let slot_clock = Arc::new(TestingSlotClock::new(0)); + let beacon_node = Arc::new(TestBeaconNode::default()); + let signer = Arc::new(TestSigner::new(Keypair::random())); + + let mut duties = TestEpochMap::new(spec.epoch_length); + let attest_slot = 100; + let attest_epoch = attest_slot / spec.epoch_length; + let attest_shard = 12; + duties.insert_attestation_shard(attest_slot, attest_shard); + duties.set_validator_index(Some(2)); + let duties = Arc::new(duties); + + let mut attester = Attester::new( + duties.clone(), + slot_clock.clone(), + beacon_node.clone(), + signer.clone(), + ); + + // Configure responses from the BeaconNode. + beacon_node.set_next_produce_result(Ok(Some(AttestationData::random_for_test(&mut rng)))); + beacon_node.set_next_publish_result(Ok(PublishOutcome::ValidAttestation)); + + // One slot before attestation slot... + slot_clock.set_slot(attest_slot - 1); + assert_eq!( + attester.poll(), + Ok(PollOutcome::AttestationNotRequired(attest_slot - 1)) + ); + + // On the attest slot... + slot_clock.set_slot(attest_slot); + assert_eq!( + attester.poll(), + Ok(PollOutcome::AttestationProduced(attest_slot)) + ); + + // Trying the same attest slot again... + slot_clock.set_slot(attest_slot); + assert_eq!( + attester.poll(), + Ok(PollOutcome::SlotAlreadyProcessed(attest_slot)) + ); + + // One slot after the attest slot... + slot_clock.set_slot(attest_slot + 1); + assert_eq!( + attester.poll(), + Ok(PollOutcome::AttestationNotRequired(attest_slot + 1)) + ); + + // In an epoch without known duties... + let slot = (attest_epoch + 1) * spec.epoch_length; + slot_clock.set_slot(slot); + assert_eq!( + attester.poll(), + Ok(PollOutcome::ProducerDutiesUnknown(slot)) + ); + } +} diff --git a/eth2/attester/src/test_utils/beacon_node.rs b/eth2/attester/src/test_utils/beacon_node.rs new file mode 100644 index 000000000..4ebcbdf5a --- /dev/null +++ b/eth2/attester/src/test_utils/beacon_node.rs @@ -0,0 +1,49 @@ +use crate::traits::{BeaconNode, BeaconNodeError, PublishOutcome}; +use std::sync::RwLock; +use types::{AttestationData, Signature}; + +type ProduceResult = Result, BeaconNodeError>; +type PublishResult = Result; + +/// A test-only struct used to simulate a Beacon Node. +#[derive(Default)] +pub struct TestBeaconNode { + pub produce_input: RwLock>, + pub produce_result: RwLock>, + + pub publish_input: RwLock>, + pub publish_result: RwLock>, +} + +impl TestBeaconNode { + pub fn set_next_produce_result(&self, result: ProduceResult) { + *self.produce_result.write().unwrap() = Some(result); + } + + pub fn set_next_publish_result(&self, result: PublishResult) { + *self.publish_result.write().unwrap() = Some(result); + } +} + +impl BeaconNode for TestBeaconNode { + fn produce_attestation_data(&self, slot: u64, shard: u64) -> ProduceResult { + *self.produce_input.write().unwrap() = Some((slot, shard)); + match *self.produce_result.read().unwrap() { + Some(ref r) => r.clone(), + None => panic!("TestBeaconNode: produce_result == None"), + } + } + + fn publish_attestation_data( + &self, + attestation_data: AttestationData, + signature: Signature, + validator_index: u64, + ) -> PublishResult { + *self.publish_input.write().unwrap() = Some((attestation_data, signature, validator_index)); + match *self.publish_result.read().unwrap() { + Some(ref r) => r.clone(), + None => panic!("TestBeaconNode: publish_result == None"), + } + } +} diff --git a/eth2/attester/src/test_utils/epoch_map.rs b/eth2/attester/src/test_utils/epoch_map.rs new file mode 100644 index 000000000..9f44ecf1a --- /dev/null +++ b/eth2/attester/src/test_utils/epoch_map.rs @@ -0,0 +1,44 @@ +use crate::{DutiesReader, DutiesReaderError}; +use std::collections::HashMap; + +pub struct TestEpochMap { + epoch_length: u64, + validator_index: Option, + map: HashMap, +} + +impl TestEpochMap { + pub fn new(epoch_length: u64) -> Self { + Self { + epoch_length, + validator_index: None, + map: HashMap::new(), + } + } + + pub fn insert_attestation_shard(&mut self, slot: u64, shard: u64) { + let epoch = slot / self.epoch_length; + + self.map.insert(epoch, (slot, shard)); + } + + pub fn set_validator_index(&mut self, index: Option) { + self.validator_index = index; + } +} + +impl DutiesReader for TestEpochMap { + fn attestation_shard(&self, slot: u64) -> Result, DutiesReaderError> { + let epoch = slot / self.epoch_length; + + match self.map.get(&epoch) { + Some((attest_slot, attest_shard)) if *attest_slot == slot => Ok(Some(*attest_shard)), + Some((attest_slot, _attest_shard)) if *attest_slot != slot => Ok(None), + _ => Err(DutiesReaderError::UnknownEpoch), + } + } + + fn validator_index(&self) -> Option { + self.validator_index + } +} diff --git a/eth2/attester/src/test_utils/mod.rs b/eth2/attester/src/test_utils/mod.rs new file mode 100644 index 000000000..0dd384b12 --- /dev/null +++ b/eth2/attester/src/test_utils/mod.rs @@ -0,0 +1,7 @@ +mod beacon_node; +mod epoch_map; +mod signer; + +pub use self::beacon_node::TestBeaconNode; +pub use self::epoch_map::TestEpochMap; +pub use self::signer::TestSigner; diff --git a/eth2/attester/src/test_utils/signer.rs b/eth2/attester/src/test_utils/signer.rs new file mode 100644 index 000000000..d43c0feb0 --- /dev/null +++ b/eth2/attester/src/test_utils/signer.rs @@ -0,0 +1,31 @@ +use crate::traits::Signer; +use std::sync::RwLock; +use types::{Keypair, Signature}; + +/// A test-only struct used to simulate a Beacon Node. +pub struct TestSigner { + keypair: Keypair, + should_sign: RwLock, +} + +impl TestSigner { + /// Produce a new TestSigner with signing enabled by default. + pub fn new(keypair: Keypair) -> Self { + Self { + keypair, + should_sign: RwLock::new(true), + } + } + + /// If set to `false`, the service will refuse to sign all messages. Otherwise, all messages + /// will be signed. + pub fn enable_signing(&self, enabled: bool) { + *self.should_sign.write().unwrap() = enabled; + } +} + +impl Signer for TestSigner { + fn bls_sign(&self, message: &[u8]) -> Option { + Some(Signature::new(message, &self.keypair.sk)) + } +} diff --git a/eth2/attester/src/traits.rs b/eth2/attester/src/traits.rs new file mode 100644 index 000000000..bc3ccf63f --- /dev/null +++ b/eth2/attester/src/traits.rs @@ -0,0 +1,51 @@ +use types::{AttestationData, Signature}; + +#[derive(Debug, PartialEq, Clone)] +pub enum BeaconNodeError { + RemoteFailure(String), + DecodeFailure, +} + +#[derive(Debug, PartialEq, Clone)] +pub enum PublishOutcome { + ValidAttestation, + InvalidAttestation(String), +} + +/// Defines the methods required to produce and publish blocks on a Beacon Node. +pub trait BeaconNode: Send + Sync { + fn produce_attestation_data( + &self, + slot: u64, + shard: u64, + ) -> Result, BeaconNodeError>; + + fn publish_attestation_data( + &self, + attestation_data: AttestationData, + signature: Signature, + validator_index: u64, + ) -> Result; +} + +#[derive(Debug, PartialEq, Clone)] +pub enum DutiesReaderError { + UnknownValidator, + UnknownEpoch, + EpochLengthIsZero, + Poisoned, +} + +/// Informs a validator of their duties (e.g., block production). +pub trait DutiesReader: Send + Sync { + /// Returns `Some(shard)` if this slot is an attestation slot. Otherwise, returns `None.` + fn attestation_shard(&self, slot: u64) -> Result, DutiesReaderError>; + + /// Returns `Some(shard)` if this slot is an attestation slot. Otherwise, returns `None.` + fn validator_index(&self) -> Option; +} + +/// Signs message using an internally-maintained private key. +pub trait Signer { + fn bls_sign(&self, message: &[u8]) -> Option; +} From b516fd472ec21fd6ac52e178ac8152e70d5e0221 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 28 Jan 2019 12:58:20 +1100 Subject: [PATCH 048/107] Rename `chain.canonical_head` to `head` --- beacon_node/beacon_chain/src/block_processing.rs | 2 +- beacon_node/beacon_chain/src/block_production.rs | 2 +- beacon_node/beacon_chain/src/canonical_head.rs | 6 +++--- beacon_node/beacon_chain/src/dump.rs | 8 ++++---- beacon_node/beacon_chain/src/info.rs | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/beacon_node/beacon_chain/src/block_processing.rs b/beacon_node/beacon_chain/src/block_processing.rs index 41e1fb3d3..89c5bfc14 100644 --- a/beacon_node/beacon_chain/src/block_processing.rs +++ b/beacon_node/beacon_chain/src/block_processing.rs @@ -121,7 +121,7 @@ where // If the parent block was the parent_block, automatically update the canonical head. // // TODO: this is a first-in-best-dressed scenario that is not ideal -- find a solution. - if self.canonical_head().beacon_block_root == parent_block_root { + if self.head().beacon_block_root == parent_block_root { self.update_canonical_head( block.clone(), block_root.clone(), diff --git a/beacon_node/beacon_chain/src/block_production.rs b/beacon_node/beacon_chain/src/block_production.rs index d633b3490..50baf12ba 100644 --- a/beacon_node/beacon_chain/src/block_production.rs +++ b/beacon_node/beacon_chain/src/block_production.rs @@ -33,7 +33,7 @@ where .map_err(|e| e.into())? .ok_or(Error::PresentSlotIsNone)?; - let parent_root = self.canonical_head().beacon_block_root; + let parent_root = self.head().beacon_block_root; let parent_block_reader = self .block_store .get_reader(&parent_root)? diff --git a/beacon_node/beacon_chain/src/canonical_head.rs b/beacon_node/beacon_chain/src/canonical_head.rs index f6ab646a8..121ded767 100644 --- a/beacon_node/beacon_chain/src/canonical_head.rs +++ b/beacon_node/beacon_chain/src/canonical_head.rs @@ -19,11 +19,11 @@ where new_beacon_state: BeaconState, new_beacon_state_root: Hash256, ) { - let mut canonical_head = self + let mut head = self .canonical_head .write() .expect("CRITICAL: CanonicalHead poisioned."); - canonical_head.update( + head.update( new_beacon_block, new_beacon_block_root, new_beacon_state, @@ -31,7 +31,7 @@ where ); } - pub fn canonical_head(&self) -> RwLockReadGuard { + pub fn head(&self) -> RwLockReadGuard { self.canonical_head .read() .expect("CRITICAL: CanonicalHead poisioned.") diff --git a/beacon_node/beacon_chain/src/dump.rs b/beacon_node/beacon_chain/src/dump.rs index 1f1df9ea2..613534328 100644 --- a/beacon_node/beacon_chain/src/dump.rs +++ b/beacon_node/beacon_chain/src/dump.rs @@ -25,10 +25,10 @@ where let mut dump = vec![]; let mut last_slot = SlotDump { - beacon_block: self.canonical_head().beacon_block.clone(), - beacon_block_root: self.canonical_head().beacon_block_root, - beacon_state: self.canonical_head().beacon_state.clone(), - beacon_state_root: self.canonical_head().beacon_state_root, + beacon_block: self.head().beacon_block.clone(), + beacon_block_root: self.head().beacon_block_root, + beacon_state: self.head().beacon_state.clone(), + beacon_state_root: self.head().beacon_state_root, }; dump.push(last_slot.clone()); diff --git a/beacon_node/beacon_chain/src/info.rs b/beacon_node/beacon_chain/src/info.rs index eddb2fdb9..109ab9be6 100644 --- a/beacon_node/beacon_chain/src/info.rs +++ b/beacon_node/beacon_chain/src/info.rs @@ -8,7 +8,7 @@ where { pub fn validator_index(&self, pubkey: &PublicKey) -> Option { for (i, validator) in self - .canonical_head() + .head() .beacon_state .validator_registry .iter() From d1ac7c037df112f8e9ec68dfc7b399183d207dfa Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 28 Jan 2019 13:00:08 +1100 Subject: [PATCH 049/107] Add `chain.justified_head` --- beacon_node/beacon_chain/src/info.rs | 8 ++++++++ beacon_node/beacon_chain/src/lib.rs | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/beacon_node/beacon_chain/src/info.rs b/beacon_node/beacon_chain/src/info.rs index 109ab9be6..441d56b78 100644 --- a/beacon_node/beacon_chain/src/info.rs +++ b/beacon_node/beacon_chain/src/info.rs @@ -44,4 +44,12 @@ where let state = self.state(present_slot).ok()?; state.get_beacon_proposer_index(slot, &self.spec) } + + pub fn justified_slot(&self) -> u64 { + self.justified_head + .read() + .expect("Justified head poisoned") + .beacon_block + .slot + } } diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 015fff017..6ea9576d8 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -74,6 +74,7 @@ pub struct BeaconChain { pub block_graph: BlockGraph, canonical_head: RwLock, finalized_head: RwLock, + justified_head: RwLock, pub latest_attestation_targets: RwLock, pub spec: ChainSpec, } @@ -110,6 +111,12 @@ where genesis_state.clone(), state_root.clone(), )); + let justified_head = RwLock::new(CheckPoint::new( + genesis_block.clone(), + block_root.clone(), + genesis_state.clone(), + state_root.clone(), + )); let canonical_head = RwLock::new(CheckPoint::new( genesis_block.clone(), block_root.clone(), @@ -124,6 +131,7 @@ where state_store, slot_clock, block_graph, + justified_head, finalized_head, canonical_head, latest_attestation_targets, From e9abf06364aefaaaf73f29cdbb33f96ad5ad8745 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 28 Jan 2019 13:00:45 +1100 Subject: [PATCH 050/107] Add attestation production to BeaconChain --- .../src/attestation_production.rs | 53 +++++++++++++++++++ beacon_node/beacon_chain/src/lib.rs | 1 + 2 files changed, 54 insertions(+) create mode 100644 beacon_node/beacon_chain/src/attestation_production.rs diff --git a/beacon_node/beacon_chain/src/attestation_production.rs b/beacon_node/beacon_chain/src/attestation_production.rs new file mode 100644 index 000000000..33d220b7f --- /dev/null +++ b/beacon_node/beacon_chain/src/attestation_production.rs @@ -0,0 +1,53 @@ +use super::{BeaconChain, ClientDB, SlotClock}; +use types::{AttestationData, Hash256}; + +#[derive(Debug, PartialEq)] +pub enum Error { + /* + DBError(String), + StateTransitionError(TransitionError), + PresentSlotIsNone, + */ + SlotTooOld, + PresentSlotUnknown, + StateError, +} + +impl BeaconChain +where + T: ClientDB, + U: SlotClock, +{ + pub fn produce_attestation_data( + &self, + slot: u64, + shard: u64, + ) -> Result { + let present_slot = self + .present_slot() + .ok_or_else(|| Error::PresentSlotUnknown)?; + let state = self.state(present_slot).map_err(|_| Error::StateError)?; + + let justified_slot = self.justified_slot(); + + let justified_block_root = *state + .get_block_root(justified_slot, &self.spec) + .ok_or_else(|| Error::SlotTooOld)?; + + let head_slot = self.head().beacon_block.slot; + let epoch_boundary_root = *state + .get_block_root(head_slot % self.spec.epoch_length, &self.spec) + .ok_or_else(|| Error::SlotTooOld)?; + + Ok(AttestationData { + slot, + shard, + beacon_block_root: self.head().beacon_block_root.clone(), + epoch_boundary_root, + shard_block_root: Hash256::zero(), + latest_crosslink_root: Hash256::zero(), + justified_slot, + justified_block_root, + }) + } +} diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 6ea9576d8..bdb041b13 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -1,3 +1,4 @@ +mod attestation_production; mod attestation_targets; mod block_graph; pub mod block_processing; From e1698102e0655840e64f00ad0d32305d64a9c243 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 28 Jan 2019 13:04:50 +1100 Subject: [PATCH 051/107] Add attestation aggregation to BeaconChain --- .../src/attestation_aggregation.rs | 129 ++++++++++++++++++ beacon_node/beacon_chain/src/lib.rs | 5 + eth2/types/src/attestation_data/mod.rs | 5 +- 3 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 beacon_node/beacon_chain/src/attestation_aggregation.rs diff --git a/beacon_node/beacon_chain/src/attestation_aggregation.rs b/beacon_node/beacon_chain/src/attestation_aggregation.rs new file mode 100644 index 000000000..4cac5056c --- /dev/null +++ b/beacon_node/beacon_chain/src/attestation_aggregation.rs @@ -0,0 +1,129 @@ +use std::collections::{HashMap, HashSet}; +use types::{ + AggregateSignature, Attestation, AttestationData, BeaconState, Bitfield, ChainSpec, Signature, +}; + +const PHASE_0_CUSTODY_BIT: bool = false; + +pub struct AttestationAggregator { + store: HashMap, Attestation>, +} + +pub enum ProcessOutcome { + AggregationNotRequired, + Aggregated, + NewAttestationCreated, +} + +pub enum ProcessError { + BadValidatorIndex, + BadSignature, +} + +impl AttestationAggregator { + pub fn new() -> Self { + Self { + store: HashMap::new(), + } + } + + pub fn process_free_attestation( + &mut self, + state: &BeaconState, + attestation_data: &AttestationData, + signature: &Signature, + validator_index: u64, + ) -> Result { + let validator_index = validator_index as usize; + + let signable_message = attestation_data.signable_message(PHASE_0_CUSTODY_BIT); + let validator_pubkey = &state + .validator_registry + .get(validator_index) + .ok_or_else(|| ProcessError::BadValidatorIndex)? + .pubkey; + + if !signature.verify(&signable_message, &validator_pubkey) { + return Err(ProcessError::BadSignature); + } + + if let Some(existing_attestation) = self.store.get(&signable_message) { + if let Some(updated_attestation) = + aggregate_attestation(existing_attestation, signature, validator_index) + { + self.store.insert(signable_message, updated_attestation); + Ok(ProcessOutcome::Aggregated) + } else { + Ok(ProcessOutcome::AggregationNotRequired) + } + } else { + let mut aggregate_signature = AggregateSignature::new(); + aggregate_signature.add(signature); + let mut aggregation_bitfield = Bitfield::new(); + aggregation_bitfield.set(validator_index, true); + let new_attestation = Attestation { + data: attestation_data.clone(), + aggregation_bitfield, + custody_bitfield: Bitfield::new(), + aggregate_signature, + }; + self.store.insert(signable_message, new_attestation); + Ok(ProcessOutcome::NewAttestationCreated) + } + } + + /// Returns all known attestations which are: + /// + /// a) valid for the given state + /// b) not already in `state.latest_attestations`. + pub fn get_attestations_for_state( + &self, + state: &BeaconState, + spec: &ChainSpec, + ) -> Vec { + let mut known_attestation_data: HashSet = HashSet::new(); + + state.latest_attestations.iter().for_each(|attestation| { + known_attestation_data.insert(attestation.data.clone()); + }); + + self.store + .values() + .filter_map(|attestation| { + if state.validate_attestation(attestation, spec).is_ok() + && !known_attestation_data.contains(&attestation.data) + { + Some(attestation.clone()) + } else { + None + } + }) + .collect() + } +} + +fn aggregate_attestation( + existing_attestation: &Attestation, + signature: &Signature, + validator_index: usize, +) -> Option { + let already_signed = existing_attestation + .aggregation_bitfield + .get(validator_index) + .unwrap_or(false); + + if already_signed { + None + } else { + let mut aggregation_bitfield = existing_attestation.aggregation_bitfield.clone(); + aggregation_bitfield.set(validator_index, true); + let mut aggregate_signature = existing_attestation.aggregate_signature.clone(); + aggregate_signature.add(&signature); + + Some(Attestation { + aggregation_bitfield, + aggregate_signature, + ..existing_attestation.clone() + }) + } +} diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index bdb041b13..c4da9099f 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -1,3 +1,4 @@ +mod attestation_aggregation; mod attestation_production; mod attestation_targets; mod block_graph; @@ -13,6 +14,7 @@ mod state_transition; use self::attestation_targets::AttestationTargets; use self::block_graph::BlockGraph; +use attestation_aggregation::AttestationAggregator; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, ClientDB, DBError, @@ -73,6 +75,7 @@ pub struct BeaconChain { pub state_store: Arc>, pub slot_clock: U, pub block_graph: BlockGraph, + pub attestation_aggregator: RwLock, canonical_head: RwLock, finalized_head: RwLock, justified_head: RwLock, @@ -124,6 +127,7 @@ where genesis_state.clone(), state_root.clone(), )); + let attestation_aggregator = RwLock::new(AttestationAggregator::new()); let latest_attestation_targets = RwLock::new(AttestationTargets::new()); @@ -132,6 +136,7 @@ where state_store, slot_clock, block_graph, + attestation_aggregator, justified_head, finalized_head, canonical_head, diff --git a/eth2/types/src/attestation_data/mod.rs b/eth2/types/src/attestation_data/mod.rs index f22b4007f..28504127e 100644 --- a/eth2/types/src/attestation_data/mod.rs +++ b/eth2/types/src/attestation_data/mod.rs @@ -3,6 +3,7 @@ use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use std::hash::Hash; mod signing; @@ -17,7 +18,7 @@ pub const SSZ_ATTESTION_DATA_LENGTH: usize = { 32 // justified_block_root }; -#[derive(Debug, Clone, PartialEq, Default, Serialize)] +#[derive(Debug, Clone, PartialEq, Default, Serialize, Hash)] pub struct AttestationData { pub slot: u64, pub shard: u64, @@ -29,6 +30,8 @@ pub struct AttestationData { pub justified_block_root: Hash256, } +impl Eq for AttestationData {} + impl AttestationData { pub fn zero() -> Self { Self { From 5bbffcb053560bf951b465cc17a93a9d4f62267b Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 28 Jan 2019 15:50:42 +1100 Subject: [PATCH 052/107] Add attester to beacon chain test harness --- ...gregation.rs => attestation_aggregator.rs} | 2 + .../src/attestation_processing.rs | 47 ++++++++++++ .../src/attestation_production.rs | 5 -- .../beacon_chain/src/canonical_head.rs | 1 + beacon_node/beacon_chain/src/info.rs | 7 ++ beacon_node/beacon_chain/src/lib.rs | 5 +- .../beacon_chain/test_harness/Cargo.toml | 2 + .../test_harness/src/direct_beacon_node.rs | 71 ------------------- .../test_harness/src/direct_duties.rs | 38 ---------- .../{beacon_chain_harness.rs => harness.rs} | 0 .../beacon_chain/test_harness/src/lib.rs | 10 +-- .../src/validator/beacon_node/attester.rs | 40 +++++++++++ .../src/validator/beacon_node/mod.rs | 34 +++++++++ .../beacon_node/producer.rs} | 36 ++-------- .../src/validator/direct_duties.rs | 70 ++++++++++++++++++ .../src/{validator.rs => validator/mod.rs} | 25 ++++++- .../test_harness/src/validator/signer.rs | 46 ++++++++++++ eth2/attester/src/lib.rs | 2 +- eth2/attester/src/test_utils/signer.rs | 2 +- eth2/attester/src/traits.rs | 2 +- eth2/block_producer/src/lib.rs | 9 ++- eth2/block_producer/src/test_utils/signer.rs | 6 +- eth2/block_producer/src/traits.rs | 3 +- .../types/src/beacon_state/slot_processing.rs | 14 ++++ 24 files changed, 311 insertions(+), 166 deletions(-) rename beacon_node/beacon_chain/src/{attestation_aggregation.rs => attestation_aggregator.rs} (98%) create mode 100644 beacon_node/beacon_chain/src/attestation_processing.rs delete mode 100644 beacon_node/beacon_chain/test_harness/src/direct_beacon_node.rs delete mode 100644 beacon_node/beacon_chain/test_harness/src/direct_duties.rs rename beacon_node/beacon_chain/test_harness/src/{beacon_chain_harness.rs => harness.rs} (100%) create mode 100644 beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs create mode 100644 beacon_node/beacon_chain/test_harness/src/validator/beacon_node/mod.rs rename beacon_node/beacon_chain/test_harness/src/{benching_beacon_node.rs => validator/beacon_node/producer.rs} (67%) create mode 100644 beacon_node/beacon_chain/test_harness/src/validator/direct_duties.rs rename beacon_node/beacon_chain/test_harness/src/{validator.rs => validator/mod.rs} (79%) create mode 100644 beacon_node/beacon_chain/test_harness/src/validator/signer.rs diff --git a/beacon_node/beacon_chain/src/attestation_aggregation.rs b/beacon_node/beacon_chain/src/attestation_aggregator.rs similarity index 98% rename from beacon_node/beacon_chain/src/attestation_aggregation.rs rename to beacon_node/beacon_chain/src/attestation_aggregator.rs index 4cac5056c..4b83bbebc 100644 --- a/beacon_node/beacon_chain/src/attestation_aggregation.rs +++ b/beacon_node/beacon_chain/src/attestation_aggregator.rs @@ -9,12 +9,14 @@ pub struct AttestationAggregator { store: HashMap, Attestation>, } +#[derive(Debug, PartialEq)] pub enum ProcessOutcome { AggregationNotRequired, Aggregated, NewAttestationCreated, } +#[derive(Debug, PartialEq)] pub enum ProcessError { BadValidatorIndex, BadSignature, diff --git a/beacon_node/beacon_chain/src/attestation_processing.rs b/beacon_node/beacon_chain/src/attestation_processing.rs new file mode 100644 index 000000000..90107dd97 --- /dev/null +++ b/beacon_node/beacon_chain/src/attestation_processing.rs @@ -0,0 +1,47 @@ +use super::{BeaconChain, ClientDB, SlotClock}; +pub use crate::attestation_aggregator::{ProcessError as AggregatorError, ProcessOutcome}; +use crate::canonical_head::Error as HeadError; +use types::{AttestationData, Signature}; + +#[derive(Debug, PartialEq)] +pub enum Error { + PresentSlotUnknown, + AggregatorError(AggregatorError), + HeadError(HeadError), +} + +impl BeaconChain +where + T: ClientDB, + U: SlotClock, +{ + pub fn process_free_attestation( + &self, + attestation_data: &AttestationData, + signature: &Signature, + validator_index: u64, + ) -> Result { + let present_slot = self + .present_slot() + .ok_or_else(|| Error::PresentSlotUnknown)?; + let state = self.state(present_slot)?; + + self.attestation_aggregator + .write() + .expect("Aggregator unlock failed.") + .process_free_attestation(&state, attestation_data, signature, validator_index) + .map_err(|e| e.into()) + } +} + +impl From for Error { + fn from(e: AggregatorError) -> Error { + Error::AggregatorError(e) + } +} + +impl From for Error { + fn from(e: HeadError) -> Error { + Error::HeadError(e) + } +} diff --git a/beacon_node/beacon_chain/src/attestation_production.rs b/beacon_node/beacon_chain/src/attestation_production.rs index 33d220b7f..5adf70a4a 100644 --- a/beacon_node/beacon_chain/src/attestation_production.rs +++ b/beacon_node/beacon_chain/src/attestation_production.rs @@ -3,11 +3,6 @@ use types::{AttestationData, Hash256}; #[derive(Debug, PartialEq)] pub enum Error { - /* - DBError(String), - StateTransitionError(TransitionError), - PresentSlotIsNone, - */ SlotTooOld, PresentSlotUnknown, StateError, diff --git a/beacon_node/beacon_chain/src/canonical_head.rs b/beacon_node/beacon_chain/src/canonical_head.rs index 121ded767..04bbe57ed 100644 --- a/beacon_node/beacon_chain/src/canonical_head.rs +++ b/beacon_node/beacon_chain/src/canonical_head.rs @@ -2,6 +2,7 @@ use crate::{BeaconChain, CheckPoint, ClientDB, SlotClock}; use std::sync::RwLockReadGuard; use types::{beacon_state::SlotProcessingError, BeaconBlock, BeaconState, Hash256}; +#[derive(Debug, PartialEq)] pub enum Error { PastSlot, UnableToDetermineProducer, diff --git a/beacon_node/beacon_chain/src/info.rs b/beacon_node/beacon_chain/src/info.rs index 441d56b78..fd23c0906 100644 --- a/beacon_node/beacon_chain/src/info.rs +++ b/beacon_node/beacon_chain/src/info.rs @@ -52,4 +52,11 @@ where .beacon_block .slot } + + pub fn validator_attestion_slot_and_shard(&self, validator_index: usize) -> Option<(u64, u64)> { + let present_slot = self.present_slot()?; + let state = self.state(present_slot).ok()?; + + Some(state.attestation_slot_and_shard_for_validator(validator_index, &self.spec)) + } } diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index c4da9099f..401479e5a 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -1,4 +1,5 @@ -mod attestation_aggregation; +mod attestation_aggregator; +pub mod attestation_processing; mod attestation_production; mod attestation_targets; mod block_graph; @@ -14,7 +15,7 @@ mod state_transition; use self::attestation_targets::AttestationTargets; use self::block_graph::BlockGraph; -use attestation_aggregation::AttestationAggregator; +use attestation_aggregator::AttestationAggregator; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, ClientDB, DBError, diff --git a/beacon_node/beacon_chain/test_harness/Cargo.toml b/beacon_node/beacon_chain/test_harness/Cargo.toml index 72fdf7164..5aa57b0b4 100644 --- a/beacon_node/beacon_chain/test_harness/Cargo.toml +++ b/beacon_node/beacon_chain/test_harness/Cargo.toml @@ -12,11 +12,13 @@ harness = false criterion = "0.2" [dependencies] +attester = { path = "../../../eth2/attester" } beacon_chain = { path = "../../beacon_chain" } block_producer = { path = "../../../eth2/block_producer" } bls = { path = "../../../eth2/utils/bls" } boolean-bitfield = { path = "../../../eth2/utils/boolean-bitfield" } db = { path = "../../db" } +parking_lot = "0.7" failure = "0.1" failure_derive = "0.1" genesis = { path = "../../../eth2/genesis" } diff --git a/beacon_node/beacon_chain/test_harness/src/direct_beacon_node.rs b/beacon_node/beacon_chain/test_harness/src/direct_beacon_node.rs deleted file mode 100644 index 9fcd7c286..000000000 --- a/beacon_node/beacon_chain/test_harness/src/direct_beacon_node.rs +++ /dev/null @@ -1,71 +0,0 @@ -use beacon_chain::block_processing::{Error as ProcessingError, Outcome as ProcessingOutcome}; -use beacon_chain::{block_production::Error as BlockProductionError, BeaconChain}; -use block_producer::{ - BeaconNode as BeaconBlockNode, BeaconNodeError as BeaconBlockNodeError, PublishOutcome, -}; -use db::ClientDB; -use slot_clock::SlotClock; -use std::sync::Arc; -use types::{BeaconBlock, PublicKey, Signature}; - -pub struct DirectBeaconNode { - beacon_chain: Arc>, -} - -impl DirectBeaconNode { - pub fn new(beacon_chain: Arc>) -> Self { - Self { beacon_chain } - } -} - -impl BeaconBlockNode for DirectBeaconNode -where - BlockProductionError: From<::Error>, - ProcessingError: From<::Error>, -{ - fn proposer_nonce(&self, pubkey: &PublicKey) -> Result { - let validator_index = self - .beacon_chain - .validator_index(pubkey) - .ok_or_else(|| BeaconBlockNodeError::RemoteFailure("pubkey unknown.".to_string()))?; - - self.beacon_chain - .proposer_slots(validator_index) - .ok_or_else(|| { - BeaconBlockNodeError::RemoteFailure("validator_index unknown.".to_string()) - }) - } - - fn produce_beacon_block( - &self, - slot: u64, - randao_reveal: &Signature, - ) -> Result, BeaconBlockNodeError> -where { - let (block, _state) = self - .beacon_chain - .produce_block(randao_reveal.clone()) - .map_err(|e| BeaconBlockNodeError::RemoteFailure(format!("{:?}", e)))?; - - if block.slot == slot { - Ok(Some(block)) - } else { - Err(BeaconBlockNodeError::RemoteFailure( - "Unable to produce at non-current slot.".to_string(), - )) - } - } - - fn publish_beacon_block( - &self, - block: BeaconBlock, - ) -> Result { - match self.beacon_chain.process_block(block) { - Ok(ProcessingOutcome::ValidBlock(_)) => Ok(PublishOutcome::ValidBlock), - Ok(ProcessingOutcome::InvalidBlock(reason)) => { - Ok(PublishOutcome::InvalidBlock(format!("{:?}", reason))) - } - Err(error) => Err(BeaconBlockNodeError::RemoteFailure(format!("{:?}", error))), - } - } -} diff --git a/beacon_node/beacon_chain/test_harness/src/direct_duties.rs b/beacon_node/beacon_chain/test_harness/src/direct_duties.rs deleted file mode 100644 index ff2cd7223..000000000 --- a/beacon_node/beacon_chain/test_harness/src/direct_duties.rs +++ /dev/null @@ -1,38 +0,0 @@ -use beacon_chain::{block_production::Error as BlockProductionError, BeaconChain}; -use block_producer::{DutiesReader, DutiesReaderError}; -use db::ClientDB; -use slot_clock::SlotClock; -use std::sync::Arc; -use types::PublicKey; - -pub struct DirectDuties { - beacon_chain: Arc>, - pubkey: PublicKey, -} - -impl DirectDuties { - pub fn new(pubkey: PublicKey, beacon_chain: Arc>) -> Self { - Self { - beacon_chain, - pubkey, - } - } -} - -impl DutiesReader for DirectDuties -where - BlockProductionError: From<::Error>, -{ - fn is_block_production_slot(&self, slot: u64) -> Result { - let validator_index = self - .beacon_chain - .validator_index(&self.pubkey) - .ok_or_else(|| DutiesReaderError::UnknownValidator)?; - - match self.beacon_chain.block_proposer(slot) { - Some(proposer) if proposer == validator_index => Ok(true), - Some(_) => Ok(false), - None => Err(DutiesReaderError::UnknownEpoch), - } - } -} diff --git a/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs b/beacon_node/beacon_chain/test_harness/src/harness.rs similarity index 100% rename from beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs rename to beacon_node/beacon_chain/test_harness/src/harness.rs diff --git a/beacon_node/beacon_chain/test_harness/src/lib.rs b/beacon_node/beacon_chain/test_harness/src/lib.rs index efdc39e1e..71f9a8855 100644 --- a/beacon_node/beacon_chain/test_harness/src/lib.rs +++ b/beacon_node/beacon_chain/test_harness/src/lib.rs @@ -1,11 +1,5 @@ -mod beacon_chain_harness; -mod benching_beacon_node; -mod direct_beacon_node; -mod direct_duties; +mod harness; mod validator; -pub use self::beacon_chain_harness::BeaconChainHarness; -pub use self::benching_beacon_node::BenchingBeaconNode; -pub use self::direct_beacon_node::DirectBeaconNode; -pub use self::direct_duties::DirectDuties; +pub use self::harness::BeaconChainHarness; pub use self::validator::TestValidator; diff --git a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs new file mode 100644 index 000000000..71c45dd31 --- /dev/null +++ b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs @@ -0,0 +1,40 @@ +use super::BenchingBeaconNode; +use attester::{BeaconNode as AttesterBeaconNode, BeaconNodeError as NodeError, PublishOutcome}; +use beacon_chain::block_processing::Error as ProcessingError; +use beacon_chain::block_production::Error as BlockProductionError; +use db::ClientDB; +use slot_clock::SlotClock; +use types::{AttestationData, Signature}; + +impl AttesterBeaconNode for BenchingBeaconNode +where + BlockProductionError: From<::Error>, + ProcessingError: From<::Error>, +{ + fn produce_attestation_data( + &self, + slot: u64, + shard: u64, + ) -> Result, NodeError> { + match self.beacon_chain.produce_attestation_data(slot, shard) { + Ok(attestation_data) => Ok(Some(attestation_data)), + Err(e) => Err(NodeError::RemoteFailure(format!("{:?}", e))), + } + } + + fn publish_attestation_data( + &self, + attestation_data: AttestationData, + signature: Signature, + validator_index: u64, + ) -> Result { + match self.beacon_chain.process_free_attestation( + &attestation_data, + &signature, + validator_index, + ) { + Ok(_) => Ok(PublishOutcome::ValidAttestation), + Err(e) => Err(NodeError::RemoteFailure(format!("{:?}", e))), + } + } +} diff --git a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/mod.rs b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/mod.rs new file mode 100644 index 000000000..60664468d --- /dev/null +++ b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/mod.rs @@ -0,0 +1,34 @@ +use beacon_chain::BeaconChain; +use db::ClientDB; +use parking_lot::RwLock; +use slot_clock::SlotClock; +use std::sync::Arc; +use types::{AttestationData, BeaconBlock, Signature}; + +mod attester; +mod producer; + +/// An attestation that hasn't been aggregated into an `Attestation`. +/// +/// (attestation_data, signature, validator_index) +pub type FreeAttestation = (AttestationData, Signature, u64); + +pub struct BenchingBeaconNode { + beacon_chain: Arc>, + published_blocks: RwLock>, + published_attestations: RwLock>, +} + +impl BenchingBeaconNode { + pub fn new(beacon_chain: Arc>) -> Self { + Self { + beacon_chain, + published_blocks: RwLock::new(vec![]), + published_attestations: RwLock::new(vec![]), + } + } + + pub fn last_published_block(&self) -> Option { + Some(self.published_blocks.read().last()?.clone()) + } +} diff --git a/beacon_node/beacon_chain/test_harness/src/benching_beacon_node.rs b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/producer.rs similarity index 67% rename from beacon_node/beacon_chain/test_harness/src/benching_beacon_node.rs rename to beacon_node/beacon_chain/test_harness/src/validator/beacon_node/producer.rs index 551fbff99..70ca4ff0c 100644 --- a/beacon_node/beacon_chain/test_harness/src/benching_beacon_node.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/producer.rs @@ -1,37 +1,13 @@ +use super::BenchingBeaconNode; use beacon_chain::block_processing::Error as ProcessingError; -use beacon_chain::{block_production::Error as BlockProductionError, BeaconChain}; +use beacon_chain::block_production::Error as BlockProductionError; use block_producer::{ BeaconNode as BeaconBlockNode, BeaconNodeError as BeaconBlockNodeError, PublishOutcome, }; use db::ClientDB; use slot_clock::SlotClock; -use std::sync::{Arc, RwLock}; use types::{BeaconBlock, PublicKey, Signature}; -pub struct BenchingBeaconNode { - beacon_chain: Arc>, - published_blocks: RwLock>, -} - -impl BenchingBeaconNode { - pub fn new(beacon_chain: Arc>) -> Self { - Self { - beacon_chain, - published_blocks: RwLock::new(vec![]), - } - } - - pub fn last_published_block(&self) -> Option { - Some( - self.published_blocks - .read() - .expect("Unable to unlock `published_blocks` for reading.") - .last()? - .clone(), - ) - } -} - impl BeaconBlockNode for BenchingBeaconNode where BlockProductionError: From<::Error>, @@ -56,8 +32,7 @@ where &self, slot: u64, randao_reveal: &Signature, - ) -> Result, BeaconBlockNodeError> -where { + ) -> Result, BeaconBlockNodeError> { let (block, _state) = self .beacon_chain .produce_block(randao_reveal.clone()) @@ -81,10 +56,7 @@ where { &self, block: BeaconBlock, ) -> Result { - self.published_blocks - .write() - .expect("Unable to unlock `published_blocks` for writing.") - .push(block); + self.published_blocks.write().push(block); Ok(PublishOutcome::ValidBlock) } } diff --git a/beacon_node/beacon_chain/test_harness/src/validator/direct_duties.rs b/beacon_node/beacon_chain/test_harness/src/validator/direct_duties.rs new file mode 100644 index 000000000..3259817ad --- /dev/null +++ b/beacon_node/beacon_chain/test_harness/src/validator/direct_duties.rs @@ -0,0 +1,70 @@ +use attester::{ + DutiesReader as AttesterDutiesReader, DutiesReaderError as AttesterDutiesReaderError, +}; +use beacon_chain::{block_production::Error as BlockProductionError, BeaconChain}; +use block_producer::{ + DutiesReader as ProducerDutiesReader, DutiesReaderError as ProducerDutiesReaderError, +}; +use db::ClientDB; +use slot_clock::SlotClock; +use std::sync::Arc; +use types::PublicKey; + +pub struct DirectDuties { + beacon_chain: Arc>, + pubkey: PublicKey, +} + +impl DirectDuties { + pub fn new(pubkey: PublicKey, beacon_chain: Arc>) -> Self { + Self { + beacon_chain, + pubkey, + } + } +} + +impl ProducerDutiesReader for DirectDuties +where + BlockProductionError: From<::Error>, +{ + fn is_block_production_slot(&self, slot: u64) -> Result { + let validator_index = self + .beacon_chain + .validator_index(&self.pubkey) + .ok_or_else(|| ProducerDutiesReaderError::UnknownValidator)?; + + match self.beacon_chain.block_proposer(slot) { + Some(proposer) if proposer == validator_index => Ok(true), + Some(_) => Ok(false), + None => Err(ProducerDutiesReaderError::UnknownEpoch), + } + } +} + +impl AttesterDutiesReader for DirectDuties +where + BlockProductionError: From<::Error>, +{ + fn validator_index(&self) -> Option { + match self.beacon_chain.validator_index(&self.pubkey) { + Some(index) => Some(index as u64), + None => None, + } + } + + fn attestation_shard(&self, slot: u64) -> Result, AttesterDutiesReaderError> { + if let Some(validator_index) = self.validator_index() { + match self + .beacon_chain + .validator_attestion_slot_and_shard(validator_index as usize) + { + Some((attest_slot, attest_shard)) if attest_slot == slot => Ok(Some(attest_shard)), + Some(_) => Ok(None), + None => Err(AttesterDutiesReaderError::UnknownEpoch), + } + } else { + Err(AttesterDutiesReaderError::UnknownValidator) + } + } +} diff --git a/beacon_node/beacon_chain/test_harness/src/validator.rs b/beacon_node/beacon_chain/test_harness/src/validator/mod.rs similarity index 79% rename from beacon_node/beacon_chain/test_harness/src/validator.rs rename to beacon_node/beacon_chain/test_harness/src/validator/mod.rs index 1d7f7bd4a..f2ac96075 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/mod.rs @@ -1,11 +1,18 @@ -use super::{BenchingBeaconNode, DirectDuties}; +use attester::Attester; use beacon_chain::BeaconChain; -use block_producer::{test_utils::TestSigner, BlockProducer, Error as PollError}; +use block_producer::{BlockProducer, Error as PollError}; use db::MemoryDB; +use signer::TestSigner; use slot_clock::TestingSlotClock; use std::sync::Arc; use types::{BeaconBlock, ChainSpec, Keypair}; +mod beacon_node; +mod direct_duties; +mod signer; + +pub use self::beacon_node::BenchingBeaconNode; +pub use self::direct_duties::DirectDuties; pub use block_producer::PollOutcome; #[derive(Debug, PartialEq)] @@ -21,6 +28,12 @@ pub struct TestValidator { DirectDuties, TestSigner, >, + pub attester: Attester< + TestingSlotClock, + BenchingBeaconNode, + DirectDuties, + TestSigner, + >, pub spec: Arc, pub epoch_map: Arc>, pub keypair: Keypair, @@ -49,8 +62,16 @@ impl TestValidator { signer.clone(), ); + let attester = Attester::new( + epoch_map.clone(), + slot_clock.clone(), + beacon_node.clone(), + signer.clone(), + ); + Self { block_producer, + attester, spec, epoch_map, keypair, diff --git a/beacon_node/beacon_chain/test_harness/src/validator/signer.rs b/beacon_node/beacon_chain/test_harness/src/validator/signer.rs new file mode 100644 index 000000000..572a870ff --- /dev/null +++ b/beacon_node/beacon_chain/test_harness/src/validator/signer.rs @@ -0,0 +1,46 @@ +use attester::Signer as AttesterSigner; +use block_producer::Signer as BlockProposerSigner; +use std::sync::RwLock; +use types::{Keypair, Signature}; + +/// A test-only struct used to simulate a Beacon Node. +pub struct TestSigner { + keypair: Keypair, + should_sign: RwLock, +} + +impl TestSigner { + /// Produce a new TestSigner with signing enabled by default. + pub fn new(keypair: Keypair) -> Self { + Self { + keypair, + should_sign: RwLock::new(true), + } + } + + /// If set to `false`, the service will refuse to sign all messages. Otherwise, all messages + /// will be signed. + pub fn enable_signing(&self, enabled: bool) { + *self.should_sign.write().unwrap() = enabled; + } + + fn bls_sign(&self, message: &[u8]) -> Option { + Some(Signature::new(message, &self.keypair.sk)) + } +} + +impl BlockProposerSigner for TestSigner { + fn sign_block_proposal(&self, message: &[u8]) -> Option { + self.bls_sign(message) + } + + fn sign_randao_reveal(&self, message: &[u8]) -> Option { + self.bls_sign(message) + } +} + +impl AttesterSigner for TestSigner { + fn sign_attestation_message(&self, message: &[u8]) -> Option { + self.bls_sign(message) + } +} diff --git a/eth2/attester/src/lib.rs b/eth2/attester/src/lib.rs index 7e3f0eb6b..604762634 100644 --- a/eth2/attester/src/lib.rs +++ b/eth2/attester/src/lib.rs @@ -131,7 +131,7 @@ impl Attester Option { + fn sign_attestation_message(&self, message: &[u8]) -> Option { Some(Signature::new(message, &self.keypair.sk)) } } diff --git a/eth2/attester/src/traits.rs b/eth2/attester/src/traits.rs index bc3ccf63f..ce6a635e7 100644 --- a/eth2/attester/src/traits.rs +++ b/eth2/attester/src/traits.rs @@ -47,5 +47,5 @@ pub trait DutiesReader: Send + Sync { /// Signs message using an internally-maintained private key. pub trait Signer { - fn bls_sign(&self, message: &[u8]) -> Option; + fn sign_attestation_message(&self, message: &[u8]) -> Option; } diff --git a/eth2/block_producer/src/lib.rs b/eth2/block_producer/src/lib.rs index 4ea08647d..ed65622d4 100644 --- a/eth2/block_producer/src/lib.rs +++ b/eth2/block_producer/src/lib.rs @@ -4,7 +4,7 @@ mod traits; use slot_clock::SlotClock; use ssz::ssz_encode; use std::sync::Arc; -use types::{BeaconBlock, ChainSpec, Hash256, ProposalSignedData, PublicKey}; +use types::{BeaconBlock, ChainSpec, PublicKey}; pub use self::traits::{ BeaconNode, BeaconNodeError, DutiesReader, DutiesReaderError, PublishOutcome, Signer, @@ -139,7 +139,7 @@ impl BlockProducer return Ok(PollOutcome::SignerRejection(slot)), Some(signature) => signature, } @@ -171,7 +171,10 @@ impl BlockProducer Option { self.store_produce(&block); - match self.signer.bls_sign(&block.proposal_root(&self.spec)[..]) { + match self + .signer + .sign_block_proposal(&block.proposal_root(&self.spec)[..]) + { None => None, Some(signature) => { block.signature = signature; diff --git a/eth2/block_producer/src/test_utils/signer.rs b/eth2/block_producer/src/test_utils/signer.rs index d43c0feb0..62163f5b2 100644 --- a/eth2/block_producer/src/test_utils/signer.rs +++ b/eth2/block_producer/src/test_utils/signer.rs @@ -25,7 +25,11 @@ impl TestSigner { } impl Signer for TestSigner { - fn bls_sign(&self, message: &[u8]) -> Option { + fn sign_block_proposal(&self, message: &[u8]) -> Option { + Some(Signature::new(message, &self.keypair.sk)) + } + + fn sign_randao_reveal(&self, message: &[u8]) -> Option { Some(Signature::new(message, &self.keypair.sk)) } } diff --git a/eth2/block_producer/src/traits.rs b/eth2/block_producer/src/traits.rs index 451ce6cc7..b09b81e67 100644 --- a/eth2/block_producer/src/traits.rs +++ b/eth2/block_producer/src/traits.rs @@ -47,5 +47,6 @@ pub trait DutiesReader: Send + Sync { /// Signs message using an internally-maintained private key. pub trait Signer { - fn bls_sign(&self, message: &[u8]) -> Option; + fn sign_block_proposal(&self, message: &[u8]) -> Option; + fn sign_randao_reveal(&self, message: &[u8]) -> Option; } diff --git a/eth2/types/src/beacon_state/slot_processing.rs b/eth2/types/src/beacon_state/slot_processing.rs index e3be9ccee..2070d57cb 100644 --- a/eth2/types/src/beacon_state/slot_processing.rs +++ b/eth2/types/src/beacon_state/slot_processing.rs @@ -38,6 +38,20 @@ impl BeaconState { let validator_count = self.validator_registry.len(); Some((slot as usize) % validator_count) } + + pub fn attestation_slot_and_shard_for_validator( + &self, + validator_index: usize, + spec: &ChainSpec, + ) -> (u64, u64) { + // TODO: this is a stub; implement it properly. + let validator_index = validator_index as u64; + + let slot = validator_index % spec.epoch_length; + let shard = validator_index % spec.shard_count; + + (slot, shard) + } } fn merkle_root(_input: &[Hash256]) -> Hash256 { From be7e326c33a5d219e1c08b4c788405f092ea3a0c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 28 Jan 2019 16:21:33 +1100 Subject: [PATCH 053/107] Add FreeAttesation type --- .../src/attestation_aggregator.rs | 28 +++++++------ .../src/attestation_processing.rs | 8 ++-- .../src/validator/beacon_node/attester.rs | 12 ++---- .../src/validator/beacon_node/mod.rs | 11 +++-- .../test_harness/src/validator/mod.rs | 42 +++++++++++++------ .../beacon_chain/test_harness/tests/chain.rs | 14 +++---- eth2/attester/src/lib.rs | 10 ++++- eth2/attester/src/test_utils/beacon_node.rs | 13 ++---- eth2/attester/src/traits.rs | 6 +-- eth2/types/src/free_attestation.rs | 12 ++++++ eth2/types/src/lib.rs | 2 + 11 files changed, 92 insertions(+), 66 deletions(-) create mode 100644 eth2/types/src/free_attestation.rs diff --git a/beacon_node/beacon_chain/src/attestation_aggregator.rs b/beacon_node/beacon_chain/src/attestation_aggregator.rs index 4b83bbebc..a67923e13 100644 --- a/beacon_node/beacon_chain/src/attestation_aggregator.rs +++ b/beacon_node/beacon_chain/src/attestation_aggregator.rs @@ -1,6 +1,7 @@ use std::collections::{HashMap, HashSet}; use types::{ - AggregateSignature, Attestation, AttestationData, BeaconState, Bitfield, ChainSpec, Signature, + AggregateSignature, Attestation, AttestationData, BeaconState, Bitfield, ChainSpec, + FreeAttestation, Signature, }; const PHASE_0_CUSTODY_BIT: bool = false; @@ -32,27 +33,30 @@ impl AttestationAggregator { pub fn process_free_attestation( &mut self, state: &BeaconState, - attestation_data: &AttestationData, - signature: &Signature, - validator_index: u64, + free_attestation: &FreeAttestation, ) -> Result { - let validator_index = validator_index as usize; + let validator_index = free_attestation.validator_index as usize; - let signable_message = attestation_data.signable_message(PHASE_0_CUSTODY_BIT); + let signable_message = free_attestation.data.signable_message(PHASE_0_CUSTODY_BIT); let validator_pubkey = &state .validator_registry .get(validator_index) .ok_or_else(|| ProcessError::BadValidatorIndex)? .pubkey; - if !signature.verify(&signable_message, &validator_pubkey) { + if !free_attestation + .signature + .verify(&signable_message, &validator_pubkey) + { return Err(ProcessError::BadSignature); } if let Some(existing_attestation) = self.store.get(&signable_message) { - if let Some(updated_attestation) = - aggregate_attestation(existing_attestation, signature, validator_index) - { + if let Some(updated_attestation) = aggregate_attestation( + existing_attestation, + &free_attestation.signature, + validator_index, + ) { self.store.insert(signable_message, updated_attestation); Ok(ProcessOutcome::Aggregated) } else { @@ -60,11 +64,11 @@ impl AttestationAggregator { } } else { let mut aggregate_signature = AggregateSignature::new(); - aggregate_signature.add(signature); + aggregate_signature.add(&free_attestation.signature); let mut aggregation_bitfield = Bitfield::new(); aggregation_bitfield.set(validator_index, true); let new_attestation = Attestation { - data: attestation_data.clone(), + data: free_attestation.data.clone(), aggregation_bitfield, custody_bitfield: Bitfield::new(), aggregate_signature, diff --git a/beacon_node/beacon_chain/src/attestation_processing.rs b/beacon_node/beacon_chain/src/attestation_processing.rs index 90107dd97..8704cd489 100644 --- a/beacon_node/beacon_chain/src/attestation_processing.rs +++ b/beacon_node/beacon_chain/src/attestation_processing.rs @@ -1,7 +1,7 @@ use super::{BeaconChain, ClientDB, SlotClock}; pub use crate::attestation_aggregator::{ProcessError as AggregatorError, ProcessOutcome}; use crate::canonical_head::Error as HeadError; -use types::{AttestationData, Signature}; +use types::FreeAttestation; #[derive(Debug, PartialEq)] pub enum Error { @@ -17,9 +17,7 @@ where { pub fn process_free_attestation( &self, - attestation_data: &AttestationData, - signature: &Signature, - validator_index: u64, + free_attestation: FreeAttestation, ) -> Result { let present_slot = self .present_slot() @@ -29,7 +27,7 @@ where self.attestation_aggregator .write() .expect("Aggregator unlock failed.") - .process_free_attestation(&state, attestation_data, signature, validator_index) + .process_free_attestation(&state, &free_attestation) .map_err(|e| e.into()) } } diff --git a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs index 71c45dd31..4ef0efe36 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs @@ -4,7 +4,7 @@ use beacon_chain::block_processing::Error as ProcessingError; use beacon_chain::block_production::Error as BlockProductionError; use db::ClientDB; use slot_clock::SlotClock; -use types::{AttestationData, Signature}; +use types::{AttestationData, FreeAttestation}; impl AttesterBeaconNode for BenchingBeaconNode where @@ -24,15 +24,9 @@ where fn publish_attestation_data( &self, - attestation_data: AttestationData, - signature: Signature, - validator_index: u64, + free_attestation: FreeAttestation, ) -> Result { - match self.beacon_chain.process_free_attestation( - &attestation_data, - &signature, - validator_index, - ) { + match self.beacon_chain.process_free_attestation(free_attestation) { Ok(_) => Ok(PublishOutcome::ValidAttestation), Err(e) => Err(NodeError::RemoteFailure(format!("{:?}", e))), } diff --git a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/mod.rs b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/mod.rs index 60664468d..caed2bc5f 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/mod.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/mod.rs @@ -3,16 +3,11 @@ use db::ClientDB; use parking_lot::RwLock; use slot_clock::SlotClock; use std::sync::Arc; -use types::{AttestationData, BeaconBlock, Signature}; +use types::{BeaconBlock, FreeAttestation}; mod attester; mod producer; -/// An attestation that hasn't been aggregated into an `Attestation`. -/// -/// (attestation_data, signature, validator_index) -pub type FreeAttestation = (AttestationData, Signature, u64); - pub struct BenchingBeaconNode { beacon_chain: Arc>, published_blocks: RwLock>, @@ -31,4 +26,8 @@ impl BenchingBeaconNode { pub fn last_published_block(&self) -> Option { Some(self.published_blocks.read().last()?.clone()) } + + pub fn last_published_free_attestation(&self) -> Option { + Some(self.published_attestations.read().last()?.clone()) + } } diff --git a/beacon_node/beacon_chain/test_harness/src/validator/mod.rs b/beacon_node/beacon_chain/test_harness/src/validator/mod.rs index f2ac96075..3f1b10d5e 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/mod.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/mod.rs @@ -1,24 +1,30 @@ -use attester::Attester; +use attester::{Attester, Error as AttestationPollError}; use beacon_chain::BeaconChain; -use block_producer::{BlockProducer, Error as PollError}; +use block_producer::{BlockProducer, Error as BlockPollError}; use db::MemoryDB; use signer::TestSigner; use slot_clock::TestingSlotClock; use std::sync::Arc; -use types::{BeaconBlock, ChainSpec, Keypair}; - +use types::{BeaconBlock, ChainSpec, FreeAttestation, Keypair}; mod beacon_node; mod direct_duties; mod signer; pub use self::beacon_node::BenchingBeaconNode; pub use self::direct_duties::DirectDuties; -pub use block_producer::PollOutcome; +pub use attester::PollOutcome as AttestationPollOutcome; +pub use block_producer::PollOutcome as BlockPollOutcome; #[derive(Debug, PartialEq)] -pub enum ProduceError { - DidNotProduce(PollOutcome), - PollError(PollError), +pub enum BlockProduceError { + DidNotProduce(BlockPollOutcome), + PollError(BlockPollError), +} + +#[derive(Debug, PartialEq)] +pub enum AttestationProduceError { + DidNotProduce(AttestationPollOutcome), + PollError(AttestationPollError), } pub struct TestValidator { @@ -81,13 +87,13 @@ impl TestValidator { } } - pub fn produce_block(&mut self) -> Result { + pub fn produce_block(&mut self) -> Result { // Using `BenchingBeaconNode`, the validator will always return sucessufully if it tries to // publish a block. match self.block_producer.poll() { - Ok(PollOutcome::BlockProduced(_)) => {} - Ok(outcome) => return Err(ProduceError::DidNotProduce(outcome)), - Err(error) => return Err(ProduceError::PollError(error)), + Ok(BlockPollOutcome::BlockProduced(_)) => {} + Ok(outcome) => return Err(BlockProduceError::DidNotProduce(outcome)), + Err(error) => return Err(BlockProduceError::PollError(error)), }; Ok(self .beacon_node @@ -95,6 +101,18 @@ impl TestValidator { .expect("Unable to obtain produced block.")) } + pub fn produce_free_attestation(&mut self) -> Result { + match self.attester.poll() { + Ok(AttestationPollOutcome::AttestationProduced(_)) => {} + Ok(outcome) => return Err(AttestationProduceError::DidNotProduce(outcome)), + Err(error) => return Err(AttestationProduceError::PollError(error)), + }; + Ok(self + .beacon_node + .last_published_free_attestation() + .expect("Unable to obtain produced attestation.")) + } + pub fn set_slot(&mut self, slot: u64) { self.slot_clock.set_slot(slot) } diff --git a/beacon_node/beacon_chain/test_harness/tests/chain.rs b/beacon_node/beacon_chain/test_harness/tests/chain.rs index 19bdf233f..bac1e1347 100644 --- a/beacon_node/beacon_chain/test_harness/tests/chain.rs +++ b/beacon_node/beacon_chain/test_harness/tests/chain.rs @@ -4,25 +4,25 @@ use types::ChainSpec; #[test] fn it_can_build_on_genesis_block() { let validator_count = 2; - let mut rig = BeaconChainHarness::new(ChainSpec::foundation(), validator_count); + let mut harness = BeaconChainHarness::new(ChainSpec::foundation(), validator_count); - rig.advance_chain_with_block(); + harness.advance_chain_with_block(); } #[test] #[ignore] fn it_can_produce_past_first_epoch_boundary() { let validator_count = 2; - let mut rig = BeaconChainHarness::new(ChainSpec::foundation(), validator_count); + let mut harness = BeaconChainHarness::new(ChainSpec::foundation(), validator_count); - let blocks = rig.spec.epoch_length + 1; + let blocks = harness.spec.epoch_length + 1; for _ in 0..blocks { - rig.advance_chain_with_block(); + harness.advance_chain_with_block(); } - let dump = rig.chain_dump().expect("Chain dump failed."); + let dump = harness.chain_dump().expect("Chain dump failed."); assert_eq!(dump.len() as u64, blocks + 1); // + 1 for genesis block. - rig.dump_to_file("/tmp/chaindump.json".to_string(), &dump); + harness.dump_to_file("/tmp/chaindump.json".to_string(), &dump); } diff --git a/eth2/attester/src/lib.rs b/eth2/attester/src/lib.rs index 604762634..447bcb9ea 100644 --- a/eth2/attester/src/lib.rs +++ b/eth2/attester/src/lib.rs @@ -3,7 +3,7 @@ mod traits; use slot_clock::SlotClock; use std::sync::Arc; -use types::{AttestationData, Signature}; +use types::{AttestationData, FreeAttestation, Signature}; pub use self::traits::{ BeaconNode, BeaconNodeError, DutiesReader, DutiesReaderError, PublishOutcome, Signer, @@ -111,8 +111,14 @@ impl Attester return Ok(PollOutcome::ValidatorIsUnknown(slot)), }; + let free_attestation = FreeAttestation { + data: attestation_data, + signature, + validator_index, + }; + self.beacon_node - .publish_attestation_data(attestation_data, signature, validator_index)?; + .publish_attestation_data(free_attestation)?; Ok(PollOutcome::AttestationProduced(slot)) } diff --git a/eth2/attester/src/test_utils/beacon_node.rs b/eth2/attester/src/test_utils/beacon_node.rs index 4ebcbdf5a..5b4061c60 100644 --- a/eth2/attester/src/test_utils/beacon_node.rs +++ b/eth2/attester/src/test_utils/beacon_node.rs @@ -1,6 +1,6 @@ use crate::traits::{BeaconNode, BeaconNodeError, PublishOutcome}; use std::sync::RwLock; -use types::{AttestationData, Signature}; +use types::{AttestationData, FreeAttestation}; type ProduceResult = Result, BeaconNodeError>; type PublishResult = Result; @@ -11,7 +11,7 @@ pub struct TestBeaconNode { pub produce_input: RwLock>, pub produce_result: RwLock>, - pub publish_input: RwLock>, + pub publish_input: RwLock>, pub publish_result: RwLock>, } @@ -34,13 +34,8 @@ impl BeaconNode for TestBeaconNode { } } - fn publish_attestation_data( - &self, - attestation_data: AttestationData, - signature: Signature, - validator_index: u64, - ) -> PublishResult { - *self.publish_input.write().unwrap() = Some((attestation_data, signature, validator_index)); + fn publish_attestation_data(&self, free_attestation: FreeAttestation) -> PublishResult { + *self.publish_input.write().unwrap() = Some(free_attestation.clone()); match *self.publish_result.read().unwrap() { Some(ref r) => r.clone(), None => panic!("TestBeaconNode: publish_result == None"), diff --git a/eth2/attester/src/traits.rs b/eth2/attester/src/traits.rs index ce6a635e7..fd07fd171 100644 --- a/eth2/attester/src/traits.rs +++ b/eth2/attester/src/traits.rs @@ -1,4 +1,4 @@ -use types::{AttestationData, Signature}; +use types::{AttestationData, FreeAttestation, Signature}; #[derive(Debug, PartialEq, Clone)] pub enum BeaconNodeError { @@ -22,9 +22,7 @@ pub trait BeaconNode: Send + Sync { fn publish_attestation_data( &self, - attestation_data: AttestationData, - signature: Signature, - validator_index: u64, + free_attestation: FreeAttestation, ) -> Result; } diff --git a/eth2/types/src/free_attestation.rs b/eth2/types/src/free_attestation.rs new file mode 100644 index 000000000..16d4f6728 --- /dev/null +++ b/eth2/types/src/free_attestation.rs @@ -0,0 +1,12 @@ +/// Note: this object does not actually exist in the spec. +/// +/// We use it for managing attestations that have not been aggregated. +use super::{AttestationData, Signature}; +use serde_derive::Serialize; + +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct FreeAttestation { + pub data: AttestationData, + pub signature: Signature, + pub validator_index: u64, +} diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index 3bf13e2d4..79dd18255 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -15,6 +15,7 @@ pub mod eth1_data; pub mod eth1_data_vote; pub mod exit; pub mod fork; +pub mod free_attestation; pub mod pending_attestation; pub mod proposal_signed_data; pub mod proposer_slashing; @@ -47,6 +48,7 @@ pub use crate::eth1_data::Eth1Data; pub use crate::eth1_data_vote::Eth1DataVote; pub use crate::exit::Exit; pub use crate::fork::Fork; +pub use crate::free_attestation::FreeAttestation; pub use crate::pending_attestation::PendingAttestation; pub use crate::proposal_signed_data::ProposalSignedData; pub use crate::proposer_slashing::ProposerSlashing; From 288211052520d5b8959ef45f5d1aeeef1e5037d9 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 28 Jan 2019 17:07:13 +1100 Subject: [PATCH 054/107] Add attesatation aggregation to test harness --- .../beacon_chain/src/block_production.rs | 10 ++- .../beacon_chain/test_harness/src/harness.rs | 64 +++++++++++++++---- .../src/validator/beacon_node/attester.rs | 6 +- .../beacon_chain/test_harness/tests/chain.rs | 2 +- 4 files changed, 61 insertions(+), 21 deletions(-) diff --git a/beacon_node/beacon_chain/src/block_production.rs b/beacon_node/beacon_chain/src/block_production.rs index 50baf12ba..58f3ae2cc 100644 --- a/beacon_node/beacon_chain/src/block_production.rs +++ b/beacon_node/beacon_chain/src/block_production.rs @@ -45,6 +45,13 @@ where .into_beacon_state() .ok_or_else(|| Error::DBError("State invalid.".to_string()))?; + let attestations = self + .attestation_aggregator + .read() + .unwrap() + // TODO: advance the parent_state slot. + .get_attestations_for_state(&parent_state, &self.spec); + let mut block = BeaconBlock { slot: present_slot, parent_root: parent_root.clone(), @@ -59,7 +66,7 @@ where body: BeaconBlockBody { proposer_slashings: vec![], casper_slashings: vec![], - attestations: vec![], + attestations: attestations, custody_reseeds: vec![], custody_challenges: vec![], custody_responses: vec![], @@ -70,7 +77,6 @@ where let state = self.state_transition_without_verifying_block_signature(parent_state, &block)?; - let state_root = state.canonical_root(); block.state_root = state_root; diff --git a/beacon_node/beacon_chain/test_harness/src/harness.rs b/beacon_node/beacon_chain/test_harness/src/harness.rs index 2f4c11aef..b96636c8d 100644 --- a/beacon_node/beacon_chain/test_harness/src/harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/harness.rs @@ -9,7 +9,7 @@ use slot_clock::TestingSlotClock; use std::fs::File; use std::io::prelude::*; use std::sync::Arc; -use types::{BeaconBlock, ChainSpec, Keypair, Validator}; +use types::{BeaconBlock, ChainSpec, FreeAttestation, Keypair, Validator}; pub struct BeaconChainHarness { pub db: Arc, @@ -74,33 +74,69 @@ impl BeaconChainHarness { } } - pub fn advance_chain_without_block(&mut self) -> BeaconBlock { - self.produce_next_slot() - } - - pub fn advance_chain_with_block(&mut self) { - let block = self.produce_next_slot(); - self.beacon_chain.process_block(block).unwrap(); - } - - fn produce_next_slot(&mut self) -> BeaconBlock { + /// Move the `slot_clock` for the `BeaconChain` forward one slot. + /// + /// This is the equivalent of advancing a system clock forward one `SLOT_DURATION`. + pub fn increment_beacon_chain_slot(&mut self) { let slot = self .beacon_chain .present_slot() .expect("Unable to determine slot.") + 1; - self.beacon_chain.slot_clock.set_slot(slot); + } + + /// Gather the `FreeAttestation`s from the valiators. + /// + /// Note: validators will only produce attestations _once per slot_. So, if you call this twice + /// you'll only get attestations on the first run. + pub fn gather_free_attesations(&mut self) -> Vec { + let present_slot = self.beacon_chain.present_slot().unwrap(); + + let mut free_attestations = vec![]; + for validator in &mut self.validators { + // Advance the validator slot. + validator.set_slot(present_slot); + + // Prompt the validator to produce an attestation (if required). + if let Ok(free_attestation) = validator.produce_free_attestation() { + free_attestations.push(free_attestation); + } + } + free_attestations + } + + /// Get the block from the proposer for the slot. + /// + /// Note: the validator will only produce it _once per slot_. So, if you call this twice you'll + /// only get a block once. + pub fn produce_block(&mut self) -> BeaconBlock { + let present_slot = self.beacon_chain.present_slot().unwrap(); let proposer = self .beacon_chain - .block_proposer(slot) + .block_proposer(present_slot) .expect("Unable to determine proposer."); - self.validators[proposer].set_slot(slot); self.validators[proposer].produce_block().unwrap() } + /// Advances the chain with a BeaconBlock and attestations from all validators. + /// + /// This is the ideal scenario for the Beacon Chain, 100% honest participation from + /// validators. + pub fn advance_chain_with_block(&mut self) { + self.increment_beacon_chain_slot(); + let free_attestations = self.gather_free_attesations(); + for free_attestation in free_attestations { + self.beacon_chain + .process_free_attestation(free_attestation.clone()) + .unwrap(); + } + let block = self.produce_block(); + self.beacon_chain.process_block(block).unwrap(); + } + pub fn chain_dump(&self) -> Result, DumpError> { self.beacon_chain.chain_dump() } diff --git a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs index 4ef0efe36..9c6ce7456 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs @@ -26,9 +26,7 @@ where &self, free_attestation: FreeAttestation, ) -> Result { - match self.beacon_chain.process_free_attestation(free_attestation) { - Ok(_) => Ok(PublishOutcome::ValidAttestation), - Err(e) => Err(NodeError::RemoteFailure(format!("{:?}", e))), - } + self.published_attestations.write().push(free_attestation); + Ok(PublishOutcome::ValidAttestation) } } diff --git a/beacon_node/beacon_chain/test_harness/tests/chain.rs b/beacon_node/beacon_chain/test_harness/tests/chain.rs index bac1e1347..bbe52ac0d 100644 --- a/beacon_node/beacon_chain/test_harness/tests/chain.rs +++ b/beacon_node/beacon_chain/test_harness/tests/chain.rs @@ -12,7 +12,7 @@ fn it_can_build_on_genesis_block() { #[test] #[ignore] fn it_can_produce_past_first_epoch_boundary() { - let validator_count = 2; + let validator_count = 100; let mut harness = BeaconChainHarness::new(ChainSpec::foundation(), validator_count); let blocks = harness.spec.epoch_length + 1; From f92b9d618a29de5403258dabbbaf3df67f68ac66 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 28 Jan 2019 17:45:54 +1100 Subject: [PATCH 055/107] Add `get_shuffling` to state --- eth2/types/Cargo.toml | 2 + .../src/beacon_state/epoch_processing.rs | 23 --------- eth2/types/src/beacon_state/mod.rs | 1 + eth2/types/src/beacon_state/shuffling.rs | 47 +++++++++++++++++++ 4 files changed, 50 insertions(+), 23 deletions(-) create mode 100644 eth2/types/src/beacon_state/shuffling.rs diff --git a/eth2/types/Cargo.toml b/eth2/types/Cargo.toml index a553e4240..3da5a1d26 100644 --- a/eth2/types/Cargo.toml +++ b/eth2/types/Cargo.toml @@ -9,9 +9,11 @@ bls = { path = "../utils/bls" } boolean-bitfield = { path = "../utils/boolean-bitfield" } ethereum-types = "0.4.0" hashing = { path = "../utils/hashing" } +honey-badger-split = { path = "../utils/honey-badger-split" } integer-sqrt = "0.1" rand = "0.5.5" serde = "1.0" serde_derive = "1.0" serde_json = "1.0" ssz = { path = "../utils/ssz" } +vec_shuffle = { path = "../utils/vec_shuffle" } diff --git a/eth2/types/src/beacon_state/epoch_processing.rs b/eth2/types/src/beacon_state/epoch_processing.rs index ac6ccbb04..948da15d7 100644 --- a/eth2/types/src/beacon_state/epoch_processing.rs +++ b/eth2/types/src/beacon_state/epoch_processing.rs @@ -717,22 +717,6 @@ impl BeaconState { self.get_committee_count_per_slot(current_active_validators.len(), spec) } - fn get_committee_count_per_slot( - &self, - active_validator_count: usize, - spec: &ChainSpec, - ) -> usize { - std::cmp::max( - 1, - std::cmp::min( - spec.shard_count as usize / spec.epoch_length as usize, - active_validator_count - / spec.epoch_length as usize - / spec.target_committee_size as usize, - ), - ) - } - fn process_ejections(&self) { //TODO: stubbed out. } @@ -784,10 +768,6 @@ impl BeaconState { self.get_effective_balance(validator_index, spec) / base_reward_quotient / 5 } - pub fn get_crosslink_committees_at_slot(&self, _slot: u64) -> Option { - Some(vec![(vec![0], 0)]) - } - pub fn get_effective_balance(&self, validator_index: usize, spec: &ChainSpec) -> u64 { std::cmp::min(self.validator_balances[validator_index], spec.max_deposit) } @@ -810,6 +790,3 @@ impl BeaconState { vec![0, 1] } } - -type CrosslinkCommittee = (Vec, usize); -type CrosslinkCommittees = Vec; diff --git a/eth2/types/src/beacon_state/mod.rs b/eth2/types/src/beacon_state/mod.rs index b287c0c7f..0032f275f 100644 --- a/eth2/types/src/beacon_state/mod.rs +++ b/eth2/types/src/beacon_state/mod.rs @@ -12,6 +12,7 @@ use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; mod attestation_validation; mod epoch_processing; +mod shuffling; mod slot_processing; mod winning_root; diff --git a/eth2/types/src/beacon_state/shuffling.rs b/eth2/types/src/beacon_state/shuffling.rs new file mode 100644 index 000000000..eb93947f3 --- /dev/null +++ b/eth2/types/src/beacon_state/shuffling.rs @@ -0,0 +1,47 @@ +use crate::{validator_registry::get_active_validator_indices, BeaconState, ChainSpec, Hash256}; +use honey_badger_split::SplitExt; +use vec_shuffle::shuffle; + +type CrosslinkCommittee = (Vec, usize); +type CrosslinkCommittees = Vec; + +impl BeaconState { + pub fn get_shuffling(&self, seed: Hash256, slot: u64, spec: &ChainSpec) -> Vec> { + let slot = slot - (slot % spec.epoch_length); + + let active_validator_indices = get_active_validator_indices(&self.validator_registry, slot); + + let committees_per_slot = + self.get_committee_count_per_slot(active_validator_indices.len(), spec); + + // TODO: check that Hash256 matches 'int_to_bytes32'. + let seed = seed ^ Hash256::from(slot); + let shuffled_active_validator_indices = + shuffle(&seed, active_validator_indices).expect("Max validator count exceed!"); + + shuffled_active_validator_indices + .honey_badger_split(committees_per_slot * spec.epoch_length as usize) + .filter_map(|slice: &[usize]| Some(slice.to_vec())) + .collect() + } + + pub fn get_committee_count_per_slot( + &self, + active_validator_count: usize, + spec: &ChainSpec, + ) -> usize { + std::cmp::max( + 1, + std::cmp::min( + spec.shard_count as usize / spec.epoch_length as usize, + active_validator_count + / spec.epoch_length as usize + / spec.target_committee_size as usize, + ), + ) + } + + pub fn get_crosslink_committees_at_slot(&self, _slot: u64) -> Option { + Some(vec![(vec![0], 0)]) + } +} From 6a4252b8c641d2b59a186fccdd8735a18cc8d92e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 28 Jan 2019 19:12:20 +1100 Subject: [PATCH 056/107] Add state helpers from #148 --- beacon_node/beacon_chain/src/info.rs | 26 +++- .../beacon_chain/src/state_transition.rs | 4 +- .../beacon_chain/test_harness/src/harness.rs | 5 +- .../src/validator/direct_duties.rs | 6 +- .../beacon_chain/test_harness/tests/chain.rs | 5 +- eth2/genesis/src/beacon_state.rs | 4 +- .../src/beacon_state/epoch_processing.rs | 25 ++-- eth2/types/src/beacon_state/mod.rs | 30 +++-- eth2/types/src/beacon_state/shuffling.rs | 117 ++++++++++++++++-- .../types/src/beacon_state/slot_processing.rs | 10 +- 10 files changed, 167 insertions(+), 65 deletions(-) diff --git a/beacon_node/beacon_chain/src/info.rs b/beacon_node/beacon_chain/src/info.rs index fd23c0906..27b844004 100644 --- a/beacon_node/beacon_chain/src/info.rs +++ b/beacon_node/beacon_chain/src/info.rs @@ -1,5 +1,11 @@ use super::{BeaconChain, ClientDB, SlotClock}; -use types::PublicKey; +use types::{beacon_state::Error as BeaconStateError, PublicKey}; + +#[derive(Debug, PartialEq)] +pub enum Error { + SlotClockError, + BeaconStateError(BeaconStateError), +} impl BeaconChain where @@ -39,10 +45,14 @@ where } } - pub fn block_proposer(&self, slot: u64) -> Option { - let present_slot = self.present_slot()?; - let state = self.state(present_slot).ok()?; - state.get_beacon_proposer_index(slot, &self.spec) + pub fn block_proposer(&self, slot: u64) -> Result { + // TODO: fix unwrap + let present_slot = self.present_slot().unwrap(); + // TODO: fix unwrap + let state = self.state(present_slot).unwrap(); + let index = state.get_beacon_proposer_index(slot, &self.spec)?; + + Ok(index) } pub fn justified_slot(&self) -> u64 { @@ -60,3 +70,9 @@ where Some(state.attestation_slot_and_shard_for_validator(validator_index, &self.spec)) } } + +impl From for Error { + fn from(e: BeaconStateError) -> Error { + Error::BeaconStateError(e) + } +} diff --git a/beacon_node/beacon_chain/src/state_transition.rs b/beacon_node/beacon_chain/src/state_transition.rs index 199bf8210..0d51ab676 100644 --- a/beacon_node/beacon_chain/src/state_transition.rs +++ b/beacon_node/beacon_chain/src/state_transition.rs @@ -98,7 +98,7 @@ where let block_proposer_index = state .get_beacon_proposer_index(block.slot, &self.spec) - .ok_or(Error::NoBlockProducer)?; + .map_err(|_| Error::NoBlockProducer)?; let block_proposer = &state.validator_registry[block_proposer_index]; if verify_block_signature { @@ -294,7 +294,7 @@ where ); if state.slot % self.spec.epoch_length == 0 { - state.per_epoch_processing(&self.spec); + state.per_epoch_processing(&self.spec).unwrap(); } Ok(state) diff --git a/beacon_node/beacon_chain/test_harness/src/harness.rs b/beacon_node/beacon_chain/test_harness/src/harness.rs index b96636c8d..3982fd61e 100644 --- a/beacon_node/beacon_chain/test_harness/src/harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/harness.rs @@ -113,10 +113,7 @@ impl BeaconChainHarness { pub fn produce_block(&mut self) -> BeaconBlock { let present_slot = self.beacon_chain.present_slot().unwrap(); - let proposer = self - .beacon_chain - .block_proposer(present_slot) - .expect("Unable to determine proposer."); + let proposer = self.beacon_chain.block_proposer(present_slot).unwrap(); self.validators[proposer].produce_block().unwrap() } diff --git a/beacon_node/beacon_chain/test_harness/src/validator/direct_duties.rs b/beacon_node/beacon_chain/test_harness/src/validator/direct_duties.rs index 3259817ad..eddc0d177 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/direct_duties.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/direct_duties.rs @@ -35,9 +35,9 @@ where .ok_or_else(|| ProducerDutiesReaderError::UnknownValidator)?; match self.beacon_chain.block_proposer(slot) { - Some(proposer) if proposer == validator_index => Ok(true), - Some(_) => Ok(false), - None => Err(ProducerDutiesReaderError::UnknownEpoch), + Ok(proposer) if proposer == validator_index => Ok(true), + Ok(_) => Ok(false), + Err(_) => Err(ProducerDutiesReaderError::UnknownEpoch), } } } diff --git a/beacon_node/beacon_chain/test_harness/tests/chain.rs b/beacon_node/beacon_chain/test_harness/tests/chain.rs index bbe52ac0d..c312dcf43 100644 --- a/beacon_node/beacon_chain/test_harness/tests/chain.rs +++ b/beacon_node/beacon_chain/test_harness/tests/chain.rs @@ -3,8 +3,9 @@ use types::ChainSpec; #[test] fn it_can_build_on_genesis_block() { - let validator_count = 2; - let mut harness = BeaconChainHarness::new(ChainSpec::foundation(), validator_count); + let validator_count = 10; + let spec = ChainSpec::foundation(); + let mut harness = BeaconChainHarness::new(spec, validator_count); harness.advance_chain_with_block(); } diff --git a/eth2/genesis/src/beacon_state.rs b/eth2/genesis/src/beacon_state.rs index 6e697af65..7d7cc7665 100644 --- a/eth2/genesis/src/beacon_state.rs +++ b/eth2/genesis/src/beacon_state.rs @@ -55,8 +55,8 @@ pub fn genesis_beacon_state(spec: &ChainSpec) -> Result { current_epoch_start_shard: spec.genesis_start_shard, previous_epoch_calculation_slot: spec.genesis_slot, current_epoch_calculation_slot: spec.genesis_slot, - previous_epoch_randao_mix: spec.zero_hash, - current_epoch_randao_mix: spec.zero_hash, + previous_epoch_seed: spec.zero_hash, + current_epoch_seed: spec.zero_hash, /* * Custody challenges */ diff --git a/eth2/types/src/beacon_state/epoch_processing.rs b/eth2/types/src/beacon_state/epoch_processing.rs index 948da15d7..d9c1e2313 100644 --- a/eth2/types/src/beacon_state/epoch_processing.rs +++ b/eth2/types/src/beacon_state/epoch_processing.rs @@ -7,6 +7,7 @@ use integer_sqrt::IntegerSquareRoot; use std::collections::{HashMap, HashSet}; use std::iter::FromIterator; +#[derive(Debug, PartialEq)] pub enum Error { UnableToDetermineProducer, NoBlockRoots, @@ -303,8 +304,8 @@ impl BeaconState { for slot in self.slot.saturating_sub(2 * spec.epoch_length)..self.slot { let crosslink_committees_at_slot = self - .get_crosslink_committees_at_slot(slot) - .ok_or_else(|| Error::UnableToGetCrosslinkCommittees)?; + .get_crosslink_committees_at_slot(slot, spec) + .map_err(|_| Error::UnableToGetCrosslinkCommittees)?; for (crosslink_committee, shard) in crosslink_committees_at_slot { let shard = shard as u64; @@ -454,7 +455,7 @@ impl BeaconState { }; let proposer_index = self .get_beacon_proposer_index(inclusion_slot, spec) - .ok_or_else(|| Error::UnableToDetermineProducer)?; + .map_err(|_| Error::UnableToDetermineProducer)?; let base_reward = self.base_reward(proposer_index, base_reward_quotient, spec); safe_add_assign!( self.validator_balances[proposer_index], @@ -467,8 +468,8 @@ impl BeaconState { */ for slot in self.slot.saturating_sub(2 * spec.epoch_length)..self.slot { let crosslink_committees_at_slot = self - .get_crosslink_committees_at_slot(slot) - .ok_or_else(|| Error::UnableToGetCrosslinkCommittees)?; + .get_crosslink_committees_at_slot(slot, spec) + .map_err(|_| Error::UnableToGetCrosslinkCommittees)?; for (_crosslink_committee, shard) in crosslink_committees_at_slot { let shard = shard as u64; @@ -514,7 +515,7 @@ impl BeaconState { */ self.previous_epoch_calculation_slot = self.current_epoch_calculation_slot; self.previous_epoch_start_shard = self.current_epoch_start_shard; - self.previous_epoch_randao_mix = self.current_epoch_randao_mix; + self.previous_epoch_seed = self.current_epoch_seed; let should_update_validator_registy = if self.finalized_slot > self.validator_registry_update_slot @@ -534,7 +535,7 @@ impl BeaconState { self.current_epoch_start_shard = (self.current_epoch_start_shard + self.get_current_epoch_committee_count_per_slot(spec) as u64 * spec.epoch_length) % spec.shard_count; - self.current_epoch_randao_mix = self.get_randao_mix( + self.current_epoch_seed = self.get_randao_mix( self.current_epoch_calculation_slot .saturating_sub(spec.seed_lookahead), spec, @@ -544,7 +545,7 @@ impl BeaconState { (self.slot - self.validator_registry_update_slot) / spec.epoch_length; if epochs_since_last_registry_change.is_power_of_two() { self.current_epoch_calculation_slot = self.slot; - self.current_epoch_randao_mix = self.get_randao_mix( + self.current_epoch_seed = self.get_randao_mix( self.current_epoch_calculation_slot .saturating_sub(spec.seed_lookahead), spec, @@ -709,14 +710,6 @@ impl BeaconState { (slot - slot % spec.epoch_length) + spec.epoch_length + spec.entry_exit_delay } - fn get_current_epoch_committee_count_per_slot(&self, spec: &ChainSpec) -> usize { - let current_active_validators = get_active_validator_indices( - &self.validator_registry, - self.current_epoch_calculation_slot, - ); - self.get_committee_count_per_slot(current_active_validators.len(), spec) - } - fn process_ejections(&self) { //TODO: stubbed out. } diff --git a/eth2/types/src/beacon_state/mod.rs b/eth2/types/src/beacon_state/mod.rs index 0032f275f..85397ee11 100644 --- a/eth2/types/src/beacon_state/mod.rs +++ b/eth2/types/src/beacon_state/mod.rs @@ -20,6 +20,12 @@ pub use self::attestation_validation::Error as AttestationValidationError; pub use self::epoch_processing::Error as EpochProcessingError; pub use self::slot_processing::Error as SlotProcessingError; +#[derive(Debug, PartialEq)] +pub enum Error { + InvalidSlot, + InsufficientNumberOfValidators, +} + // Custody will not be added to the specs until Phase 1 (Sharding Phase) so dummy class used. type CustodyChallenge = usize; @@ -44,8 +50,8 @@ pub struct BeaconState { pub current_epoch_start_shard: u64, pub previous_epoch_calculation_slot: u64, pub current_epoch_calculation_slot: u64, - pub previous_epoch_randao_mix: Hash256, - pub current_epoch_randao_mix: Hash256, + pub previous_epoch_seed: Hash256, + pub current_epoch_seed: Hash256, // Custody challenges pub custody_challenges: Vec, @@ -90,8 +96,8 @@ impl Encodable for BeaconState { s.append(&self.current_epoch_start_shard); s.append(&self.previous_epoch_calculation_slot); s.append(&self.current_epoch_calculation_slot); - s.append(&self.previous_epoch_randao_mix); - s.append(&self.current_epoch_randao_mix); + s.append(&self.previous_epoch_seed); + s.append(&self.current_epoch_seed); s.append(&self.custody_challenges); s.append(&self.previous_justified_slot); s.append(&self.justified_slot); @@ -123,8 +129,8 @@ impl Decodable for BeaconState { let (current_epoch_start_shard, i) = <_>::ssz_decode(bytes, i)?; let (previous_epoch_calculation_slot, i) = <_>::ssz_decode(bytes, i)?; let (current_epoch_calculation_slot, i) = <_>::ssz_decode(bytes, i)?; - let (previous_epoch_randao_mix, i) = <_>::ssz_decode(bytes, i)?; - let (current_epoch_randao_mix, i) = <_>::ssz_decode(bytes, i)?; + let (previous_epoch_seed, i) = <_>::ssz_decode(bytes, i)?; + let (current_epoch_seed, i) = <_>::ssz_decode(bytes, i)?; let (custody_challenges, i) = <_>::ssz_decode(bytes, i)?; let (previous_justified_slot, i) = <_>::ssz_decode(bytes, i)?; let (justified_slot, i) = <_>::ssz_decode(bytes, i)?; @@ -154,8 +160,8 @@ impl Decodable for BeaconState { current_epoch_start_shard, previous_epoch_calculation_slot, current_epoch_calculation_slot, - previous_epoch_randao_mix, - current_epoch_randao_mix, + previous_epoch_seed, + current_epoch_seed, custody_challenges, previous_justified_slot, justified_slot, @@ -191,8 +197,8 @@ impl TreeHash for BeaconState { result.append(&mut self.current_epoch_start_shard.hash_tree_root()); result.append(&mut self.previous_epoch_calculation_slot.hash_tree_root()); result.append(&mut self.current_epoch_calculation_slot.hash_tree_root()); - result.append(&mut self.previous_epoch_randao_mix.hash_tree_root()); - result.append(&mut self.current_epoch_randao_mix.hash_tree_root()); + result.append(&mut self.previous_epoch_seed.hash_tree_root()); + result.append(&mut self.current_epoch_seed.hash_tree_root()); result.append(&mut self.custody_challenges.hash_tree_root()); result.append(&mut self.previous_justified_slot.hash_tree_root()); result.append(&mut self.justified_slot.hash_tree_root()); @@ -226,8 +232,8 @@ impl TestRandom for BeaconState { current_epoch_start_shard: <_>::random_for_test(rng), previous_epoch_calculation_slot: <_>::random_for_test(rng), current_epoch_calculation_slot: <_>::random_for_test(rng), - previous_epoch_randao_mix: <_>::random_for_test(rng), - current_epoch_randao_mix: <_>::random_for_test(rng), + previous_epoch_seed: <_>::random_for_test(rng), + current_epoch_seed: <_>::random_for_test(rng), custody_challenges: <_>::random_for_test(rng), previous_justified_slot: <_>::random_for_test(rng), justified_slot: <_>::random_for_test(rng), diff --git a/eth2/types/src/beacon_state/shuffling.rs b/eth2/types/src/beacon_state/shuffling.rs index eb93947f3..95b402f8a 100644 --- a/eth2/types/src/beacon_state/shuffling.rs +++ b/eth2/types/src/beacon_state/shuffling.rs @@ -1,9 +1,15 @@ +use super::Error; use crate::{validator_registry::get_active_validator_indices, BeaconState, ChainSpec, Hash256}; use honey_badger_split::SplitExt; +use std::ops::Range; use vec_shuffle::shuffle; -type CrosslinkCommittee = (Vec, usize); -type CrosslinkCommittees = Vec; +// utility function pending this functionality being stabilized on the `Range` type. +fn range_contains(range: &Range, target: T) -> bool { + range.start <= target && target < range.end +} + +type Result = std::result::Result; impl BeaconState { pub fn get_shuffling(&self, seed: Hash256, slot: u64, spec: &ChainSpec) -> Vec> { @@ -20,7 +26,7 @@ impl BeaconState { shuffle(&seed, active_validator_indices).expect("Max validator count exceed!"); shuffled_active_validator_indices - .honey_badger_split(committees_per_slot * spec.epoch_length as usize) + .honey_badger_split((committees_per_slot * spec.epoch_length) as usize) .filter_map(|slice: &[usize]| Some(slice.to_vec())) .collect() } @@ -29,19 +35,110 @@ impl BeaconState { &self, active_validator_count: usize, spec: &ChainSpec, - ) -> usize { + ) -> u64 { std::cmp::max( 1, std::cmp::min( - spec.shard_count as usize / spec.epoch_length as usize, - active_validator_count - / spec.epoch_length as usize - / spec.target_committee_size as usize, + spec.shard_count / spec.epoch_length, + active_validator_count as u64 / spec.epoch_length / spec.target_committee_size, ), ) } - pub fn get_crosslink_committees_at_slot(&self, _slot: u64) -> Option { - Some(vec![(vec![0], 0)]) + /// Returns the start slot and end slot of the current epoch containing `self.slot`. + fn get_current_epoch_boundaries(&self, epoch_length: u64) -> Range { + let slot_in_epoch = self.slot % epoch_length; + let start = self.slot - slot_in_epoch; + let end = self.slot + (epoch_length - slot_in_epoch); + start..end + } + + fn get_previous_epoch_committee_count_per_slot( + &self, + spec: &ChainSpec, + /* + shard_count: u64, + epoch_length: u64, + target_committee_size: u64, + */ + ) -> u64 { + let previous_active_validators = get_active_validator_indices( + &self.validator_registry, + self.previous_epoch_calculation_slot, + ); + self.get_committee_count_per_slot(previous_active_validators.len(), spec) as u64 + } + + pub fn get_current_epoch_committee_count_per_slot(&self, spec: &ChainSpec) -> u64 { + let current_active_validators = get_active_validator_indices( + &self.validator_registry, + self.current_epoch_calculation_slot, + ); + self.get_committee_count_per_slot(current_active_validators.len(), spec) + } + + pub fn get_crosslink_committees_at_slot( + &self, + slot: u64, + spec: &ChainSpec, + /* + epoch_length: u64, + shard_count: u64, + target_committee_size: u64, + */ + ) -> Result, u64)>> { + let current_epoch_range = self.get_current_epoch_boundaries(spec.epoch_length); + if !range_contains(¤t_epoch_range, slot) { + return Err(Error::InvalidSlot); + } + let state_epoch_slot = current_epoch_range.start; + let offset = slot % spec.epoch_length; + + let (committees_per_slot, shuffling, slot_start_shard) = if slot < state_epoch_slot { + let committees_per_slot = self.get_previous_epoch_committee_count_per_slot(spec); + let shuffling = self.get_shuffling( + self.previous_epoch_seed, + self.previous_epoch_calculation_slot, + spec, + ); + let slot_start_shard = + (self.previous_epoch_start_shard + committees_per_slot * offset) % spec.shard_count; + (committees_per_slot, shuffling, slot_start_shard) + } else { + let committees_per_slot = self.get_current_epoch_committee_count_per_slot(spec); + let shuffling = self.get_shuffling( + self.current_epoch_seed, + self.current_epoch_calculation_slot, + spec, + ); + let slot_start_shard = + (self.current_epoch_start_shard + committees_per_slot * offset) % spec.shard_count; + (committees_per_slot, shuffling, slot_start_shard) + }; + + let shard_range = slot_start_shard..; + Ok(shuffling + .into_iter() + .skip((committees_per_slot * offset) as usize) + .zip(shard_range.into_iter()) + .take(committees_per_slot as usize) + .map(|(committees, shard_number)| (committees, shard_number % spec.shard_count)) + .collect::>()) + } + + /// Returns the beacon proposer index for the `slot`. + /// If the state does not contain an index for a beacon proposer at the requested `slot`, then `None` is returned. + pub fn get_beacon_proposer_index(&self, slot: u64, spec: &ChainSpec) -> Result { + let committees = self.get_crosslink_committees_at_slot(slot, spec)?; + committees + .first() + .ok_or(Error::InsufficientNumberOfValidators) + .and_then(|(first_committee, _)| { + let index = (slot as usize) + .checked_rem(first_committee.len()) + .ok_or(Error::InsufficientNumberOfValidators)?; + // NOTE: next index will not panic as we have already returned if this is the case + Ok(first_committee[index]) + }) } } diff --git a/eth2/types/src/beacon_state/slot_processing.rs b/eth2/types/src/beacon_state/slot_processing.rs index 2070d57cb..5dc351dae 100644 --- a/eth2/types/src/beacon_state/slot_processing.rs +++ b/eth2/types/src/beacon_state/slot_processing.rs @@ -14,7 +14,7 @@ impl BeaconState { let block_proposer = self .get_beacon_proposer_index(self.slot, spec) - .ok_or_else(|| Error::UnableToDetermineProducer)?; + .map_err(|_| Error::UnableToDetermineProducer)?; self.validator_registry[block_proposer].proposer_slots += 1; self.latest_randao_mixes[(self.slot % spec.latest_randao_mixes_length) as usize] = @@ -31,14 +31,6 @@ impl BeaconState { Ok(()) } - pub fn get_beacon_proposer_index(&self, slot: u64, spec: &ChainSpec) -> Option { - // TODO: this is a stub; implement it properly. - // - // https://github.com/sigp/lighthouse/pull/148/files - let validator_count = self.validator_registry.len(); - Some((slot as usize) % validator_count) - } - pub fn attestation_slot_and_shard_for_validator( &self, validator_index: usize, From 7d94cfb0e499042c69a462fc7405654ee56687c4 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 31 Jan 2019 00:39:34 +1100 Subject: [PATCH 057/107] Refine state transition to allow first transition --- beacon_node/beacon_chain/Cargo.toml | 2 + .../src/attestation_aggregator.rs | 4 +- .../beacon_chain/src/block_processing.rs | 3 + .../beacon_chain/src/block_production.rs | 9 + .../beacon_chain/src/canonical_head.rs | 12 +- .../beacon_chain/src/epoch_processing.rs | 9 - beacon_node/beacon_chain/src/info.rs | 16 +- beacon_node/beacon_chain/src/lib.rs | 1 - .../beacon_chain/src/state_transition.rs | 46 +- .../beacon_chain/test_harness/Cargo.toml | 3 + .../beacon_chain/test_harness/src/harness.rs | 86 ++-- .../test_harness/src/validator/mod.rs | 5 +- .../beacon_chain/test_harness/tests/chain.rs | 27 +- eth2/types/Cargo.toml | 1 + .../beacon_state/attestation_participants.rs | 88 ++++ .../beacon_state/attestation_validation.rs | 72 +++- eth2/types/src/beacon_state/committees.rs | 127 ++++++ .../src/beacon_state/epoch_processing.rs | 399 +++++++++--------- eth2/types/src/beacon_state/mod.rs | 11 +- eth2/types/src/beacon_state/shuffling.rs | 113 +---- .../types/src/beacon_state/slot_processing.rs | 26 +- eth2/types/src/beacon_state/winning_root.rs | 56 ++- 22 files changed, 680 insertions(+), 436 deletions(-) delete mode 100644 beacon_node/beacon_chain/src/epoch_processing.rs create mode 100644 eth2/types/src/beacon_state/attestation_participants.rs create mode 100644 eth2/types/src/beacon_state/committees.rs diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index e12476553..7e6dc209d 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -13,6 +13,8 @@ failure = "0.1" failure_derive = "0.1" genesis = { path = "../../eth2/genesis" } hashing = { path = "../../eth2/utils/hashing" } +log = "0.4" +env_logger = "0.6" serde = "1.0" serde_derive = "1.0" serde_json = "1.0" diff --git a/beacon_node/beacon_chain/src/attestation_aggregator.rs b/beacon_node/beacon_chain/src/attestation_aggregator.rs index a67923e13..c95feb1e1 100644 --- a/beacon_node/beacon_chain/src/attestation_aggregator.rs +++ b/beacon_node/beacon_chain/src/attestation_aggregator.rs @@ -96,7 +96,9 @@ impl AttestationAggregator { self.store .values() .filter_map(|attestation| { - if state.validate_attestation(attestation, spec).is_ok() + if state + .validate_attestation_without_signature(attestation, spec) + .is_ok() && !known_attestation_data.contains(&attestation.data) { Some(attestation.clone()) diff --git a/beacon_node/beacon_chain/src/block_processing.rs b/beacon_node/beacon_chain/src/block_processing.rs index 89c5bfc14..b4126ed9e 100644 --- a/beacon_node/beacon_chain/src/block_processing.rs +++ b/beacon_node/beacon_chain/src/block_processing.rs @@ -1,5 +1,6 @@ use super::state_transition::Error as TransitionError; use super::{BeaconChain, ClientDB, DBError, SlotClock}; +use log::debug; use slot_clock::{SystemTimeSlotClockError, TestingSlotClockError}; use ssz::{ssz_encode, Encodable}; use types::{ @@ -65,6 +66,8 @@ where where V: BeaconBlockReader + Encodable + Sized, { + debug!("Processing block with slot {}...", block.slot()); + let block = block .into_beacon_block() .ok_or(Error::UnableToDecodeBlock)?; diff --git a/beacon_node/beacon_chain/src/block_production.rs b/beacon_node/beacon_chain/src/block_production.rs index 58f3ae2cc..289bc485d 100644 --- a/beacon_node/beacon_chain/src/block_production.rs +++ b/beacon_node/beacon_chain/src/block_production.rs @@ -1,6 +1,7 @@ use super::state_transition::Error as TransitionError; use super::{BeaconChain, ClientDB, DBError, SlotClock}; use bls::Signature; +use log::debug; use slot_clock::TestingSlotClockError; use types::{ readers::{BeaconBlockReader, BeaconStateReader}, @@ -33,6 +34,8 @@ where .map_err(|e| e.into())? .ok_or(Error::PresentSlotIsNone)?; + debug!("Producing block for slot {}...", present_slot); + let parent_root = self.head().beacon_block_root; let parent_block_reader = self .block_store @@ -45,6 +48,8 @@ where .into_beacon_state() .ok_or_else(|| Error::DBError("State invalid.".to_string()))?; + debug!("Finding attesatations for block..."); + let attestations = self .attestation_aggregator .read() @@ -52,6 +57,8 @@ where // TODO: advance the parent_state slot. .get_attestations_for_state(&parent_state, &self.spec); + debug!("Found {} attestation(s).", attestations.len()); + let mut block = BeaconBlock { slot: present_slot, parent_root: parent_root.clone(), @@ -81,6 +88,8 @@ where block.state_root = state_root; + debug!("Block produced."); + Ok((block, state)) } } diff --git a/beacon_node/beacon_chain/src/canonical_head.rs b/beacon_node/beacon_chain/src/canonical_head.rs index 04bbe57ed..4bcac5b64 100644 --- a/beacon_node/beacon_chain/src/canonical_head.rs +++ b/beacon_node/beacon_chain/src/canonical_head.rs @@ -1,11 +1,11 @@ use crate::{BeaconChain, CheckPoint, ClientDB, SlotClock}; use std::sync::RwLockReadGuard; -use types::{beacon_state::SlotProcessingError, BeaconBlock, BeaconState, Hash256}; +use types::{beacon_state::CommitteesError, BeaconBlock, BeaconState, Hash256}; #[derive(Debug, PartialEq)] pub enum Error { PastSlot, - UnableToDetermineProducer, + CommitteesError(CommitteesError), } impl BeaconChain @@ -64,10 +64,8 @@ where } } -impl From for Error { - fn from(e: SlotProcessingError) -> Error { - match e { - SlotProcessingError::UnableToDetermineProducer => Error::UnableToDetermineProducer, - } +impl From for Error { + fn from(e: CommitteesError) -> Error { + Error::CommitteesError(e) } } diff --git a/beacon_node/beacon_chain/src/epoch_processing.rs b/beacon_node/beacon_chain/src/epoch_processing.rs deleted file mode 100644 index 405ba7df2..000000000 --- a/beacon_node/beacon_chain/src/epoch_processing.rs +++ /dev/null @@ -1,9 +0,0 @@ -use super::{BeaconChain, ClientDB, DBError, SlotClock}; - -impl BeaconChain -where - T: ClientDB, - U: SlotClock, -{ - pub fn per_epoch_processing(&self) {} -} diff --git a/beacon_node/beacon_chain/src/info.rs b/beacon_node/beacon_chain/src/info.rs index 27b844004..1ecd97c1e 100644 --- a/beacon_node/beacon_chain/src/info.rs +++ b/beacon_node/beacon_chain/src/info.rs @@ -1,10 +1,10 @@ use super::{BeaconChain, ClientDB, SlotClock}; -use types::{beacon_state::Error as BeaconStateError, PublicKey}; +use types::{beacon_state::CommitteesError, PublicKey}; #[derive(Debug, PartialEq)] pub enum Error { SlotClockError, - BeaconStateError(BeaconStateError), + CommitteesError(CommitteesError), } impl BeaconChain @@ -45,7 +45,7 @@ where } } - pub fn block_proposer(&self, slot: u64) -> Result { + pub fn block_proposer(&self, slot: u64) -> Result { // TODO: fix unwrap let present_slot = self.present_slot().unwrap(); // TODO: fix unwrap @@ -67,12 +67,14 @@ where let present_slot = self.present_slot()?; let state = self.state(present_slot).ok()?; - Some(state.attestation_slot_and_shard_for_validator(validator_index, &self.spec)) + state + .attestation_slot_and_shard_for_validator(validator_index, &self.spec) + .ok() } } -impl From for Error { - fn from(e: BeaconStateError) -> Error { - Error::BeaconStateError(e) +impl From for Error { + fn from(e: CommitteesError) -> Error { + Error::CommitteesError(e) } } diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 401479e5a..5fdadd34f 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -7,7 +7,6 @@ pub mod block_processing; pub mod block_production; mod canonical_head; pub mod dump; -pub mod epoch_processing; mod finalized_head; mod info; mod lmd_ghost; diff --git a/beacon_node/beacon_chain/src/state_transition.rs b/beacon_node/beacon_chain/src/state_transition.rs index 0d51ab676..fc29a63c4 100644 --- a/beacon_node/beacon_chain/src/state_transition.rs +++ b/beacon_node/beacon_chain/src/state_transition.rs @@ -1,13 +1,13 @@ use super::{BeaconChain, ClientDB, DBError, SlotClock}; use bls::{PublicKey, Signature}; -use boolean_bitfield::BooleanBitfield; use hashing::hash; +use log::debug; use slot_clock::{SystemTimeSlotClockError, TestingSlotClockError}; use ssz::{ssz_encode, TreeHash}; use types::{ - beacon_state::{AttestationValidationError, SlotProcessingError}, + beacon_state::{AttestationValidationError, CommitteesError, EpochProcessingError}, readers::BeaconBlockReader, - AttestationData, BeaconBlock, BeaconState, Exit, Fork, Hash256, PendingAttestation, + BeaconBlock, BeaconState, Exit, Fork, Hash256, PendingAttestation, }; // TODO: define elsehwere. @@ -51,6 +51,8 @@ pub enum Error { BadCustodyChallenges, BadCustodyResponses, SlotClockError(SystemTimeSlotClockError), + CommitteesError(CommitteesError), + EpochProcessingError(EpochProcessingError), } impl BeaconChain @@ -82,6 +84,11 @@ where ) -> Result { ensure!(state.slot < block.slot, Error::StateAlreadyTransitioned); + debug!( + "Starting state transition from slot {} to {}...", + state.slot, block.slot + ); + for _ in state.slot..block.slot { state.per_slot_processing(block.parent_root.clone(), &self.spec)?; } @@ -113,6 +120,8 @@ where ); } + debug!("Block signature is valid."); + /* * RANDAO */ @@ -127,6 +136,8 @@ where Error::BadRandaoSignature ); + debug!("RANDAO signature is valid."); + // TODO: check this is correct. let new_mix = { let mut mix = state.latest_randao_mixes @@ -228,6 +239,11 @@ where state.latest_attestations.push(pending_attestation); } + debug!( + "{} attestations verified & processed.", + block.body.attestations.len() + ); + /* * Deposits */ @@ -294,9 +310,11 @@ where ); if state.slot % self.spec.epoch_length == 0 { - state.per_epoch_processing(&self.spec).unwrap(); + state.per_epoch_processing(&self.spec)?; } + debug!("State transition complete."); + Ok(state) } } @@ -337,16 +355,20 @@ impl From for Error { } } -impl From for Error { - fn from(e: SlotProcessingError) -> Error { - match e { - SlotProcessingError::UnableToDetermineProducer => Error::NoBlockProducer, - } - } -} - impl From for Error { fn from(e: AttestationValidationError) -> Error { Error::InvalidAttestation(e) } } + +impl From for Error { + fn from(e: CommitteesError) -> Error { + Error::CommitteesError(e) + } +} + +impl From for Error { + fn from(e: EpochProcessingError) -> Error { + Error::EpochProcessingError(e) + } +} diff --git a/beacon_node/beacon_chain/test_harness/Cargo.toml b/beacon_node/beacon_chain/test_harness/Cargo.toml index 5aa57b0b4..ce32b94c6 100644 --- a/beacon_node/beacon_chain/test_harness/Cargo.toml +++ b/beacon_node/beacon_chain/test_harness/Cargo.toml @@ -23,6 +23,9 @@ failure = "0.1" failure_derive = "0.1" genesis = { path = "../../../eth2/genesis" } hashing = { path = "../../../eth2/utils/hashing" } +log = "0.4" +env_logger = "0.6.0" +rayon = "1.0" serde = "1.0" serde_derive = "1.0" serde_json = "1.0" diff --git a/beacon_node/beacon_chain/test_harness/src/harness.rs b/beacon_node/beacon_chain/test_harness/src/harness.rs index 3982fd61e..a6bb6a02c 100644 --- a/beacon_node/beacon_chain/test_harness/src/harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/harness.rs @@ -5,6 +5,8 @@ use db::{ stores::{BeaconBlockStore, BeaconStateStore}, MemoryDB, }; +use log::debug; +use rayon::prelude::*; use slot_clock::TestingSlotClock; use std::fs::File; use std::io::prelude::*; @@ -26,26 +28,40 @@ impl BeaconChainHarness { let block_store = Arc::new(BeaconBlockStore::new(db.clone())); let state_store = Arc::new(BeaconStateStore::new(db.clone())); - let slot_clock = TestingSlotClock::new(0); + let slot_clock = TestingSlotClock::new(spec.genesis_slot); // Remove the validators present in the spec (if any). spec.initial_validators = Vec::with_capacity(validator_count); spec.initial_balances = Vec::with_capacity(validator_count); - // Insert `validator_count` new `Validator` records into the spec, retaining the keypairs - // for later user. - let mut keypairs = Vec::with_capacity(validator_count); - for _ in 0..validator_count { - let keypair = Keypair::random(); + debug!("Generating validator keypairs..."); - spec.initial_validators.push(Validator { + let keypairs: Vec = (0..validator_count) + .collect::>() + .par_iter() + .map(|_| Keypair::random()) + .collect(); + + debug!("Creating validator records..."); + + spec.initial_validators = keypairs + .par_iter() + .map(|keypair| Validator { pubkey: keypair.pk.clone(), + activation_slot: 0, ..std::default::Default::default() - }); - spec.initial_balances.push(32_000_000_000); // 32 ETH + }) + .collect(); - keypairs.push(keypair); - } + debug!("Setting validator balances..."); + + spec.initial_balances = spec + .initial_validators + .par_iter() + .map(|_| 32_000_000_000) // 32 ETH + .collect(); + + debug!("Creating the BeaconChain..."); // Create the Beacon Chain let beacon_chain = Arc::new( @@ -58,11 +74,15 @@ impl BeaconChainHarness { .unwrap(), ); + debug!("Creating validator producer and attester instances..."); + // Spawn the test validator instances. - let mut validators = Vec::with_capacity(validator_count); - for keypair in keypairs { - validators.push(TestValidator::new(keypair.clone(), beacon_chain.clone())); - } + let validators: Vec = keypairs + .par_iter() + .map(|keypair| TestValidator::new(keypair.clone(), beacon_chain.clone(), &spec)) + .collect(); + + debug!("Created {} TestValidators", validators.len()); Self { db, @@ -83,6 +103,9 @@ impl BeaconChainHarness { .present_slot() .expect("Unable to determine slot.") + 1; + + debug!("Incrementing BeaconChain slot to {}.", slot); + self.beacon_chain.slot_clock.set_slot(slot); } @@ -93,16 +116,24 @@ impl BeaconChainHarness { pub fn gather_free_attesations(&mut self) -> Vec { let present_slot = self.beacon_chain.present_slot().unwrap(); - let mut free_attestations = vec![]; - for validator in &mut self.validators { - // Advance the validator slot. - validator.set_slot(present_slot); + let free_attestations: Vec = self + .validators + .par_iter_mut() + .filter_map(|validator| { + // Advance the validator slot. + validator.set_slot(present_slot); + + // Prompt the validator to produce an attestation (if required). + validator.produce_free_attestation().ok() + }) + .collect(); + + debug!( + "Gathered {} FreeAttestations for slot {}.", + free_attestations.len(), + present_slot + ); - // Prompt the validator to produce an attestation (if required). - if let Ok(free_attestation) = validator.produce_free_attestation() { - free_attestations.push(free_attestation); - } - } free_attestations } @@ -115,6 +146,11 @@ impl BeaconChainHarness { let proposer = self.beacon_chain.block_proposer(present_slot).unwrap(); + debug!( + "Producing block from validator #{} for slot {}.", + proposer, present_slot + ); + self.validators[proposer].produce_block().unwrap() } @@ -131,7 +167,9 @@ impl BeaconChainHarness { .unwrap(); } let block = self.produce_block(); + debug!("Submitting block for processing..."); self.beacon_chain.process_block(block).unwrap(); + debug!("...block processed by BeaconChain."); } pub fn chain_dump(&self) -> Result, DumpError> { diff --git a/beacon_node/beacon_chain/test_harness/src/validator/mod.rs b/beacon_node/beacon_chain/test_harness/src/validator/mod.rs index 3f1b10d5e..08da1d197 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/mod.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/mod.rs @@ -52,9 +52,10 @@ impl TestValidator { pub fn new( keypair: Keypair, beacon_chain: Arc>, + spec: &ChainSpec, ) -> Self { - let spec = Arc::new(ChainSpec::foundation()); - let slot_clock = Arc::new(TestingSlotClock::new(0)); + let spec = Arc::new(spec.clone()); + let slot_clock = Arc::new(TestingSlotClock::new(spec.genesis_slot)); let signer = Arc::new(TestSigner::new(keypair.clone())); let beacon_node = Arc::new(BenchingBeaconNode::new(beacon_chain.clone())); let epoch_map = Arc::new(DirectDuties::new(keypair.pk.clone(), beacon_chain.clone())); diff --git a/beacon_node/beacon_chain/test_harness/tests/chain.rs b/beacon_node/beacon_chain/test_harness/tests/chain.rs index c312dcf43..f84157533 100644 --- a/beacon_node/beacon_chain/test_harness/tests/chain.rs +++ b/beacon_node/beacon_chain/test_harness/tests/chain.rs @@ -1,11 +1,20 @@ +use env_logger::{Builder, Env}; +use log::debug; use test_harness::BeaconChainHarness; use types::ChainSpec; #[test] fn it_can_build_on_genesis_block() { - let validator_count = 10; - let spec = ChainSpec::foundation(); - let mut harness = BeaconChainHarness::new(spec, validator_count); + let mut spec = ChainSpec::foundation(); + spec.genesis_slot = spec.epoch_length * 8; + + /* + spec.shard_count = spec.shard_count / 8; + spec.target_committee_size = spec.target_committee_size / 8; + */ + let validator_count = 1000; + + let mut harness = BeaconChainHarness::new(spec, validator_count as usize); harness.advance_chain_with_block(); } @@ -13,13 +22,21 @@ fn it_can_build_on_genesis_block() { #[test] #[ignore] fn it_can_produce_past_first_epoch_boundary() { + Builder::from_env(Env::default().default_filter_or("debug")).init(); + let validator_count = 100; + + debug!("Starting harness build..."); + let mut harness = BeaconChainHarness::new(ChainSpec::foundation(), validator_count); - let blocks = harness.spec.epoch_length + 1; + debug!("Harness built, tests starting.."); - for _ in 0..blocks { + let blocks = harness.spec.epoch_length * 3 + 1; + + for i in 0..blocks { harness.advance_chain_with_block(); + debug!("Produced block {}/{}.", i, blocks); } let dump = harness.chain_dump().expect("Chain dump failed."); diff --git a/eth2/types/Cargo.toml b/eth2/types/Cargo.toml index 3da5a1d26..a779c7e7e 100644 --- a/eth2/types/Cargo.toml +++ b/eth2/types/Cargo.toml @@ -11,6 +11,7 @@ ethereum-types = "0.4.0" hashing = { path = "../utils/hashing" } honey-badger-split = { path = "../utils/honey-badger-split" } integer-sqrt = "0.1" +log = "0.4" rand = "0.5.5" serde = "1.0" serde_derive = "1.0" diff --git a/eth2/types/src/beacon_state/attestation_participants.rs b/eth2/types/src/beacon_state/attestation_participants.rs new file mode 100644 index 000000000..282cee110 --- /dev/null +++ b/eth2/types/src/beacon_state/attestation_participants.rs @@ -0,0 +1,88 @@ +use crate::{ + beacon_state::CommitteesError, PendingAttestation, AttestationData, BeaconState, Bitfield, ChainSpec, +}; + +#[derive(Debug, PartialEq)] +pub enum Error { + NoCommitteeForShard, + NoCommittees, + BadBitfieldLength, + CommitteesError(CommitteesError), +} + +impl BeaconState { + pub fn get_attestation_participants_union( + &self, + attestations: &[&PendingAttestation], + spec: &ChainSpec, + ) -> Result, Error> { + attestations.iter().try_fold(vec![], |mut acc, a| { + acc.append(&mut self.get_attestation_participants( + &a.data, + &a.aggregation_bitfield, + spec, + )?); + Ok(acc) + }) + } + + // TODO: analyse for efficiency improvments. This implementation is naive. + pub fn get_attestation_participants( + &self, + attestation_data: &AttestationData, + aggregation_bitfield: &Bitfield, + spec: &ChainSpec, + ) -> Result, Error> { + let crosslink_committees = + self.get_crosslink_committees_at_slot(attestation_data.slot, spec)?; + + /* + let mut shard_present = false; + for (_committee, shard) in &crosslink_committees { + println!("want shard: {}, got shard: {}", shard, attestation_data.shard); + if *shard == attestation_data.shard { + shard_present = true; + } + } + if !shard_present { + return Err(Error::NoCommitteeForShard); + } + */ + + let crosslink_committee: Vec = crosslink_committees + .iter() + .filter_map(|(committee, shard)| { + if *shard == attestation_data.shard { + Some(committee.clone()) + } else { + None + } + }) + .collect::>>() + .first() + .ok_or_else(|| Error::NoCommitteeForShard)? + .clone(); + + /* + * TODO: check for this condition. + * + if aggregation_bitfield.len() != (crosslink_committee.len() + 7) / 8 { + return Err(Error::BadBitfieldLength); + } + */ + + let mut participants = vec![]; + for (i, validator_index) in crosslink_committee.iter().enumerate() { + if aggregation_bitfield.get(i).unwrap() { + participants.push(*validator_index); + } + } + Ok(participants) + } +} + +impl From for Error { + fn from(e: CommitteesError) -> Error { + Error::CommitteesError(e) + } +} diff --git a/eth2/types/src/beacon_state/attestation_validation.rs b/eth2/types/src/beacon_state/attestation_validation.rs index 26cc4e2b4..33aea85a3 100644 --- a/eth2/types/src/beacon_state/attestation_validation.rs +++ b/eth2/types/src/beacon_state/attestation_validation.rs @@ -1,4 +1,7 @@ -use crate::{AggregatePublicKey, Attestation, BeaconState, ChainSpec, Fork}; +use crate::{ + beacon_state::AttestationParticipantsError, AggregatePublicKey, Attestation, BeaconState, + ChainSpec, Fork, +}; use bls::bls_verify_aggregate; #[derive(Debug, PartialEq)] @@ -11,6 +14,7 @@ pub enum Error { BadSignature, ShardBlockRootNotZero, NoBlockRoot, + AttestationParticipantsError(AttestationParticipantsError), } macro_rules! ensure { @@ -32,6 +36,25 @@ impl BeaconState { attestation: &Attestation, spec: &ChainSpec, ) -> Result<(), Error> { + self.validate_attestation_signature_optional(attestation, spec, true) + } + + pub fn validate_attestation_without_signature( + &self, + attestation: &Attestation, + spec: &ChainSpec, + ) -> Result<(), Error> { + self.validate_attestation_signature_optional(attestation, spec, false) + } + + fn validate_attestation_signature_optional( + &self, + attestation: &Attestation, + spec: &ChainSpec, + verify_signature: bool, + ) -> Result<(), Error> { + // TODO: IMPORTANT: enable signature verification + let verify_signature = false; ensure!( attestation.data.slot + spec.min_attestation_inclusion_delay <= self.slot, Error::IncludedTooEarly @@ -65,25 +88,30 @@ impl BeaconState { == self.latest_crosslinks[attestation.data.shard as usize].shard_block_root), Error::BadLatestCrosslinkRoot ); - let participants = - self.get_attestation_participants(&attestation.data, &attestation.aggregation_bitfield); - let mut group_public_key = AggregatePublicKey::new(); - for participant in participants { - group_public_key.add( - self.validator_registry[participant as usize] - .pubkey - .as_raw(), - ) + if verify_signature { + let participants = self.get_attestation_participants( + &attestation.data, + &attestation.aggregation_bitfield, + spec, + )?; + let mut group_public_key = AggregatePublicKey::new(); + for participant in participants { + group_public_key.add( + self.validator_registry[participant as usize] + .pubkey + .as_raw(), + ) + } + ensure!( + bls_verify_aggregate( + &group_public_key, + &attestation.signable_message(PHASE_0_CUSTODY_BIT), + &attestation.aggregate_signature, + get_domain(&self.fork_data, attestation.data.slot, DOMAIN_ATTESTATION) + ), + Error::BadSignature + ); } - ensure!( - bls_verify_aggregate( - &group_public_key, - &attestation.signable_message(PHASE_0_CUSTODY_BIT), - &attestation.aggregate_signature, - get_domain(&self.fork_data, attestation.data.slot, DOMAIN_ATTESTATION) - ), - Error::BadSignature - ); ensure!( attestation.data.shard_block_root == spec.zero_hash, Error::ShardBlockRootNotZero @@ -96,3 +124,9 @@ pub fn get_domain(_fork: &Fork, _slot: u64, _domain_type: u64) -> u64 { // TODO: stubbed out. 0 } + +impl From for Error { + fn from(e: AttestationParticipantsError) -> Error { + Error::AttestationParticipantsError(e) + } +} diff --git a/eth2/types/src/beacon_state/committees.rs b/eth2/types/src/beacon_state/committees.rs new file mode 100644 index 000000000..ae9ad6fac --- /dev/null +++ b/eth2/types/src/beacon_state/committees.rs @@ -0,0 +1,127 @@ +use crate::{validator_registry::get_active_validator_indices, BeaconState, ChainSpec}; +use std::ops::Range; + +#[derive(Debug, PartialEq)] +pub enum Error { + InvalidSlot(u64, Range), + InsufficientNumberOfValidators, +} + +type Result = std::result::Result; + +impl BeaconState { + /// Returns the number of committees per slot. + /// + /// Note: this is _not_ the committee size. + pub fn get_committee_count_per_slot( + &self, + active_validator_count: usize, + spec: &ChainSpec, + ) -> u64 { + std::cmp::max( + 1, + std::cmp::min( + spec.shard_count / spec.epoch_length, + active_validator_count as u64 / spec.epoch_length / spec.target_committee_size, + ), + ) + } + + /// Returns the start slot and end slot of the current epoch containing `self.slot`. + pub fn get_current_epoch_boundaries(&self, epoch_length: u64) -> Range { + let slot_in_epoch = self.slot % epoch_length; + let start = self.slot - slot_in_epoch; + let end = self.slot + (epoch_length - slot_in_epoch); + start..end + } + + /// Returns the start slot and end slot of the current epoch containing `self.slot`. + pub fn get_previous_epoch_boundaries(&self, spec: &ChainSpec) -> Range { + let current_epoch = self.slot / spec.epoch_length; + let previous_epoch = current_epoch.saturating_sub(1); + let start = previous_epoch * spec.epoch_length; + let end = start + spec.epoch_length; + start..end + } + + fn get_previous_epoch_committee_count_per_slot(&self, spec: &ChainSpec) -> u64 { + let previous_active_validators = get_active_validator_indices( + &self.validator_registry, + self.previous_epoch_calculation_slot, + ); + self.get_committee_count_per_slot(previous_active_validators.len(), spec) as u64 + } + + pub fn get_current_epoch_committee_count_per_slot(&self, spec: &ChainSpec) -> u64 { + let current_active_validators = get_active_validator_indices( + &self.validator_registry, + self.current_epoch_calculation_slot, + ); + self.get_committee_count_per_slot(current_active_validators.len(), spec) + } + + pub fn get_crosslink_committees_at_slot( + &self, + slot: u64, + spec: &ChainSpec, + ) -> Result, u64)>> { + /* + let previous_epoch_range = self.get_current_epoch_boundaries(spec.epoch_length); + let current_epoch_range = self.get_current_epoch_boundaries(spec.epoch_length); + if !range_contains(¤t_epoch_range, slot) { + return Err(Error::InvalidSlot(slot, current_epoch_range)); + } + */ + let epoch = slot / spec.epoch_length; + let current_epoch = self.slot / spec.epoch_length; + let previous_epoch = if current_epoch == spec.genesis_slot { + current_epoch + } else { + current_epoch.saturating_sub(1) + }; + let next_epoch = current_epoch + 1; + + if !((previous_epoch <= epoch) & (epoch < next_epoch)) { + return Err(Error::InvalidSlot(slot, previous_epoch..current_epoch)); + } + + let offset = slot % spec.epoch_length; + + let (committees_per_slot, shuffling, slot_start_shard) = if epoch < current_epoch { + let committees_per_slot = self.get_previous_epoch_committee_count_per_slot(spec); + let shuffling = self.get_shuffling( + self.previous_epoch_seed, + self.previous_epoch_calculation_slot, + spec, + ); + let slot_start_shard = + (self.previous_epoch_start_shard + committees_per_slot * offset) % spec.shard_count; + (committees_per_slot, shuffling, slot_start_shard) + } else { + let committees_per_slot = self.get_current_epoch_committee_count_per_slot(spec); + let shuffling = self.get_shuffling( + self.current_epoch_seed, + self.current_epoch_calculation_slot, + spec, + ); + let slot_start_shard = + (self.current_epoch_start_shard + committees_per_slot * offset) % spec.shard_count; + (committees_per_slot, shuffling, slot_start_shard) + }; + + let mut crosslinks_at_slot = vec![]; + for i in 0..committees_per_slot { + let tuple = ( + shuffling[(committees_per_slot * offset + i) as usize].clone(), + (slot_start_shard + i) % spec.shard_count, + ); + crosslinks_at_slot.push(tuple) + } + Ok(crosslinks_at_slot) + } +} + +/// Utility function pending this functionality being stabilized on the `Range` type. +fn range_contains(range: &Range, target: T) -> bool { + range.start <= target && target < range.end +} diff --git a/eth2/types/src/beacon_state/epoch_processing.rs b/eth2/types/src/beacon_state/epoch_processing.rs index d9c1e2313..042bac518 100644 --- a/eth2/types/src/beacon_state/epoch_processing.rs +++ b/eth2/types/src/beacon_state/epoch_processing.rs @@ -1,9 +1,12 @@ -use super::winning_root::WinningRoot; +use super::winning_root::{Error as WinningRootError, WinningRoot}; use crate::{ - validator::StatusFlags, validator_registry::get_active_validator_indices, AttestationData, - BeaconState, Bitfield, ChainSpec, Crosslink, Hash256, PendingAttestation, + beacon_state::{AttestationParticipantsError, CommitteesError}, + validator::StatusFlags, + validator_registry::get_active_validator_indices, + BeaconState, ChainSpec, Crosslink, Hash256, PendingAttestation, }; use integer_sqrt::IntegerSquareRoot; +use log::debug; use std::collections::{HashMap, HashSet}; use std::iter::FromIterator; @@ -11,8 +14,17 @@ use std::iter::FromIterator; pub enum Error { UnableToDetermineProducer, NoBlockRoots, - UnableToGetCrosslinkCommittees, BaseRewardQuotientIsZero, + CommitteesError(CommitteesError), + AttestationParticipantsError(AttestationParticipantsError), + InclusionError(InclusionError), + WinningRootError(WinningRootError), +} + +#[derive(Debug, PartialEq)] +pub enum InclusionError { + NoIncludedAttestations, + AttestationParticipantsError(AttestationParticipantsError), } macro_rules! safe_add_assign { @@ -28,14 +40,19 @@ macro_rules! safe_sub_assign { impl BeaconState { pub fn per_epoch_processing(&mut self, spec: &ChainSpec) -> Result<(), Error> { + debug!("Starting per-epoch processing..."); /* * All Validators */ let active_validator_indices = get_active_validator_indices(&self.validator_registry, self.slot); - let total_balance: u64 = active_validator_indices - .iter() - .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)); + let total_balance = self.get_effective_balances(&active_validator_indices[..], spec); + + debug!( + "{} validators with a total balance of {} wei.", + active_validator_indices.len(), + total_balance + ); let current_epoch_attestations: Vec<&PendingAttestation> = self .latest_attestations @@ -53,42 +70,26 @@ impl BeaconState { let current_epoch_boundary_attestations: Vec<&PendingAttestation> = current_epoch_attestations .iter() - // `filter_map` is used to avoid a double borrow (`&&..`). - .filter_map(|a| { + .filter(|a| { // TODO: ensure this saturating sub is correct. - let block_root = match self - .get_block_root(self.slot.saturating_sub(spec.epoch_length), spec) - { - Some(root) => root, + match self.get_block_root(self.slot.saturating_sub(spec.epoch_length), spec) { + Some(block_root) => { + (a.data.epoch_boundary_root == *block_root) + && (a.data.justified_slot == self.justified_slot) + } // Protected by a check that latest_block_roots isn't empty. // // TODO: provide detailed reasoning. None => unreachable!(), - }; - - if (a.data.epoch_boundary_root == *block_root) - && (a.data.justified_slot == self.justified_slot) - { - Some(*a) - } else { - None } }) + .cloned() .collect(); - let current_epoch_boundary_attester_indices: Vec = - current_epoch_boundary_attestations - .iter() - .fold(vec![], |mut acc, a| { - acc.append( - &mut self.get_attestation_participants(&a.data, &a.aggregation_bitfield), - ); - acc - }); - - let current_epoch_boundary_attesting_balance = current_epoch_boundary_attester_indices - .iter() - .fold(0_u64, |acc, i| acc + self.get_effective_balance(*i, spec)); + let current_epoch_boundary_attester_indices = self + .get_attestation_participants_union(¤t_epoch_boundary_attestations[..], spec)?; + let current_epoch_boundary_attesting_balance = + self.get_effective_balances(¤t_epoch_boundary_attester_indices[..], spec); /* * Validators attesting during the previous epoch @@ -107,15 +108,8 @@ impl BeaconState { }) .collect(); - let previous_epoch_attester_indices: Vec = - previous_epoch_attestations - .iter() - .fold(vec![], |mut acc, a| { - acc.append( - &mut self.get_attestation_participants(&a.data, &a.aggregation_bitfield), - ); - acc - }); + let previous_epoch_attester_indices = + self.get_attestation_participants_union(&previous_epoch_attestations[..], spec)?; /* * Validators targetting the previous justified slot @@ -123,43 +117,22 @@ impl BeaconState { let previous_epoch_justified_attestations: Vec<&PendingAttestation> = { let mut a: Vec<&PendingAttestation> = current_epoch_attestations .iter() - // `filter_map` is used to avoid a double borrow (`&&..`). - .filter_map(|a| { - if a.data.justified_slot == self.previous_justified_slot { - Some(*a) - } else { - None - } - }) + .filter(|a| a.data.justified_slot == self.previous_justified_slot) + .cloned() .collect(); let mut b: Vec<&PendingAttestation> = previous_epoch_attestations .iter() - // `filter_map` is used to avoid a double borrow (`&&..`). - .filter_map(|a| { - if a.data.justified_slot == self.previous_justified_slot { - Some(*a) - } else { - None - } - }) + .filter(|a| a.data.justified_slot == self.previous_justified_slot) + .cloned() .collect(); a.append(&mut b); a }; - let previous_epoch_justified_attester_indices: Vec = - previous_epoch_justified_attestations - .iter() - .fold(vec![], |mut acc, a| { - acc.append( - &mut self.get_attestation_participants(&a.data, &a.aggregation_bitfield), - ); - acc - }); - - let previous_epoch_justified_attesting_balance = previous_epoch_justified_attester_indices - .iter() - .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)); + let previous_epoch_justified_attester_indices = self + .get_attestation_participants_union(&previous_epoch_justified_attestations[..], spec)?; + let previous_epoch_justified_attesting_balance = + self.get_effective_balances(&previous_epoch_justified_attester_indices[..], spec); /* * Validators justifying the epoch boundary block at the start of the previous epoch @@ -167,41 +140,24 @@ impl BeaconState { let previous_epoch_boundary_attestations: Vec<&PendingAttestation> = previous_epoch_justified_attestations .iter() - // `filter_map` is used to avoid a double borrow (`&&..`). - .filter_map(|a| { + .filter(|a| { // TODO: ensure this saturating sub is correct. - let block_root = match self - .get_block_root(self.slot.saturating_sub(2 * spec.epoch_length), spec) + match self.get_block_root(self.slot.saturating_sub(2 * spec.epoch_length), spec) { - Some(root) => root, + Some(block_root) => a.data.epoch_boundary_root == *block_root, // Protected by a check that latest_block_roots isn't empty. // // TODO: provide detailed reasoning. None => unreachable!(), - }; - - if a.data.epoch_boundary_root == *block_root { - Some(*a) - } else { - None } }) + .cloned() .collect(); - let previous_epoch_boundary_attester_indices: Vec = - previous_epoch_boundary_attestations - .iter() - .fold(vec![], |mut acc, a| { - acc.append( - &mut self.get_attestation_participants(&a.data, &a.aggregation_bitfield), - ); - acc - }); - - let previous_epoch_boundary_attesting_balance: u64 = - previous_epoch_boundary_attester_indices - .iter() - .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)); + let previous_epoch_boundary_attester_indices = self + .get_attestation_participants_union(&previous_epoch_boundary_attestations[..], spec)?; + let previous_epoch_boundary_attesting_balance = + self.get_effective_balances(&previous_epoch_boundary_attester_indices[..], spec); /* * Validators attesting to the expected beacon chain head during the previous epoch. @@ -209,37 +165,23 @@ impl BeaconState { let previous_epoch_head_attestations: Vec<&PendingAttestation> = previous_epoch_attestations .iter() - .filter_map(|a| { - let block_root = match self - .get_block_root(self.slot.saturating_sub(2 * spec.epoch_length), spec) + .filter(|a| { + match self.get_block_root(self.slot.saturating_sub(2 * spec.epoch_length), spec) { - Some(root) => root, + Some(block_root) => a.data.beacon_block_root == *block_root, // Protected by a check that latest_block_roots isn't empty. // // TODO: provide detailed reasoning. None => unreachable!(), - }; - - if a.data.beacon_block_root == *block_root { - Some(*a) - } else { - None } }) + .cloned() .collect(); - let previous_epoch_head_attester_indices: Vec = previous_epoch_head_attestations - .iter() - .fold(vec![], |mut acc, a| { - acc.append( - &mut self.get_attestation_participants(&a.data, &a.aggregation_bitfield), - ); - acc - }); - - let previous_epoch_head_attesting_balance: u64 = previous_epoch_head_attester_indices - .iter() - .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)); + let previous_epoch_head_attester_indices = + self.get_attestation_participants_union(&previous_epoch_head_attestations[..], spec)?; + let previous_epoch_head_attesting_balance = + self.get_effective_balances(&previous_epoch_head_attester_indices[..], spec); /* * Eth1 Data @@ -295,17 +237,22 @@ impl BeaconState { self.finalized_slot = self.previous_justified_slot; } + debug!( + "Finalized slot {}, justified slot {}.", + self.finalized_slot, self.justified_slot + ); + /* * Crosslinks */ // Cached for later lookups. - let mut winning_root_for_shards: HashMap = HashMap::new(); + let mut winning_root_for_shards: HashMap> = + HashMap::new(); - for slot in self.slot.saturating_sub(2 * spec.epoch_length)..self.slot { - let crosslink_committees_at_slot = self - .get_crosslink_committees_at_slot(slot, spec) - .map_err(|_| Error::UnableToGetCrosslinkCommittees)?; + // for slot in self.slot.saturating_sub(2 * spec.epoch_length)..self.slot { + for slot in self.get_previous_epoch_boundaries(spec) { + let crosslink_committees_at_slot = self.get_crosslink_committees_at_slot(slot, spec)?; for (crosslink_committee, shard) in crosslink_committees_at_slot { let shard = shard as u64; @@ -317,12 +264,9 @@ impl BeaconState { spec, ); - if let Some(winning_root) = winning_root { - let total_committee_balance: u64 = crosslink_committee - .iter() - .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)); - - winning_root_for_shards.insert(shard, winning_root.clone()); + if let Ok(winning_root) = &winning_root { + let total_committee_balance = + self.get_effective_balances(&crosslink_committee[..], spec); if (3 * winning_root.total_attesting_balance) >= (2 * total_committee_balance) { self.latest_crosslinks[shard as usize] = Crosslink { @@ -331,9 +275,15 @@ impl BeaconState { } } } + winning_root_for_shards.insert(shard, winning_root); } } + debug!( + "Found {} winning shard roots.", + winning_root_for_shards.len() + ); + /* * Rewards and Penalities */ @@ -342,13 +292,6 @@ impl BeaconState { return Err(Error::BaseRewardQuotientIsZero); } - /* - let base_reward = |i| match self.get_effective_balance(i, spec) { - Some(effective_balance) => effective_balance / base_reward_quotient / 5, - None => unreachable!(), - }; - */ - /* * Justification and finalization */ @@ -398,10 +341,7 @@ impl BeaconState { for index in previous_epoch_attester_indices { let base_reward = self.base_reward(index, base_reward_quotient, spec); - let inclusion_distance = match self.inclusion_distance(index) { - Some(distance) => distance, - None => unreachable!(), - }; + let inclusion_distance = self.inclusion_distance(index, spec)?; safe_add_assign!( self.validator_balances[index], @@ -432,10 +372,7 @@ impl BeaconState { for index in previous_epoch_attester_indices { let base_reward = self.base_reward(index, base_reward_quotient, spec); - let inclusion_distance = match self.inclusion_distance(index) { - Some(distance) => distance, - None => unreachable!(), - }; + let inclusion_distance = self.inclusion_distance(index, spec)?; safe_sub_assign!( self.validator_balances[index], @@ -445,14 +382,13 @@ impl BeaconState { } } + debug!("Processed validator justification and finalization rewards/penalities."); + /* * Attestation inclusion */ for index in previous_epoch_attester_indices_hashset { - let inclusion_slot = match self.inclusion_slot(index) { - Some(slot) => slot, - None => unreachable!(), - }; + let inclusion_slot = self.inclusion_slot(index, spec)?; let proposer_index = self .get_beacon_proposer_index(inclusion_slot, spec) .map_err(|_| Error::UnableToDetermineProducer)?; @@ -463,45 +399,46 @@ impl BeaconState { ); } + debug!("Processed validator attestation inclusdion rewards."); + /* * Crosslinks */ - for slot in self.slot.saturating_sub(2 * spec.epoch_length)..self.slot { - let crosslink_committees_at_slot = self - .get_crosslink_committees_at_slot(slot, spec) - .map_err(|_| Error::UnableToGetCrosslinkCommittees)?; + for slot in self.get_previous_epoch_boundaries(spec) { + let crosslink_committees_at_slot = self.get_crosslink_committees_at_slot(slot, spec)?; for (_crosslink_committee, shard) in crosslink_committees_at_slot { let shard = shard as u64; - let winning_root = winning_root_for_shards.get(&shard).expect("unreachable"); + if let Some(Ok(winning_root)) = winning_root_for_shards.get(&shard) { + // TODO: remove the map. + let attesting_validator_indices: HashSet = HashSet::from_iter( + winning_root.attesting_validator_indices.iter().map(|i| *i), + ); - // TODO: remove the map. - let attesting_validator_indices: HashSet = - HashSet::from_iter(winning_root.attesting_validator_indices.iter().map(|i| *i)); + for index in 0..self.validator_balances.len() { + let base_reward = self.base_reward(index, base_reward_quotient, spec); - for index in 0..self.validator_balances.len() { - let base_reward = self.base_reward(index, base_reward_quotient, spec); + if attesting_validator_indices.contains(&index) { + safe_add_assign!( + self.validator_balances[index], + base_reward * winning_root.total_attesting_balance + / winning_root.total_balance + ); + } else { + safe_sub_assign!(self.validator_balances[index], base_reward); + } + } - if attesting_validator_indices.contains(&index) { + for index in &winning_root.attesting_validator_indices { + let base_reward = self.base_reward(*index, base_reward_quotient, spec); safe_add_assign!( - self.validator_balances[index], + self.validator_balances[*index], base_reward * winning_root.total_attesting_balance / winning_root.total_balance ); - } else { - safe_sub_assign!(self.validator_balances[index], base_reward); } } - - for index in &winning_root.attesting_validator_indices { - let base_reward = self.base_reward(*index, base_reward_quotient, spec); - safe_add_assign!( - self.validator_balances[*index], - base_reward * winning_root.total_attesting_balance - / winning_root.total_balance - ); - } } } @@ -562,13 +499,8 @@ impl BeaconState { self.latest_attestations = self .latest_attestations .iter() - .filter_map(|a| { - if a.data.slot < self.slot - spec.epoch_length { - Some(a.clone()) - } else { - None - } - }) + .filter(|a| a.data.slot < self.slot - spec.epoch_length) + .cloned() .collect(); Ok(()) @@ -577,9 +509,7 @@ impl BeaconState { fn process_penalties_and_exits(&mut self, spec: &ChainSpec) { let active_validator_indices = get_active_validator_indices(&self.validator_registry, self.slot); - let total_balance = active_validator_indices - .iter() - .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)); + let total_balance = self.get_effective_balances(&active_validator_indices[..], spec); for index in 0..self.validator_balances.len() { let validator = &self.validator_registry[index]; @@ -640,9 +570,7 @@ impl BeaconState { fn update_validator_registry(&mut self, spec: &ChainSpec) { let active_validator_indices = get_active_validator_indices(&self.validator_registry, self.slot); - let total_balance = active_validator_indices - .iter() - .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)); + let total_balance = self.get_effective_balances(&active_validator_indices[..], spec); let max_balance_churn = std::cmp::max( spec.max_deposit, @@ -726,30 +654,64 @@ impl BeaconState { + effective_balance * epochs_since_finality / spec.inactivity_penalty_quotient / 2 } - fn inclusion_distance(&self, validator_index: usize) -> Option { - let attestation = self.earliest_included_attestation(validator_index)?; - Some( - attestation - .slot_included - .saturating_sub(attestation.data.slot), - ) + fn inclusion_distance( + &self, + validator_index: usize, + spec: &ChainSpec, + ) -> Result { + let attestation = self.earliest_included_attestation(validator_index, spec)?; + Ok(attestation + .slot_included + .saturating_sub(attestation.data.slot)) } - fn inclusion_slot(&self, validator_index: usize) -> Option { - let attestation = self.earliest_included_attestation(validator_index)?; - Some(attestation.slot_included) + fn inclusion_slot( + &self, + validator_index: usize, + spec: &ChainSpec, + ) -> Result { + let attestation = self.earliest_included_attestation(validator_index, spec)?; + Ok(attestation.slot_included) } - fn earliest_included_attestation(&self, validator_index: usize) -> Option<&PendingAttestation> { + fn earliest_included_attestation( + &self, + validator_index: usize, + spec: &ChainSpec, + ) -> Result<&PendingAttestation, InclusionError> { + let mut included_attestations = vec![]; + + for a in &self.latest_attestations { + let participants = + self.get_attestation_participants(&a.data, &a.aggregation_bitfield, spec)?; + if participants + .iter() + .find(|i| **i == validator_index) + .is_some() + { + included_attestations.push(a); + } + } + + Ok(included_attestations + .iter() + .min_by_key(|a| a.slot_included) + .and_then(|x| Some(*x)) + .ok_or_else(|| InclusionError::NoIncludedAttestations)?) + /* self.latest_attestations .iter() - .filter(|a| { - self.get_attestation_participants(&a.data, &a.aggregation_bitfield) + .try_for_each(|a| { + self.get_attestation_participants(&a.data, &a.aggregation_bitfield, spec) + })? + .filter(|participants| { + participants .iter() .find(|i| **i == validator_index) .is_some() }) .min_by_key(|a| a.slot_included) + */ } fn base_reward( @@ -761,6 +723,12 @@ impl BeaconState { self.get_effective_balance(validator_index, spec) / base_reward_quotient / 5 } + pub fn get_effective_balances(&self, validator_indices: &[usize], spec: &ChainSpec) -> u64 { + validator_indices + .iter() + .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)) + } + pub fn get_effective_balance(&self, validator_index: usize, spec: &ChainSpec) -> u64 { std::cmp::min(self.validator_balances[validator_index], spec.max_deposit) } @@ -773,13 +741,34 @@ impl BeaconState { None } } +} - pub fn get_attestation_participants( - &self, - _attestation_data: &AttestationData, - _aggregation_bitfield: &Bitfield, - ) -> Vec { - // TODO: stubbed out. - vec![0, 1] +impl From for Error { + fn from(e: CommitteesError) -> Error { + Error::CommitteesError(e) + } +} + +impl From for Error { + fn from(e: AttestationParticipantsError) -> Error { + Error::AttestationParticipantsError(e) + } +} + +impl From for Error { + fn from(e: InclusionError) -> Error { + Error::InclusionError(e) + } +} + +impl From for InclusionError { + fn from(e: AttestationParticipantsError) -> InclusionError { + InclusionError::AttestationParticipantsError(e) + } +} + +impl From for Error { + fn from(e: WinningRootError) -> Error { + Error::WinningRootError(e) } } diff --git a/eth2/types/src/beacon_state/mod.rs b/eth2/types/src/beacon_state/mod.rs index 85397ee11..4ccb3322a 100644 --- a/eth2/types/src/beacon_state/mod.rs +++ b/eth2/types/src/beacon_state/mod.rs @@ -10,21 +10,18 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +mod attestation_participants; mod attestation_validation; +mod committees; mod epoch_processing; mod shuffling; mod slot_processing; mod winning_root; +pub use self::attestation_participants::Error as AttestationParticipantsError; pub use self::attestation_validation::Error as AttestationValidationError; +pub use self::committees::Error as CommitteesError; pub use self::epoch_processing::Error as EpochProcessingError; -pub use self::slot_processing::Error as SlotProcessingError; - -#[derive(Debug, PartialEq)] -pub enum Error { - InvalidSlot, - InsufficientNumberOfValidators, -} // Custody will not be added to the specs until Phase 1 (Sharding Phase) so dummy class used. type CustodyChallenge = usize; diff --git a/eth2/types/src/beacon_state/shuffling.rs b/eth2/types/src/beacon_state/shuffling.rs index 95b402f8a..bef3abe71 100644 --- a/eth2/types/src/beacon_state/shuffling.rs +++ b/eth2/types/src/beacon_state/shuffling.rs @@ -1,16 +1,8 @@ -use super::Error; +use super::CommitteesError; use crate::{validator_registry::get_active_validator_indices, BeaconState, ChainSpec, Hash256}; use honey_badger_split::SplitExt; -use std::ops::Range; use vec_shuffle::shuffle; -// utility function pending this functionality being stabilized on the `Range` type. -fn range_contains(range: &Range, target: T) -> bool { - range.start <= target && target < range.end -} - -type Result = std::result::Result; - impl BeaconState { pub fn get_shuffling(&self, seed: Hash256, slot: u64, spec: &ChainSpec) -> Vec> { let slot = slot - (slot % spec.epoch_length); @@ -31,112 +23,21 @@ impl BeaconState { .collect() } - pub fn get_committee_count_per_slot( - &self, - active_validator_count: usize, - spec: &ChainSpec, - ) -> u64 { - std::cmp::max( - 1, - std::cmp::min( - spec.shard_count / spec.epoch_length, - active_validator_count as u64 / spec.epoch_length / spec.target_committee_size, - ), - ) - } - - /// Returns the start slot and end slot of the current epoch containing `self.slot`. - fn get_current_epoch_boundaries(&self, epoch_length: u64) -> Range { - let slot_in_epoch = self.slot % epoch_length; - let start = self.slot - slot_in_epoch; - let end = self.slot + (epoch_length - slot_in_epoch); - start..end - } - - fn get_previous_epoch_committee_count_per_slot( - &self, - spec: &ChainSpec, - /* - shard_count: u64, - epoch_length: u64, - target_committee_size: u64, - */ - ) -> u64 { - let previous_active_validators = get_active_validator_indices( - &self.validator_registry, - self.previous_epoch_calculation_slot, - ); - self.get_committee_count_per_slot(previous_active_validators.len(), spec) as u64 - } - - pub fn get_current_epoch_committee_count_per_slot(&self, spec: &ChainSpec) -> u64 { - let current_active_validators = get_active_validator_indices( - &self.validator_registry, - self.current_epoch_calculation_slot, - ); - self.get_committee_count_per_slot(current_active_validators.len(), spec) - } - - pub fn get_crosslink_committees_at_slot( + /// Returns the beacon proposer index for the `slot`. + /// If the state does not contain an index for a beacon proposer at the requested `slot`, then `None` is returned. + pub fn get_beacon_proposer_index( &self, slot: u64, spec: &ChainSpec, - /* - epoch_length: u64, - shard_count: u64, - target_committee_size: u64, - */ - ) -> Result, u64)>> { - let current_epoch_range = self.get_current_epoch_boundaries(spec.epoch_length); - if !range_contains(¤t_epoch_range, slot) { - return Err(Error::InvalidSlot); - } - let state_epoch_slot = current_epoch_range.start; - let offset = slot % spec.epoch_length; - - let (committees_per_slot, shuffling, slot_start_shard) = if slot < state_epoch_slot { - let committees_per_slot = self.get_previous_epoch_committee_count_per_slot(spec); - let shuffling = self.get_shuffling( - self.previous_epoch_seed, - self.previous_epoch_calculation_slot, - spec, - ); - let slot_start_shard = - (self.previous_epoch_start_shard + committees_per_slot * offset) % spec.shard_count; - (committees_per_slot, shuffling, slot_start_shard) - } else { - let committees_per_slot = self.get_current_epoch_committee_count_per_slot(spec); - let shuffling = self.get_shuffling( - self.current_epoch_seed, - self.current_epoch_calculation_slot, - spec, - ); - let slot_start_shard = - (self.current_epoch_start_shard + committees_per_slot * offset) % spec.shard_count; - (committees_per_slot, shuffling, slot_start_shard) - }; - - let shard_range = slot_start_shard..; - Ok(shuffling - .into_iter() - .skip((committees_per_slot * offset) as usize) - .zip(shard_range.into_iter()) - .take(committees_per_slot as usize) - .map(|(committees, shard_number)| (committees, shard_number % spec.shard_count)) - .collect::>()) - } - - /// Returns the beacon proposer index for the `slot`. - /// If the state does not contain an index for a beacon proposer at the requested `slot`, then `None` is returned. - pub fn get_beacon_proposer_index(&self, slot: u64, spec: &ChainSpec) -> Result { + ) -> Result { let committees = self.get_crosslink_committees_at_slot(slot, spec)?; committees .first() - .ok_or(Error::InsufficientNumberOfValidators) + .ok_or(CommitteesError::InsufficientNumberOfValidators) .and_then(|(first_committee, _)| { let index = (slot as usize) .checked_rem(first_committee.len()) - .ok_or(Error::InsufficientNumberOfValidators)?; + .ok_or(CommitteesError::InsufficientNumberOfValidators)?; // NOTE: next index will not panic as we have already returned if this is the case Ok(first_committee[index]) }) diff --git a/eth2/types/src/beacon_state/slot_processing.rs b/eth2/types/src/beacon_state/slot_processing.rs index 5dc351dae..5974596b6 100644 --- a/eth2/types/src/beacon_state/slot_processing.rs +++ b/eth2/types/src/beacon_state/slot_processing.rs @@ -1,20 +1,14 @@ -use crate::{BeaconState, ChainSpec, Hash256}; - -pub enum Error { - UnableToDetermineProducer, -} +use crate::{beacon_state::CommitteesError, BeaconState, ChainSpec, Hash256}; impl BeaconState { pub fn per_slot_processing( &mut self, previous_block_root: Hash256, spec: &ChainSpec, - ) -> Result<(), Error> { + ) -> Result<(), CommitteesError> { self.slot += 1; - let block_proposer = self - .get_beacon_proposer_index(self.slot, spec) - .map_err(|_| Error::UnableToDetermineProducer)?; + let block_proposer = self.get_beacon_proposer_index(self.slot, spec)?; self.validator_registry[block_proposer].proposer_slots += 1; self.latest_randao_mixes[(self.slot % spec.latest_randao_mixes_length) as usize] = @@ -35,7 +29,18 @@ impl BeaconState { &self, validator_index: usize, spec: &ChainSpec, - ) -> (u64, u64) { + ) -> Result<(u64, u64), CommitteesError> { + let mut result = None; + for slot in self.get_current_epoch_boundaries(spec.epoch_length) { + for (committee, shard) in self.get_crosslink_committees_at_slot(slot, spec)? { + if committee.iter().find(|i| **i == validator_index).is_some() { + result = Some(Ok((slot, shard))); + } + + } + } + result.unwrap() + /* // TODO: this is a stub; implement it properly. let validator_index = validator_index as u64; @@ -43,6 +48,7 @@ impl BeaconState { let shard = validator_index % spec.shard_count; (slot, shard) + */ } } diff --git a/eth2/types/src/beacon_state/winning_root.rs b/eth2/types/src/beacon_state/winning_root.rs index 04adcb007..20fdca09b 100644 --- a/eth2/types/src/beacon_state/winning_root.rs +++ b/eth2/types/src/beacon_state/winning_root.rs @@ -1,11 +1,12 @@ -use crate::{BeaconState, ChainSpec, Hash256, PendingAttestation}; +use crate::{ + beacon_state::AttestationParticipantsError, BeaconState, ChainSpec, Hash256, PendingAttestation, +}; use std::collections::HashMap; +#[derive(Debug, PartialEq)] pub enum Error { - UnableToDetermineProducer, - NoBlockRoots, - UnableToGetCrosslinkCommittees, - BaseRewardQuotientIsZero, + NoWinningRoot, + AttestationParticipantsError(AttestationParticipantsError), } #[derive(Clone)] @@ -23,7 +24,7 @@ impl BeaconState { current_epoch_attestations: &[&PendingAttestation], previous_epoch_attestations: &[&PendingAttestation], spec: &ChainSpec, - ) -> Option { + ) -> Result { let mut attestations = current_epoch_attestations.to_vec(); attestations.append(&mut previous_epoch_attestations.to_vec()); @@ -42,14 +43,23 @@ impl BeaconState { continue; } - let attesting_validator_indices = attestations.iter().fold(vec![], |mut acc, a| { - if (a.data.shard == shard) && (a.data.shard_block_root == *shard_block_root) { - acc.append( - &mut self.get_attestation_participants(&a.data, &a.aggregation_bitfield), - ); - } - acc - }); + // TODO: `cargo fmt` makes this rather ugly; tidy up. + let attesting_validator_indices = attestations.iter().try_fold::<_, _, Result< + _, + AttestationParticipantsError, + >>( + vec![], + |mut acc, a| { + if (a.data.shard == shard) && (a.data.shard_block_root == *shard_block_root) { + acc.append(&mut self.get_attestation_participants( + &a.data, + &a.aggregation_bitfield, + spec, + )?); + } + Ok(acc) + }, + )?; let total_balance: u64 = attesting_validator_indices .iter() @@ -73,7 +83,7 @@ impl BeaconState { candidates.insert(*shard_block_root, candidate_root); } - let winner = candidates + Ok(candidates .iter() .filter_map(|(_hash, candidate)| { if candidate.total_attesting_balance == highest_seen_balance { @@ -82,11 +92,15 @@ impl BeaconState { None } }) - .min_by_key(|candidate| candidate.shard_block_root); - - match winner { - Some(winner) => Some(winner.clone()), - None => None, - } + .min_by_key(|candidate| candidate.shard_block_root) + .ok_or_else(|| Error::NoWinningRoot)? + // TODO: avoid clone. + .clone()) + } +} + +impl From for Error { + fn from (e: AttestationParticipantsError) -> Error { + Error::AttestationParticipantsError(e) } } From 2fc6dbb02a16706bced7b20c35b3cdd90afda218 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 31 Jan 2019 08:49:50 +1100 Subject: [PATCH 058/107] Fix bug with `inclusion_distance` --- .../beacon_chain/src/block_production.rs | 22 +++++-- eth2/types/src/beacon_state/committees.rs | 31 +++++++-- .../src/beacon_state/epoch_processing.rs | 66 ++++++++++--------- .../types/src/beacon_state/slot_processing.rs | 10 --- 4 files changed, 77 insertions(+), 52 deletions(-) diff --git a/beacon_node/beacon_chain/src/block_production.rs b/beacon_node/beacon_chain/src/block_production.rs index 289bc485d..2a9998ccf 100644 --- a/beacon_node/beacon_chain/src/block_production.rs +++ b/beacon_node/beacon_chain/src/block_production.rs @@ -4,6 +4,7 @@ use bls::Signature; use log::debug; use slot_clock::TestingSlotClockError; use types::{ + beacon_state::CommitteesError, readers::{BeaconBlockReader, BeaconStateReader}, BeaconBlock, BeaconBlockBody, BeaconState, Eth1Data, Hash256, }; @@ -13,6 +14,7 @@ pub enum Error { DBError(String), StateTransitionError(TransitionError), PresentSlotIsNone, + CommitteesError(CommitteesError), } impl BeaconChain @@ -50,12 +52,14 @@ where debug!("Finding attesatations for block..."); - let attestations = self - .attestation_aggregator - .read() - .unwrap() - // TODO: advance the parent_state slot. - .get_attestations_for_state(&parent_state, &self.spec); + let attestations = { + let mut next_state = parent_state.clone(); + next_state.per_slot_processing(Hash256::zero(), &self.spec)?; + self.attestation_aggregator + .read() + .unwrap() + .get_attestations_for_state(&next_state, &self.spec) + }; debug!("Found {} attestation(s).", attestations.len()); @@ -111,3 +115,9 @@ impl From for Error { unreachable!(); // Testing clock never throws an error. } } + +impl From for Error { + fn from(e: CommitteesError) -> Error { + Error::CommitteesError(e) + } +} diff --git a/eth2/types/src/beacon_state/committees.rs b/eth2/types/src/beacon_state/committees.rs index ae9ad6fac..cbf2dc054 100644 --- a/eth2/types/src/beacon_state/committees.rs +++ b/eth2/types/src/beacon_state/committees.rs @@ -1,15 +1,28 @@ use crate::{validator_registry::get_active_validator_indices, BeaconState, ChainSpec}; +use log::debug; use std::ops::Range; #[derive(Debug, PartialEq)] pub enum Error { - InvalidSlot(u64, Range), + InvalidEpoch(u64, Range), InsufficientNumberOfValidators, } +macro_rules! ensure { + ($condition: expr, $result: expr) => { + if !$condition { + return Err($result); + } + }; +} + type Result = std::result::Result; impl BeaconState { + pub fn current_epoch(&self, spec: &ChainSpec) -> u64 { + self.slot / spec.epoch_length + } + /// Returns the number of committees per slot. /// /// Note: this is _not_ the committee size. @@ -69,7 +82,7 @@ impl BeaconState { let previous_epoch_range = self.get_current_epoch_boundaries(spec.epoch_length); let current_epoch_range = self.get_current_epoch_boundaries(spec.epoch_length); if !range_contains(¤t_epoch_range, slot) { - return Err(Error::InvalidSlot(slot, current_epoch_range)); + return Err(Error::InvalidEpoch(slot, current_epoch_range)); } */ let epoch = slot / spec.epoch_length; @@ -81,9 +94,17 @@ impl BeaconState { }; let next_epoch = current_epoch + 1; - if !((previous_epoch <= epoch) & (epoch < next_epoch)) { - return Err(Error::InvalidSlot(slot, previous_epoch..current_epoch)); - } + /* + debug!( + "state.slot: {}, slot: {}, current_epoch: {}, previous_epoch: {}, next_epoch: {}", + self.slot, slot, current_epoch, previous_epoch, next_epoch + ); + */ + + ensure!( + (previous_epoch <= epoch) & (epoch < next_epoch), + Error::InvalidEpoch(slot, previous_epoch..current_epoch) + ); let offset = slot % spec.epoch_length; diff --git a/eth2/types/src/beacon_state/epoch_processing.rs b/eth2/types/src/beacon_state/epoch_processing.rs index 042bac518..8b429b967 100644 --- a/eth2/types/src/beacon_state/epoch_processing.rs +++ b/eth2/types/src/beacon_state/epoch_processing.rs @@ -54,10 +54,18 @@ impl BeaconState { total_balance ); + debug!( + "latest_attestations = {:?}", + self.latest_attestations + .iter() + .map(|a| a.data.slot) + .collect::>() + ); + let current_epoch_attestations: Vec<&PendingAttestation> = self .latest_attestations .iter() - .filter(|a| (self.slot - spec.epoch_length <= a.data.slot) && (a.data.slot < self.slot)) + .filter(|a| a.data.slot / spec.epoch_length == self.current_epoch(spec)) .collect(); /* @@ -103,8 +111,7 @@ impl BeaconState { .iter() .filter(|a| { //TODO: ensure these saturating subs are correct. - (self.slot.saturating_sub(2 * spec.epoch_length) <= a.data.slot) - && (a.data.slot < self.slot.saturating_sub(spec.epoch_length)) + a.data.slot / spec.epoch_length == self.current_epoch(spec).saturating_sub(1) }) .collect(); @@ -183,6 +190,11 @@ impl BeaconState { let previous_epoch_head_attesting_balance = self.get_effective_balances(&previous_epoch_head_attester_indices[..], spec); + debug!( + "previous_epoch_head_attester_balance of {} wei.", + previous_epoch_head_attesting_balance + ); + /* * Eth1 Data */ @@ -295,7 +307,8 @@ impl BeaconState { /* * Justification and finalization */ - let epochs_since_finality = (self.slot - self.finalized_slot) / spec.epoch_length; + let epochs_since_finality = + self.slot.saturating_sub(self.finalized_slot) / spec.epoch_length; // TODO: fix this extra map let previous_epoch_justified_attester_indices_hashset: HashSet = @@ -307,6 +320,8 @@ impl BeaconState { let previous_epoch_attester_indices_hashset: HashSet = HashSet::from_iter(previous_epoch_attester_indices.iter().map(|i| *i)); + debug!("{} epochs since finality.", epochs_since_finality); + if epochs_since_finality <= 4 { for index in 0..self.validator_balances.len() { let base_reward = self.base_reward(index, base_reward_quotient, spec); @@ -341,7 +356,7 @@ impl BeaconState { for index in previous_epoch_attester_indices { let base_reward = self.base_reward(index, base_reward_quotient, spec); - let inclusion_distance = self.inclusion_distance(index, spec)?; + let inclusion_distance = self.inclusion_distance(&previous_epoch_attestations, index, spec)?; safe_add_assign!( self.validator_balances[index], @@ -372,7 +387,7 @@ impl BeaconState { for index in previous_epoch_attester_indices { let base_reward = self.base_reward(index, base_reward_quotient, spec); - let inclusion_distance = self.inclusion_distance(index, spec)?; + let inclusion_distance = self.inclusion_distance(&previous_epoch_attestations, index, spec)?; safe_sub_assign!( self.validator_balances[index], @@ -388,7 +403,7 @@ impl BeaconState { * Attestation inclusion */ for index in previous_epoch_attester_indices_hashset { - let inclusion_slot = self.inclusion_slot(index, spec)?; + let inclusion_slot = self.inclusion_slot(&previous_epoch_attestations[..], index, spec)?; let proposer_index = self .get_beacon_proposer_index(inclusion_slot, spec) .map_err(|_| Error::UnableToDetermineProducer)?; @@ -499,7 +514,7 @@ impl BeaconState { self.latest_attestations = self .latest_attestations .iter() - .filter(|a| a.data.slot < self.slot - spec.epoch_length) + .filter(|a| a.data.slot / spec.epoch_length >= self.current_epoch(spec)) .cloned() .collect(); @@ -656,10 +671,11 @@ impl BeaconState { fn inclusion_distance( &self, + attestations: &[&PendingAttestation], validator_index: usize, spec: &ChainSpec, ) -> Result { - let attestation = self.earliest_included_attestation(validator_index, spec)?; + let attestation = self.earliest_included_attestation(attestations, validator_index, spec)?; Ok(attestation .slot_included .saturating_sub(attestation.data.slot)) @@ -667,21 +683,23 @@ impl BeaconState { fn inclusion_slot( &self, + attestations: &[&PendingAttestation], validator_index: usize, spec: &ChainSpec, ) -> Result { - let attestation = self.earliest_included_attestation(validator_index, spec)?; + let attestation = self.earliest_included_attestation(attestations, validator_index, spec)?; Ok(attestation.slot_included) } fn earliest_included_attestation( &self, + attestations: &[&PendingAttestation], validator_index: usize, spec: &ChainSpec, - ) -> Result<&PendingAttestation, InclusionError> { + ) -> Result { let mut included_attestations = vec![]; - for a in &self.latest_attestations { + for (i, a) in attestations.iter().enumerate() { let participants = self.get_attestation_participants(&a.data, &a.aggregation_bitfield, spec)?; if participants @@ -689,29 +707,15 @@ impl BeaconState { .find(|i| **i == validator_index) .is_some() { - included_attestations.push(a); + included_attestations.push(i); } } - Ok(included_attestations + let earliest_attestation_index = included_attestations .iter() - .min_by_key(|a| a.slot_included) - .and_then(|x| Some(*x)) - .ok_or_else(|| InclusionError::NoIncludedAttestations)?) - /* - self.latest_attestations - .iter() - .try_for_each(|a| { - self.get_attestation_participants(&a.data, &a.aggregation_bitfield, spec) - })? - .filter(|participants| { - participants - .iter() - .find(|i| **i == validator_index) - .is_some() - }) - .min_by_key(|a| a.slot_included) - */ + .min_by_key(|i| attestations[**i].slot_included) + .ok_or_else(|| InclusionError::NoIncludedAttestations)?; + Ok(attestations[*earliest_attestation_index].clone()) } fn base_reward( diff --git a/eth2/types/src/beacon_state/slot_processing.rs b/eth2/types/src/beacon_state/slot_processing.rs index 5974596b6..9a82bcdf4 100644 --- a/eth2/types/src/beacon_state/slot_processing.rs +++ b/eth2/types/src/beacon_state/slot_processing.rs @@ -36,19 +36,9 @@ impl BeaconState { if committee.iter().find(|i| **i == validator_index).is_some() { result = Some(Ok((slot, shard))); } - } } result.unwrap() - /* - // TODO: this is a stub; implement it properly. - let validator_index = validator_index as u64; - - let slot = validator_index % spec.epoch_length; - let shard = validator_index % spec.shard_count; - - (slot, shard) - */ } } From 5ec9d82e40d21fe715159ddc4e1eeb0b78f181eb Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 31 Jan 2019 10:04:40 +1100 Subject: [PATCH 059/107] Minor bug fixes and formatting changes. --- .../beacon_state/attestation_participants.rs | 2 ++ eth2/types/src/beacon_state/committees.rs | 6 ++++ .../src/beacon_state/epoch_processing.rs | 29 ++++++++++--------- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/eth2/types/src/beacon_state/attestation_participants.rs b/eth2/types/src/beacon_state/attestation_participants.rs index 282cee110..4b0cc3c39 100644 --- a/eth2/types/src/beacon_state/attestation_participants.rs +++ b/eth2/types/src/beacon_state/attestation_participants.rs @@ -75,6 +75,8 @@ impl BeaconState { for (i, validator_index) in crosslink_committee.iter().enumerate() { if aggregation_bitfield.get(i).unwrap() { participants.push(*validator_index); + } else { + debug!("get_attestation_participants: validator missing."); } } Ok(participants) diff --git a/eth2/types/src/beacon_state/committees.rs b/eth2/types/src/beacon_state/committees.rs index cbf2dc054..f532552c6 100644 --- a/eth2/types/src/beacon_state/committees.rs +++ b/eth2/types/src/beacon_state/committees.rs @@ -23,6 +23,10 @@ impl BeaconState { self.slot / spec.epoch_length } + pub fn previous_epoch(&self, spec: &ChainSpec) -> u64 { + self.current_epoch(spec).saturating_sub(1) + } + /// Returns the number of committees per slot. /// /// Note: this is _not_ the committee size. @@ -142,7 +146,9 @@ impl BeaconState { } } +/* /// Utility function pending this functionality being stabilized on the `Range` type. fn range_contains(range: &Range, target: T) -> bool { range.start <= target && target < range.end } +*/ diff --git a/eth2/types/src/beacon_state/epoch_processing.rs b/eth2/types/src/beacon_state/epoch_processing.rs index 8b429b967..2ff6d58c3 100644 --- a/eth2/types/src/beacon_state/epoch_processing.rs +++ b/eth2/types/src/beacon_state/epoch_processing.rs @@ -54,14 +54,6 @@ impl BeaconState { total_balance ); - debug!( - "latest_attestations = {:?}", - self.latest_attestations - .iter() - .map(|a| a.data.slot) - .collect::>() - ); - let current_epoch_attestations: Vec<&PendingAttestation> = self .latest_attestations .iter() @@ -111,10 +103,12 @@ impl BeaconState { .iter() .filter(|a| { //TODO: ensure these saturating subs are correct. - a.data.slot / spec.epoch_length == self.current_epoch(spec).saturating_sub(1) + a.data.slot / spec.epoch_length == self.previous_epoch(spec) }) .collect(); + debug!("previous epoch attestations: {}", previous_epoch_attestations.len()); + let previous_epoch_attester_indices = self.get_attestation_participants_union(&previous_epoch_attestations[..], spec)?; @@ -320,6 +314,8 @@ impl BeaconState { let previous_epoch_attester_indices_hashset: HashSet = HashSet::from_iter(previous_epoch_attester_indices.iter().map(|i| *i)); + debug!("previous epoch justified attesters: {}, previous epoch boundary attesters: {}, previous epoch head attesters: {}, previous epoch attesters: {}", previous_epoch_justified_attester_indices.len(), previous_epoch_boundary_attester_indices.len(), previous_epoch_head_attestations.len(), previous_epoch_attester_indices.len()); + debug!("{} epochs since finality.", epochs_since_finality); if epochs_since_finality <= 4 { @@ -356,7 +352,8 @@ impl BeaconState { for index in previous_epoch_attester_indices { let base_reward = self.base_reward(index, base_reward_quotient, spec); - let inclusion_distance = self.inclusion_distance(&previous_epoch_attestations, index, spec)?; + let inclusion_distance = + self.inclusion_distance(&previous_epoch_attestations, index, spec)?; safe_add_assign!( self.validator_balances[index], @@ -387,7 +384,8 @@ impl BeaconState { for index in previous_epoch_attester_indices { let base_reward = self.base_reward(index, base_reward_quotient, spec); - let inclusion_distance = self.inclusion_distance(&previous_epoch_attestations, index, spec)?; + let inclusion_distance = + self.inclusion_distance(&previous_epoch_attestations, index, spec)?; safe_sub_assign!( self.validator_balances[index], @@ -403,7 +401,8 @@ impl BeaconState { * Attestation inclusion */ for index in previous_epoch_attester_indices_hashset { - let inclusion_slot = self.inclusion_slot(&previous_epoch_attestations[..], index, spec)?; + let inclusion_slot = + self.inclusion_slot(&previous_epoch_attestations[..], index, spec)?; let proposer_index = self .get_beacon_proposer_index(inclusion_slot, spec) .map_err(|_| Error::UnableToDetermineProducer)?; @@ -675,7 +674,8 @@ impl BeaconState { validator_index: usize, spec: &ChainSpec, ) -> Result { - let attestation = self.earliest_included_attestation(attestations, validator_index, spec)?; + let attestation = + self.earliest_included_attestation(attestations, validator_index, spec)?; Ok(attestation .slot_included .saturating_sub(attestation.data.slot)) @@ -687,7 +687,8 @@ impl BeaconState { validator_index: usize, spec: &ChainSpec, ) -> Result { - let attestation = self.earliest_included_attestation(attestations, validator_index, spec)?; + let attestation = + self.earliest_included_attestation(attestations, validator_index, spec)?; Ok(attestation.slot_included) } From ae39a24e71b9a2e2049e206c1c5c0359e8e7a051 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 31 Jan 2019 14:16:28 +1100 Subject: [PATCH 060/107] Add committee_index to aggregator Fixes a bug where the validator index bit was set on the bitfield, instead of the committee index --- .../src/attestation_aggregator.rs | 39 ++++++++++++++----- .../src/attestation_processing.rs | 2 +- beacon_node/beacon_chain/src/info.rs | 5 ++- .../beacon_state/attestation_participants.rs | 13 ++++++- eth2/types/src/beacon_state/committees.rs | 14 ------- .../types/src/beacon_state/slot_processing.rs | 6 +-- 6 files changed, 48 insertions(+), 31 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_aggregator.rs b/beacon_node/beacon_chain/src/attestation_aggregator.rs index c95feb1e1..d8e409801 100644 --- a/beacon_node/beacon_chain/src/attestation_aggregator.rs +++ b/beacon_node/beacon_chain/src/attestation_aggregator.rs @@ -1,7 +1,7 @@ use std::collections::{HashMap, HashSet}; use types::{ - AggregateSignature, Attestation, AttestationData, BeaconState, Bitfield, ChainSpec, - FreeAttestation, Signature, + beacon_state::CommitteesError, AggregateSignature, Attestation, AttestationData, BeaconState, + Bitfield, ChainSpec, FreeAttestation, Signature, }; const PHASE_0_CUSTODY_BIT: bool = false; @@ -21,6 +21,9 @@ pub enum ProcessOutcome { pub enum ProcessError { BadValidatorIndex, BadSignature, + BadSlot, + BadShard, + CommitteesError(CommitteesError), } impl AttestationAggregator { @@ -34,13 +37,25 @@ impl AttestationAggregator { &mut self, state: &BeaconState, free_attestation: &FreeAttestation, + spec: &ChainSpec, ) -> Result { - let validator_index = free_attestation.validator_index as usize; + // let validator_index = free_attestation.validator_index as usize; + let (slot, shard, committee_index) = state.attestation_slot_and_shard_for_validator( + free_attestation.validator_index as usize, + spec, + )?; + + if free_attestation.data.slot != slot { + return Err(ProcessError::BadSlot); + } + if free_attestation.data.shard != shard { + return Err(ProcessError::BadShard); + } let signable_message = free_attestation.data.signable_message(PHASE_0_CUSTODY_BIT); let validator_pubkey = &state .validator_registry - .get(validator_index) + .get(free_attestation.validator_index as usize) .ok_or_else(|| ProcessError::BadValidatorIndex)? .pubkey; @@ -55,7 +70,7 @@ impl AttestationAggregator { if let Some(updated_attestation) = aggregate_attestation( existing_attestation, &free_attestation.signature, - validator_index, + committee_index as usize, ) { self.store.insert(signable_message, updated_attestation); Ok(ProcessOutcome::Aggregated) @@ -66,7 +81,7 @@ impl AttestationAggregator { let mut aggregate_signature = AggregateSignature::new(); aggregate_signature.add(&free_attestation.signature); let mut aggregation_bitfield = Bitfield::new(); - aggregation_bitfield.set(validator_index, true); + aggregation_bitfield.set(committee_index as usize, true); let new_attestation = Attestation { data: free_attestation.data.clone(), aggregation_bitfield, @@ -113,18 +128,18 @@ impl AttestationAggregator { fn aggregate_attestation( existing_attestation: &Attestation, signature: &Signature, - validator_index: usize, + committee_index: usize, ) -> Option { let already_signed = existing_attestation .aggregation_bitfield - .get(validator_index) + .get(committee_index) .unwrap_or(false); if already_signed { None } else { let mut aggregation_bitfield = existing_attestation.aggregation_bitfield.clone(); - aggregation_bitfield.set(validator_index, true); + aggregation_bitfield.set(committee_index, true); let mut aggregate_signature = existing_attestation.aggregate_signature.clone(); aggregate_signature.add(&signature); @@ -135,3 +150,9 @@ fn aggregate_attestation( }) } } + +impl From for ProcessError { + fn from(e: CommitteesError) -> ProcessError { + ProcessError::CommitteesError(e) + } +} diff --git a/beacon_node/beacon_chain/src/attestation_processing.rs b/beacon_node/beacon_chain/src/attestation_processing.rs index 8704cd489..ecbe355df 100644 --- a/beacon_node/beacon_chain/src/attestation_processing.rs +++ b/beacon_node/beacon_chain/src/attestation_processing.rs @@ -27,7 +27,7 @@ where self.attestation_aggregator .write() .expect("Aggregator unlock failed.") - .process_free_attestation(&state, &free_attestation) + .process_free_attestation(&state, &free_attestation, &self.spec) .map_err(|e| e.into()) } } diff --git a/beacon_node/beacon_chain/src/info.rs b/beacon_node/beacon_chain/src/info.rs index 1ecd97c1e..a67af7f39 100644 --- a/beacon_node/beacon_chain/src/info.rs +++ b/beacon_node/beacon_chain/src/info.rs @@ -67,9 +67,10 @@ where let present_slot = self.present_slot()?; let state = self.state(present_slot).ok()?; - state + let (slot, shard, _committee) = state .attestation_slot_and_shard_for_validator(validator_index, &self.spec) - .ok() + .ok()?; + Some((slot, shard)) } } diff --git a/eth2/types/src/beacon_state/attestation_participants.rs b/eth2/types/src/beacon_state/attestation_participants.rs index 4b0cc3c39..3a120a621 100644 --- a/eth2/types/src/beacon_state/attestation_participants.rs +++ b/eth2/types/src/beacon_state/attestation_participants.rs @@ -1,6 +1,8 @@ use crate::{ - beacon_state::CommitteesError, PendingAttestation, AttestationData, BeaconState, Bitfield, ChainSpec, + beacon_state::CommitteesError, AttestationData, BeaconState, Bitfield, ChainSpec, + PendingAttestation, }; +use log::debug; #[derive(Debug, PartialEq)] pub enum Error { @@ -74,9 +76,16 @@ impl BeaconState { let mut participants = vec![]; for (i, validator_index) in crosslink_committee.iter().enumerate() { if aggregation_bitfield.get(i).unwrap() { + debug!( + "committee index {} found in attestation on slot {}", + i, attestation_data.slot + ); participants.push(*validator_index); } else { - debug!("get_attestation_participants: validator missing."); + debug!( + "committee index {} not found in attestation on slot {}", + i, attestation_data.slot + ); } } Ok(participants) diff --git a/eth2/types/src/beacon_state/committees.rs b/eth2/types/src/beacon_state/committees.rs index f532552c6..d4fd725da 100644 --- a/eth2/types/src/beacon_state/committees.rs +++ b/eth2/types/src/beacon_state/committees.rs @@ -82,13 +82,6 @@ impl BeaconState { slot: u64, spec: &ChainSpec, ) -> Result, u64)>> { - /* - let previous_epoch_range = self.get_current_epoch_boundaries(spec.epoch_length); - let current_epoch_range = self.get_current_epoch_boundaries(spec.epoch_length); - if !range_contains(¤t_epoch_range, slot) { - return Err(Error::InvalidEpoch(slot, current_epoch_range)); - } - */ let epoch = slot / spec.epoch_length; let current_epoch = self.slot / spec.epoch_length; let previous_epoch = if current_epoch == spec.genesis_slot { @@ -98,13 +91,6 @@ impl BeaconState { }; let next_epoch = current_epoch + 1; - /* - debug!( - "state.slot: {}, slot: {}, current_epoch: {}, previous_epoch: {}, next_epoch: {}", - self.slot, slot, current_epoch, previous_epoch, next_epoch - ); - */ - ensure!( (previous_epoch <= epoch) & (epoch < next_epoch), Error::InvalidEpoch(slot, previous_epoch..current_epoch) diff --git a/eth2/types/src/beacon_state/slot_processing.rs b/eth2/types/src/beacon_state/slot_processing.rs index 9a82bcdf4..211f0c290 100644 --- a/eth2/types/src/beacon_state/slot_processing.rs +++ b/eth2/types/src/beacon_state/slot_processing.rs @@ -29,12 +29,12 @@ impl BeaconState { &self, validator_index: usize, spec: &ChainSpec, - ) -> Result<(u64, u64), CommitteesError> { + ) -> Result<(u64, u64, u64), CommitteesError> { let mut result = None; for slot in self.get_current_epoch_boundaries(spec.epoch_length) { for (committee, shard) in self.get_crosslink_committees_at_slot(slot, spec)? { - if committee.iter().find(|i| **i == validator_index).is_some() { - result = Some(Ok((slot, shard))); + if let Some(committee_index) = committee.iter().find(|i| **i == validator_index) { + result = Some(Ok((slot, shard, *committee_index as u64))); } } } From 8073296f5dd813cf58cf1c9a6b0e3b8a074cc843 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 31 Jan 2019 16:39:44 +1100 Subject: [PATCH 061/107] Ensure per_epoch processing always runs. Previously, it was running _after_ a state transition, not before it with the slot processing. --- .../beacon_chain/src/block_production.rs | 10 +++---- .../beacon_chain/src/canonical_head.rs | 10 +++---- .../beacon_chain/src/state_transition.rs | 14 ++++----- eth2/types/src/beacon_state/mod.rs | 1 + .../types/src/beacon_state/slot_processing.rs | 30 +++++++++++++++++-- 5 files changed, 44 insertions(+), 21 deletions(-) diff --git a/beacon_node/beacon_chain/src/block_production.rs b/beacon_node/beacon_chain/src/block_production.rs index 2a9998ccf..2e8665a77 100644 --- a/beacon_node/beacon_chain/src/block_production.rs +++ b/beacon_node/beacon_chain/src/block_production.rs @@ -4,7 +4,7 @@ use bls::Signature; use log::debug; use slot_clock::TestingSlotClockError; use types::{ - beacon_state::CommitteesError, + beacon_state::SlotProcessingError, readers::{BeaconBlockReader, BeaconStateReader}, BeaconBlock, BeaconBlockBody, BeaconState, Eth1Data, Hash256, }; @@ -14,7 +14,7 @@ pub enum Error { DBError(String), StateTransitionError(TransitionError), PresentSlotIsNone, - CommitteesError(CommitteesError), + SlotProcessingError(SlotProcessingError), } impl BeaconChain @@ -116,8 +116,8 @@ impl From for Error { } } -impl From for Error { - fn from(e: CommitteesError) -> Error { - Error::CommitteesError(e) +impl From for Error { + fn from(e: SlotProcessingError) -> Error { + Error::SlotProcessingError(e) } } diff --git a/beacon_node/beacon_chain/src/canonical_head.rs b/beacon_node/beacon_chain/src/canonical_head.rs index 4bcac5b64..a92cff327 100644 --- a/beacon_node/beacon_chain/src/canonical_head.rs +++ b/beacon_node/beacon_chain/src/canonical_head.rs @@ -1,11 +1,11 @@ use crate::{BeaconChain, CheckPoint, ClientDB, SlotClock}; use std::sync::RwLockReadGuard; -use types::{beacon_state::CommitteesError, BeaconBlock, BeaconState, Hash256}; +use types::{beacon_state::SlotProcessingError, BeaconBlock, BeaconState, Hash256}; #[derive(Debug, PartialEq)] pub enum Error { PastSlot, - CommitteesError(CommitteesError), + SlotProcessingError(SlotProcessingError), } impl BeaconChain @@ -64,8 +64,8 @@ where } } -impl From for Error { - fn from(e: CommitteesError) -> Error { - Error::CommitteesError(e) +impl From for Error { + fn from(e: SlotProcessingError) -> Error { + Error::SlotProcessingError(e) } } diff --git a/beacon_node/beacon_chain/src/state_transition.rs b/beacon_node/beacon_chain/src/state_transition.rs index fc29a63c4..df6712273 100644 --- a/beacon_node/beacon_chain/src/state_transition.rs +++ b/beacon_node/beacon_chain/src/state_transition.rs @@ -5,7 +5,7 @@ use log::debug; use slot_clock::{SystemTimeSlotClockError, TestingSlotClockError}; use ssz::{ssz_encode, TreeHash}; use types::{ - beacon_state::{AttestationValidationError, CommitteesError, EpochProcessingError}, + beacon_state::{AttestationValidationError, CommitteesError, SlotProcessingError}, readers::BeaconBlockReader, BeaconBlock, BeaconState, Exit, Fork, Hash256, PendingAttestation, }; @@ -52,7 +52,7 @@ pub enum Error { BadCustodyResponses, SlotClockError(SystemTimeSlotClockError), CommitteesError(CommitteesError), - EpochProcessingError(EpochProcessingError), + SlotProcessingError(SlotProcessingError), } impl BeaconChain @@ -309,10 +309,6 @@ where Error::BadCustodyResponses ); - if state.slot % self.spec.epoch_length == 0 { - state.per_epoch_processing(&self.spec)?; - } - debug!("State transition complete."); Ok(state) @@ -367,8 +363,8 @@ impl From for Error { } } -impl From for Error { - fn from(e: EpochProcessingError) -> Error { - Error::EpochProcessingError(e) +impl From for Error { + fn from(e: SlotProcessingError) -> Error { + Error::SlotProcessingError(e) } } diff --git a/eth2/types/src/beacon_state/mod.rs b/eth2/types/src/beacon_state/mod.rs index 4ccb3322a..3972eaa82 100644 --- a/eth2/types/src/beacon_state/mod.rs +++ b/eth2/types/src/beacon_state/mod.rs @@ -22,6 +22,7 @@ pub use self::attestation_participants::Error as AttestationParticipantsError; pub use self::attestation_validation::Error as AttestationValidationError; pub use self::committees::Error as CommitteesError; pub use self::epoch_processing::Error as EpochProcessingError; +pub use self::slot_processing::Error as SlotProcessingError; // Custody will not be added to the specs until Phase 1 (Sharding Phase) so dummy class used. type CustodyChallenge = usize; diff --git a/eth2/types/src/beacon_state/slot_processing.rs b/eth2/types/src/beacon_state/slot_processing.rs index 211f0c290..ab5311d21 100644 --- a/eth2/types/src/beacon_state/slot_processing.rs +++ b/eth2/types/src/beacon_state/slot_processing.rs @@ -1,11 +1,21 @@ -use crate::{beacon_state::CommitteesError, BeaconState, ChainSpec, Hash256}; +use crate::{ + beacon_state::{CommitteesError, EpochProcessingError}, + BeaconState, ChainSpec, Hash256, +}; +use log::debug; + +#[derive(Debug, PartialEq)] +pub enum Error { + CommitteesError(CommitteesError), + EpochProcessingError(EpochProcessingError), +} impl BeaconState { pub fn per_slot_processing( &mut self, previous_block_root: Hash256, spec: &ChainSpec, - ) -> Result<(), CommitteesError> { + ) -> Result<(), Error> { self.slot += 1; let block_proposer = self.get_beacon_proposer_index(self.slot, spec)?; @@ -22,6 +32,10 @@ impl BeaconState { let root = merkle_root(&self.latest_block_roots[..]); self.batched_block_roots.push(root); } + + if self.slot % spec.epoch_length == 0 { + self.per_epoch_processing(spec)?; + } Ok(()) } @@ -45,3 +59,15 @@ impl BeaconState { fn merkle_root(_input: &[Hash256]) -> Hash256 { Hash256::zero() } + +impl From for Error { + fn from(e: CommitteesError) -> Error { + Error::CommitteesError(e) + } +} + +impl From for Error { + fn from(e: EpochProcessingError) -> Error { + Error::EpochProcessingError(e) + } +} From c5a0170a0a47530d7a9a0b967bac76fdfbe6273e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 31 Jan 2019 16:40:44 +1100 Subject: [PATCH 062/107] Tidy out some verbose debugs --- beacon_node/beacon_chain/src/state_transition.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/beacon_node/beacon_chain/src/state_transition.rs b/beacon_node/beacon_chain/src/state_transition.rs index df6712273..405da055c 100644 --- a/beacon_node/beacon_chain/src/state_transition.rs +++ b/beacon_node/beacon_chain/src/state_transition.rs @@ -120,8 +120,6 @@ where ); } - debug!("Block signature is valid."); - /* * RANDAO */ @@ -136,8 +134,6 @@ where Error::BadRandaoSignature ); - debug!("RANDAO signature is valid."); - // TODO: check this is correct. let new_mix = { let mut mix = state.latest_randao_mixes From 5da955e3ce8f0510ef016f6588cc07481ddd200b Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 31 Jan 2019 16:41:17 +1100 Subject: [PATCH 063/107] Ensure harness produces block before attestations. --- .../beacon_chain/test_harness/src/harness.rs | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/src/harness.rs b/beacon_node/beacon_chain/test_harness/src/harness.rs index a6bb6a02c..ee4d94148 100644 --- a/beacon_node/beacon_chain/test_harness/src/harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/harness.rs @@ -97,7 +97,9 @@ impl BeaconChainHarness { /// Move the `slot_clock` for the `BeaconChain` forward one slot. /// /// This is the equivalent of advancing a system clock forward one `SLOT_DURATION`. - pub fn increment_beacon_chain_slot(&mut self) { + /// + /// Returns the new slot. + pub fn increment_beacon_chain_slot(&mut self) -> u64 { let slot = self .beacon_chain .present_slot() @@ -107,6 +109,7 @@ impl BeaconChainHarness { debug!("Incrementing BeaconChain slot to {}.", slot); self.beacon_chain.slot_clock.set_slot(slot); + slot } /// Gather the `FreeAttestation`s from the valiators. @@ -151,6 +154,8 @@ impl BeaconChainHarness { proposer, present_slot ); + // Ensure the validators slot clock is accurate. + self.validators[proposer].set_slot(present_slot); self.validators[proposer].produce_block().unwrap() } @@ -160,16 +165,20 @@ impl BeaconChainHarness { /// validators. pub fn advance_chain_with_block(&mut self) { self.increment_beacon_chain_slot(); - let free_attestations = self.gather_free_attesations(); - for free_attestation in free_attestations { - self.beacon_chain - .process_free_attestation(free_attestation.clone()) - .unwrap(); - } + + // Produce a new block. let block = self.produce_block(); debug!("Submitting block for processing..."); self.beacon_chain.process_block(block).unwrap(); debug!("...block processed by BeaconChain."); + + // Produce new attestations. + let free_attestations = self.gather_free_attesations(); + for free_attestation in free_attestations { + self.beacon_chain + .process_free_attestation(free_attestation) + .unwrap(); + } } pub fn chain_dump(&self) -> Result, DumpError> { From a1cdc0402d70172dd170ec4ed3379c7af85650ba Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 31 Jan 2019 16:41:54 +1100 Subject: [PATCH 064/107] Tidy attestation_participants function. It was using a map where it could use a find. --- .../beacon_state/attestation_participants.rs | 45 +++---------------- 1 file changed, 6 insertions(+), 39 deletions(-) diff --git a/eth2/types/src/beacon_state/attestation_participants.rs b/eth2/types/src/beacon_state/attestation_participants.rs index 3a120a621..f70f131ce 100644 --- a/eth2/types/src/beacon_state/attestation_participants.rs +++ b/eth2/types/src/beacon_state/attestation_participants.rs @@ -38,54 +38,21 @@ impl BeaconState { let crosslink_committees = self.get_crosslink_committees_at_slot(attestation_data.slot, spec)?; - /* - let mut shard_present = false; - for (_committee, shard) in &crosslink_committees { - println!("want shard: {}, got shard: {}", shard, attestation_data.shard); - if *shard == attestation_data.shard { - shard_present = true; - } - } - if !shard_present { - return Err(Error::NoCommitteeForShard); - } - */ - - let crosslink_committee: Vec = crosslink_committees + let committee_index: usize = crosslink_committees .iter() - .filter_map(|(committee, shard)| { - if *shard == attestation_data.shard { - Some(committee.clone()) - } else { - None - } - }) - .collect::>>() - .first() - .ok_or_else(|| Error::NoCommitteeForShard)? - .clone(); + .position(|(_committee, shard)| *shard == attestation_data.shard) + .ok_or_else(|| Error::NoCommitteeForShard)?; + let (crosslink_committee, _shard) = &crosslink_committees[committee_index]; /* - * TODO: check for this condition. + * TODO: that bitfield length is valid. * - if aggregation_bitfield.len() != (crosslink_committee.len() + 7) / 8 { - return Err(Error::BadBitfieldLength); - } - */ + */ let mut participants = vec![]; for (i, validator_index) in crosslink_committee.iter().enumerate() { if aggregation_bitfield.get(i).unwrap() { - debug!( - "committee index {} found in attestation on slot {}", - i, attestation_data.slot - ); participants.push(*validator_index); - } else { - debug!( - "committee index {} not found in attestation on slot {}", - i, attestation_data.slot - ); } } Ok(participants) From 48801e4674d0468f48c4a6a24e2267258e7b76e2 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 31 Jan 2019 16:42:31 +1100 Subject: [PATCH 065/107] Re-enable signature checking for validations --- eth2/types/src/beacon_state/attestation_validation.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/eth2/types/src/beacon_state/attestation_validation.rs b/eth2/types/src/beacon_state/attestation_validation.rs index 33aea85a3..c4a1f0d8a 100644 --- a/eth2/types/src/beacon_state/attestation_validation.rs +++ b/eth2/types/src/beacon_state/attestation_validation.rs @@ -53,8 +53,6 @@ impl BeaconState { spec: &ChainSpec, verify_signature: bool, ) -> Result<(), Error> { - // TODO: IMPORTANT: enable signature verification - let verify_signature = false; ensure!( attestation.data.slot + spec.min_attestation_inclusion_delay <= self.slot, Error::IncludedTooEarly From 5c44f97fba4fb93433b757f50f6557ea41e9412d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 31 Jan 2019 16:42:53 +1100 Subject: [PATCH 066/107] Fix bug with committee index for attester duties It was returning the validator_index instead of the `committee_index`. --- eth2/types/src/beacon_state/slot_processing.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eth2/types/src/beacon_state/slot_processing.rs b/eth2/types/src/beacon_state/slot_processing.rs index ab5311d21..9501083f0 100644 --- a/eth2/types/src/beacon_state/slot_processing.rs +++ b/eth2/types/src/beacon_state/slot_processing.rs @@ -47,8 +47,9 @@ impl BeaconState { let mut result = None; for slot in self.get_current_epoch_boundaries(spec.epoch_length) { for (committee, shard) in self.get_crosslink_committees_at_slot(slot, spec)? { - if let Some(committee_index) = committee.iter().find(|i| **i == validator_index) { - result = Some(Ok((slot, shard, *committee_index as u64))); + if let Some(committee_index) = committee.iter().position(|&i| i == validator_index) + { + result = Some(Ok((slot, shard, committee_index as u64))); } } } From 3f13c25c05a4c6084ec8e51bca7c31297b337bb8 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 31 Jan 2019 18:28:54 +1100 Subject: [PATCH 067/107] Fix issues with per_epoch block_root calls They were being called with the wrong slot. --- beacon_node/beacon_chain/src/attestation_production.rs | 3 ++- eth2/types/src/beacon_state/committees.rs | 8 ++++++++ eth2/types/src/beacon_state/epoch_processing.rs | 10 +++------- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_production.rs b/beacon_node/beacon_chain/src/attestation_production.rs index 5adf70a4a..9755f48af 100644 --- a/beacon_node/beacon_chain/src/attestation_production.rs +++ b/beacon_node/beacon_chain/src/attestation_production.rs @@ -30,8 +30,9 @@ where .ok_or_else(|| Error::SlotTooOld)?; let head_slot = self.head().beacon_block.slot; + let previous_epoch_start_slot = head_slot - (head_slot % self.spec.epoch_length); let epoch_boundary_root = *state - .get_block_root(head_slot % self.spec.epoch_length, &self.spec) + .get_block_root(previous_epoch_start_slot, &self.spec) .ok_or_else(|| Error::SlotTooOld)?; Ok(AttestationData { diff --git a/eth2/types/src/beacon_state/committees.rs b/eth2/types/src/beacon_state/committees.rs index d4fd725da..1e1689e92 100644 --- a/eth2/types/src/beacon_state/committees.rs +++ b/eth2/types/src/beacon_state/committees.rs @@ -27,6 +27,14 @@ impl BeaconState { self.current_epoch(spec).saturating_sub(1) } + pub fn current_epoch_start_slot(&self, spec: &ChainSpec) -> u64 { + self.current_epoch(spec) * spec.epoch_length + } + + pub fn previous_epoch_start_slot(&self, spec: &ChainSpec) -> u64 { + self.previous_epoch(spec) * spec.epoch_length + } + /// Returns the number of committees per slot. /// /// Note: this is _not_ the committee size. diff --git a/eth2/types/src/beacon_state/epoch_processing.rs b/eth2/types/src/beacon_state/epoch_processing.rs index 2ff6d58c3..f38024181 100644 --- a/eth2/types/src/beacon_state/epoch_processing.rs +++ b/eth2/types/src/beacon_state/epoch_processing.rs @@ -71,8 +71,7 @@ impl BeaconState { current_epoch_attestations .iter() .filter(|a| { - // TODO: ensure this saturating sub is correct. - match self.get_block_root(self.slot.saturating_sub(spec.epoch_length), spec) { + match self.get_block_root(self.current_epoch_start_slot(spec), spec) { Some(block_root) => { (a.data.epoch_boundary_root == *block_root) && (a.data.justified_slot == self.justified_slot) @@ -142,9 +141,7 @@ impl BeaconState { previous_epoch_justified_attestations .iter() .filter(|a| { - // TODO: ensure this saturating sub is correct. - match self.get_block_root(self.slot.saturating_sub(2 * spec.epoch_length), spec) - { + match self.get_block_root(self.previous_epoch_start_slot(spec), spec) { Some(block_root) => a.data.epoch_boundary_root == *block_root, // Protected by a check that latest_block_roots isn't empty. // @@ -167,8 +164,7 @@ impl BeaconState { previous_epoch_attestations .iter() .filter(|a| { - match self.get_block_root(self.slot.saturating_sub(2 * spec.epoch_length), spec) - { + match self.get_block_root(a.data.slot, spec) { Some(block_root) => a.data.beacon_block_root == *block_root, // Protected by a check that latest_block_roots isn't empty. // From d6adfc7655a170999e77090a3a00d5c73e9ddd7e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 31 Jan 2019 18:30:43 +1100 Subject: [PATCH 068/107] Refactor justified_slot fn Now it reads from the head state --- beacon_node/beacon_chain/src/info.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/beacon_node/beacon_chain/src/info.rs b/beacon_node/beacon_chain/src/info.rs index a67af7f39..e23aaef78 100644 --- a/beacon_node/beacon_chain/src/info.rs +++ b/beacon_node/beacon_chain/src/info.rs @@ -56,11 +56,18 @@ where } pub fn justified_slot(&self) -> u64 { + // TODO: fix unwrap + let present_slot = self.present_slot().unwrap(); + // TODO: fix unwrap + let state = self.state(present_slot).unwrap(); + state.justified_slot + /* self.justified_head .read() .expect("Justified head poisoned") .beacon_block .slot + */ } pub fn validator_attestion_slot_and_shard(&self, validator_index: usize) -> Option<(u64, u64)> { From e1239a1ecc881a9b4fc58cb89c5aa8abbcc6146f Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 31 Jan 2019 18:31:20 +1100 Subject: [PATCH 069/107] Ensure attestation_participants is deduped --- .../beacon_state/attestation_participants.rs | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/eth2/types/src/beacon_state/attestation_participants.rs b/eth2/types/src/beacon_state/attestation_participants.rs index f70f131ce..28d3ed4a9 100644 --- a/eth2/types/src/beacon_state/attestation_participants.rs +++ b/eth2/types/src/beacon_state/attestation_participants.rs @@ -18,14 +18,19 @@ impl BeaconState { attestations: &[&PendingAttestation], spec: &ChainSpec, ) -> Result, Error> { - attestations.iter().try_fold(vec![], |mut acc, a| { - acc.append(&mut self.get_attestation_participants( - &a.data, - &a.aggregation_bitfield, - spec, - )?); - Ok(acc) - }) + let mut all_participants = attestations + .iter() + .try_fold::<_, _, Result, Error>>(vec![], |mut acc, a| { + acc.append(&mut self.get_attestation_participants( + &a.data, + &a.aggregation_bitfield, + spec, + )?); + Ok(acc) + })?; + all_participants.sort_unstable(); + all_participants.dedup(); + Ok(all_participants) } // TODO: analyse for efficiency improvments. This implementation is naive. From c1b3d1b63ec7262833be9d7e293580adc75942ae Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 31 Jan 2019 18:32:01 +1100 Subject: [PATCH 070/107] Tidy up and add some logs --- .../src/attestation_aggregator.rs | 1 - .../src/beacon_state/epoch_processing.rs | 33 ++++++++++++++++--- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_aggregator.rs b/beacon_node/beacon_chain/src/attestation_aggregator.rs index d8e409801..40b574399 100644 --- a/beacon_node/beacon_chain/src/attestation_aggregator.rs +++ b/beacon_node/beacon_chain/src/attestation_aggregator.rs @@ -39,7 +39,6 @@ impl AttestationAggregator { free_attestation: &FreeAttestation, spec: &ChainSpec, ) -> Result { - // let validator_index = free_attestation.validator_index as usize; let (slot, shard, committee_index) = state.attestation_slot_and_shard_for_validator( free_attestation.validator_index as usize, spec, diff --git a/eth2/types/src/beacon_state/epoch_processing.rs b/eth2/types/src/beacon_state/epoch_processing.rs index f38024181..296c2e848 100644 --- a/eth2/types/src/beacon_state/epoch_processing.rs +++ b/eth2/types/src/beacon_state/epoch_processing.rs @@ -40,7 +40,10 @@ macro_rules! safe_sub_assign { impl BeaconState { pub fn per_epoch_processing(&mut self, spec: &ChainSpec) -> Result<(), Error> { - debug!("Starting per-epoch processing..."); + debug!( + "Starting per-epoch processing on epoch {}...", + self.current_epoch(spec) + ); /* * All Validators */ @@ -60,6 +63,11 @@ impl BeaconState { .filter(|a| a.data.slot / spec.epoch_length == self.current_epoch(spec)) .collect(); + debug!( + "Current epoch attestations: {}", + current_epoch_attestations.len() + ); + /* * Validators attesting during the current epoch. */ @@ -90,6 +98,11 @@ impl BeaconState { let current_epoch_boundary_attesting_balance = self.get_effective_balances(¤t_epoch_boundary_attester_indices[..], spec); + debug!( + "Current epoch boundary attesters: {}", + current_epoch_boundary_attester_indices.len() + ); + /* * Validators attesting during the previous epoch */ @@ -106,7 +119,10 @@ impl BeaconState { }) .collect(); - debug!("previous epoch attestations: {}", previous_epoch_attestations.len()); + debug!( + "previous epoch attestations: {}", + previous_epoch_attestations.len() + ); let previous_epoch_attester_indices = self.get_attestation_participants_union(&previous_epoch_attestations[..], spec)?; @@ -209,6 +225,7 @@ impl BeaconState { // TODO: check saturating_sub is correct. self.justification_bitfield |= 2; self.justified_slot = self.slot.saturating_sub(2 * spec.epoch_length); + debug!(">= 2/3 voted for previous epoch boundary"); } // If >= 2/3 of validators voted for the current epoch boundary @@ -216,6 +233,7 @@ impl BeaconState { // TODO: check saturating_sub is correct. self.justification_bitfield |= 1; self.justified_slot = self.slot.saturating_sub(1 * spec.epoch_length); + debug!(">= 2/3 voted for current epoch boundary"); } if (self.previous_justified_slot == self.slot.saturating_sub(2 * spec.epoch_length)) @@ -310,7 +328,7 @@ impl BeaconState { let previous_epoch_attester_indices_hashset: HashSet = HashSet::from_iter(previous_epoch_attester_indices.iter().map(|i| *i)); - debug!("previous epoch justified attesters: {}, previous epoch boundary attesters: {}, previous epoch head attesters: {}, previous epoch attesters: {}", previous_epoch_justified_attester_indices.len(), previous_epoch_boundary_attester_indices.len(), previous_epoch_head_attestations.len(), previous_epoch_attester_indices.len()); + debug!("previous epoch justified attesters: {}, previous epoch boundary attesters: {}, previous epoch head attesters: {}, previous epoch attesters: {}", previous_epoch_justified_attester_indices.len(), previous_epoch_boundary_attester_indices.len(), previous_epoch_head_attester_indices.len(), previous_epoch_attester_indices.len()); debug!("{} epochs since finality.", epochs_since_finality); @@ -396,7 +414,7 @@ impl BeaconState { /* * Attestation inclusion */ - for index in previous_epoch_attester_indices_hashset { + for &index in &previous_epoch_attester_indices_hashset { let inclusion_slot = self.inclusion_slot(&previous_epoch_attestations[..], index, spec)?; let proposer_index = self @@ -409,7 +427,10 @@ impl BeaconState { ); } - debug!("Processed validator attestation inclusdion rewards."); + debug!( + "Previous epoch attesters: {}.", + previous_epoch_attester_indices_hashset.len() + ); /* * Crosslinks @@ -513,6 +534,8 @@ impl BeaconState { .cloned() .collect(); + debug!("Epoch transition complete."); + Ok(()) } From 02a962d35dab046e54e437616272146b57d024fb Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 31 Jan 2019 18:32:23 +1100 Subject: [PATCH 071/107] Ensure per_epoch trans. happens before per_slot. --- eth2/types/src/beacon_state/slot_processing.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eth2/types/src/beacon_state/slot_processing.rs b/eth2/types/src/beacon_state/slot_processing.rs index 9501083f0..4693bd069 100644 --- a/eth2/types/src/beacon_state/slot_processing.rs +++ b/eth2/types/src/beacon_state/slot_processing.rs @@ -16,6 +16,10 @@ impl BeaconState { previous_block_root: Hash256, spec: &ChainSpec, ) -> Result<(), Error> { + if (self.slot + 1) % spec.epoch_length == 0 { + self.per_epoch_processing(spec)?; + } + self.slot += 1; let block_proposer = self.get_beacon_proposer_index(self.slot, spec)?; @@ -32,10 +36,6 @@ impl BeaconState { let root = merkle_root(&self.latest_block_roots[..]); self.batched_block_roots.push(root); } - - if self.slot % spec.epoch_length == 0 { - self.per_epoch_processing(spec)?; - } Ok(()) } From becb81d842b099d70927ce74490f564ca506c7b1 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 31 Jan 2019 20:24:37 +1100 Subject: [PATCH 072/107] Fix memory blow-up with Arc Previously it was cloning the ChainSpec, now it shares an Arc. --- beacon_node/beacon_chain/test_harness/src/harness.rs | 8 +++++--- .../beacon_chain/test_harness/src/validator/mod.rs | 3 +-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/src/harness.rs b/beacon_node/beacon_chain/test_harness/src/harness.rs index ee4d94148..0b493b64a 100644 --- a/beacon_node/beacon_chain/test_harness/src/harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/harness.rs @@ -19,7 +19,7 @@ pub struct BeaconChainHarness { pub block_store: Arc>, pub state_store: Arc>, pub validators: Vec, - pub spec: ChainSpec, + pub spec: Arc, } impl BeaconChainHarness { @@ -74,12 +74,14 @@ impl BeaconChainHarness { .unwrap(), ); + let spec = Arc::new(spec); + debug!("Creating validator producer and attester instances..."); // Spawn the test validator instances. let validators: Vec = keypairs - .par_iter() - .map(|keypair| TestValidator::new(keypair.clone(), beacon_chain.clone(), &spec)) + .iter() + .map(|keypair| TestValidator::new(keypair.clone(), beacon_chain.clone(), spec.clone())) .collect(); debug!("Created {} TestValidators", validators.len()); diff --git a/beacon_node/beacon_chain/test_harness/src/validator/mod.rs b/beacon_node/beacon_chain/test_harness/src/validator/mod.rs index 08da1d197..018ffbc86 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/mod.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/mod.rs @@ -52,9 +52,8 @@ impl TestValidator { pub fn new( keypair: Keypair, beacon_chain: Arc>, - spec: &ChainSpec, + spec: Arc, ) -> Self { - let spec = Arc::new(spec.clone()); let slot_clock = Arc::new(TestingSlotClock::new(spec.genesis_slot)); let signer = Arc::new(TestSigner::new(keypair.clone())); let beacon_node = Arc::new(BenchingBeaconNode::new(beacon_chain.clone())); From b6b738e83a71c8c70e04abb483d2cd832f65bb29 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 31 Jan 2019 20:27:15 +1100 Subject: [PATCH 073/107] Improve test_harness efficiency for attesatations Reduce the amount of attester.poll() calls by using a hashset --- .../beacon_chain/test_harness/src/harness.rs | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/src/harness.rs b/beacon_node/beacon_chain/test_harness/src/harness.rs index 0b493b64a..3cdfc8024 100644 --- a/beacon_node/beacon_chain/test_harness/src/harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/harness.rs @@ -8,8 +8,10 @@ use db::{ use log::debug; use rayon::prelude::*; use slot_clock::TestingSlotClock; +use std::collections::HashSet; use std::fs::File; use std::io::prelude::*; +use std::iter::FromIterator; use std::sync::Arc; use types::{BeaconBlock, ChainSpec, FreeAttestation, Keypair, Validator}; @@ -121,15 +123,34 @@ impl BeaconChainHarness { pub fn gather_free_attesations(&mut self) -> Vec { let present_slot = self.beacon_chain.present_slot().unwrap(); + let attesting_validators = self + .beacon_chain + .state(present_slot) + .unwrap() + .get_crosslink_committees_at_slot(present_slot, &self.spec) + .unwrap() + .iter() + .fold(vec![], |mut acc, (committee, _slot)| { + acc.append(&mut committee.clone()); + acc + }); + let attesting_validators: HashSet = + HashSet::from_iter(attesting_validators.iter().cloned()); + let free_attestations: Vec = self .validators .par_iter_mut() - .filter_map(|validator| { - // Advance the validator slot. - validator.set_slot(present_slot); + .enumerate() + .filter_map(|(i, validator)| { + if attesting_validators.contains(&i) { + // Advance the validator slot. + validator.set_slot(present_slot); - // Prompt the validator to produce an attestation (if required). - validator.produce_free_attestation().ok() + // Prompt the validator to produce an attestation (if required). + validator.produce_free_attestation().ok() + } else { + None + } }) .collect(); From 20e45b3369f7ffcae44956c4af11b1a2e77c7dd9 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Feb 2019 14:48:09 +1100 Subject: [PATCH 074/107] Refactor block_processing - Add the rayon library for parallelization - Move from std::sync::Rwlock to rayon::RwLock - Add `state` field to BeaconChain - Fix major bug in attestation validator where justified slot was incorrectly looked up. --- beacon_node/beacon_chain/Cargo.toml | 1 + .../src/attestation_processing.rs | 8 +- .../src/attestation_production.rs | 35 ++-- .../beacon_chain/src/attestation_targets.rs | 10 +- .../beacon_chain/src/block_processing.rs | 54 +++--- .../beacon_chain/src/block_production.rs | 78 ++++----- .../beacon_chain/src/canonical_head.rs | 41 ++--- .../beacon_chain/src/finalized_head.rs | 11 +- beacon_node/beacon_chain/src/info.rs | 34 +--- beacon_node/beacon_chain/src/lib.rs | 7 +- .../test_harness/benches/state_transition.rs | 70 +++++--- .../beacon_chain/test_harness/src/harness.rs | 19 ++- .../src/validator/beacon_node/attester.rs | 4 +- .../test_harness/src/validator/mod.rs | 1 + .../beacon_chain/test_harness/tests/chain.rs | 2 +- eth2/types/Cargo.toml | 1 + .../beacon_state/attestation_validation.rs | 2 +- .../src/beacon_state/block_processing.rs | 158 ++++++------------ .../src/beacon_state/epoch_processing.rs | 7 +- eth2/types/src/beacon_state/mod.rs | 2 + 20 files changed, 236 insertions(+), 309 deletions(-) rename beacon_node/beacon_chain/src/state_transition.rs => eth2/types/src/beacon_state/block_processing.rs (66%) diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index 7e6dc209d..5c930403c 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -13,6 +13,7 @@ failure = "0.1" failure_derive = "0.1" genesis = { path = "../../eth2/genesis" } hashing = { path = "../../eth2/utils/hashing" } +parking_lot = "0.7" log = "0.4" env_logger = "0.6" serde = "1.0" diff --git a/beacon_node/beacon_chain/src/attestation_processing.rs b/beacon_node/beacon_chain/src/attestation_processing.rs index ecbe355df..1b7e8ace4 100644 --- a/beacon_node/beacon_chain/src/attestation_processing.rs +++ b/beacon_node/beacon_chain/src/attestation_processing.rs @@ -19,15 +19,9 @@ where &self, free_attestation: FreeAttestation, ) -> Result { - let present_slot = self - .present_slot() - .ok_or_else(|| Error::PresentSlotUnknown)?; - let state = self.state(present_slot)?; - self.attestation_aggregator .write() - .expect("Aggregator unlock failed.") - .process_free_attestation(&state, &free_attestation, &self.spec) + .process_free_attestation(&self.state.read(), &free_attestation, &self.spec) .map_err(|e| e.into()) } } diff --git a/beacon_node/beacon_chain/src/attestation_production.rs b/beacon_node/beacon_chain/src/attestation_production.rs index 9755f48af..febd2ff9a 100644 --- a/beacon_node/beacon_chain/src/attestation_production.rs +++ b/beacon_node/beacon_chain/src/attestation_production.rs @@ -13,30 +13,27 @@ where T: ClientDB, U: SlotClock, { - pub fn produce_attestation_data( - &self, - slot: u64, - shard: u64, - ) -> Result { - let present_slot = self - .present_slot() - .ok_or_else(|| Error::PresentSlotUnknown)?; - let state = self.state(present_slot).map_err(|_| Error::StateError)?; - + pub fn produce_attestation_data(&self, shard: u64) -> Result { let justified_slot = self.justified_slot(); - - let justified_block_root = *state + let justified_block_root = self + .state + .read() .get_block_root(justified_slot, &self.spec) - .ok_or_else(|| Error::SlotTooOld)?; + .ok_or_else(|| Error::SlotTooOld)? + .clone(); - let head_slot = self.head().beacon_block.slot; - let previous_epoch_start_slot = head_slot - (head_slot % self.spec.epoch_length); - let epoch_boundary_root = *state - .get_block_root(previous_epoch_start_slot, &self.spec) - .ok_or_else(|| Error::SlotTooOld)?; + let epoch_boundary_root = self + .state + .read() + .get_block_root( + self.state.read().current_epoch_start_slot(&self.spec), + &self.spec, + ) + .ok_or_else(|| Error::SlotTooOld)? + .clone(); Ok(AttestationData { - slot, + slot: self.state.read().slot, shard, beacon_block_root: self.head().beacon_block_root.clone(), epoch_boundary_root, diff --git a/beacon_node/beacon_chain/src/attestation_targets.rs b/beacon_node/beacon_chain/src/attestation_targets.rs index d968abcf2..62f0bbdfa 100644 --- a/beacon_node/beacon_chain/src/attestation_targets.rs +++ b/beacon_node/beacon_chain/src/attestation_targets.rs @@ -28,18 +28,12 @@ where U: SlotClock, { pub fn insert_latest_attestation_target(&self, validator_index: u64, block_root: Hash256) { - let mut targets = self - .latest_attestation_targets - .write() - .expect("CRITICAL: CanonicalHead poisioned."); + let mut targets = self.latest_attestation_targets.write(); targets.insert(validator_index, block_root); } pub fn get_latest_attestation_target(&self, validator_index: u64) -> Option { - let targets = self - .latest_attestation_targets - .read() - .expect("CRITICAL: CanonicalHead poisioned."); + let targets = self.latest_attestation_targets.read(); match targets.get(validator_index) { Some(hash) => Some(hash.clone()), diff --git a/beacon_node/beacon_chain/src/block_processing.rs b/beacon_node/beacon_chain/src/block_processing.rs index b4126ed9e..2c637031e 100644 --- a/beacon_node/beacon_chain/src/block_processing.rs +++ b/beacon_node/beacon_chain/src/block_processing.rs @@ -1,9 +1,9 @@ -use super::state_transition::Error as TransitionError; use super::{BeaconChain, ClientDB, DBError, SlotClock}; use log::debug; use slot_clock::{SystemTimeSlotClockError, TestingSlotClockError}; use ssz::{ssz_encode, Encodable}; use types::{ + beacon_state::{BlockProcessingError, SlotProcessingError}, readers::{BeaconBlockReader, BeaconStateReader}, Hash256, }; @@ -16,7 +16,6 @@ pub enum ValidBlock { #[derive(Debug, PartialEq)] pub enum InvalidBlock { FutureSlot, - StateTransitionFailed(TransitionError), StateRootMismatch, } @@ -29,31 +28,16 @@ pub enum Outcome { #[derive(Debug, PartialEq)] pub enum Error { DBError(String), - SlotClockError(SystemTimeSlotClockError), - - NotImplemented, - PresentSlotIsNone, UnableToDecodeBlock, + PresentSlotIsNone, + SlotClockError(SystemTimeSlotClockError), MissingParentState(Hash256), InvalidParentState(Hash256), MissingBeaconBlock(Hash256), InvalidBeaconBlock(Hash256), MissingParentBlock(Hash256), - NoBlockProducer, - StateSlotMismatch, - BadBlockSignature, - BadRandaoSignature, - MaxProposerSlashingsExceeded, - BadProposerSlashing, - MaxAttestationsExceeded, - BadAttestation, - NoBlockRoot, - MaxDepositsExceeded, - MaxExitsExceeded, - BadExit, - BadCustodyReseeds, - BadCustodyChallenges, - BadCustodyResponses, + SlotProcessingError(SlotProcessingError), + PerBlockProcessingError(BlockProcessingError), } impl BeaconChain @@ -99,14 +83,13 @@ where .into_beacon_state() .ok_or(Error::InvalidParentState(parent_state_root))?; - let state = match self.state_transition(parent_state, &block) { - Ok(state) => state, - Err(error) => { - return Ok(Outcome::InvalidBlock(InvalidBlock::StateTransitionFailed( - error, - ))); - } - }; + let mut state = parent_state; + + for _ in state.slot..present_slot { + state.per_slot_processing(parent_block_root.clone(), &self.spec)?; + } + + state.per_block_processing(&block, &self.spec)?; let state_root = state.canonical_root(); @@ -131,6 +114,7 @@ where state.clone(), state_root.clone(), ); + *self.state.write() = state.clone(); } // The block was sucessfully processed. @@ -144,6 +128,18 @@ impl From for Error { } } +impl From for Error { + fn from(e: SlotProcessingError) -> Error { + Error::SlotProcessingError(e) + } +} + +impl From for Error { + fn from(e: BlockProcessingError) -> Error { + Error::PerBlockProcessingError(e) + } +} + impl From for Error { fn from(_: TestingSlotClockError) -> Error { unreachable!(); // Testing clock never throws an error. diff --git a/beacon_node/beacon_chain/src/block_production.rs b/beacon_node/beacon_chain/src/block_production.rs index 2e8665a77..6c1fa10c4 100644 --- a/beacon_node/beacon_chain/src/block_production.rs +++ b/beacon_node/beacon_chain/src/block_production.rs @@ -1,10 +1,9 @@ -use super::state_transition::Error as TransitionError; use super::{BeaconChain, ClientDB, DBError, SlotClock}; use bls::Signature; use log::debug; -use slot_clock::TestingSlotClockError; +use slot_clock::{SystemTimeSlotClockError, TestingSlotClockError}; use types::{ - beacon_state::SlotProcessingError, + beacon_state::{BlockProcessingError, SlotProcessingError}, readers::{BeaconBlockReader, BeaconStateReader}, BeaconBlock, BeaconBlockBody, BeaconState, Eth1Data, Hash256, }; @@ -12,9 +11,10 @@ use types::{ #[derive(Debug, PartialEq)] pub enum Error { DBError(String), - StateTransitionError(TransitionError), PresentSlotIsNone, SlotProcessingError(SlotProcessingError), + PerBlockProcessingError(BlockProcessingError), + SlotClockError(SystemTimeSlotClockError), } impl BeaconChain @@ -29,43 +29,31 @@ where where Error: From<::Error>, { - // TODO: allow producing a block from a previous (or future?) slot. - let present_slot = self - .slot_clock - .present_slot() - .map_err(|e| e.into())? - .ok_or(Error::PresentSlotIsNone)?; + debug!("Starting block production..."); - debug!("Producing block for slot {}...", present_slot); + let mut state = self.state.read().clone(); - let parent_root = self.head().beacon_block_root; - let parent_block_reader = self - .block_store - .get_reader(&parent_root)? - .ok_or_else(|| Error::DBError("Block not found.".to_string()))?; - let parent_state = self - .state_store - .get_reader(&parent_block_reader.state_root())? - .ok_or_else(|| Error::DBError("State not found.".to_string()))? - .into_beacon_state() - .ok_or_else(|| Error::DBError("State invalid.".to_string()))?; + debug!("Finding attesatations for new block..."); - debug!("Finding attesatations for block..."); + let attestations = self + .attestation_aggregator + .read() + .get_attestations_for_state(&state, &self.spec); - let attestations = { - let mut next_state = parent_state.clone(); - next_state.per_slot_processing(Hash256::zero(), &self.spec)?; - self.attestation_aggregator - .read() - .unwrap() - .get_attestations_for_state(&next_state, &self.spec) - }; + debug!( + "Inserting {} attestation(s) into new block.", + attestations.len() + ); - debug!("Found {} attestation(s).", attestations.len()); + let parent_root = state + .get_block_root(state.slot.saturating_sub(1), &self.spec) + // TODO: fix unwrap + .unwrap() + .clone(); let mut block = BeaconBlock { - slot: present_slot, - parent_root: parent_root.clone(), + slot: state.slot, + parent_root, state_root: Hash256::zero(), // Updated after the state is calculated. randao_reveal: randao_reveal, eth1_data: Eth1Data { @@ -86,8 +74,8 @@ where }, }; - let state = - self.state_transition_without_verifying_block_signature(parent_state, &block)?; + state.per_block_processing_without_verifying_block_signature(&block, &self.spec)?; + let state_root = state.canonical_root(); block.state_root = state_root; @@ -104,9 +92,15 @@ impl From for Error { } } -impl From for Error { - fn from(e: TransitionError) -> Error { - Error::StateTransitionError(e) +impl From for Error { + fn from(e: SlotProcessingError) -> Error { + Error::SlotProcessingError(e) + } +} + +impl From for Error { + fn from(e: BlockProcessingError) -> Error { + Error::PerBlockProcessingError(e) } } @@ -116,8 +110,8 @@ impl From for Error { } } -impl From for Error { - fn from(e: SlotProcessingError) -> Error { - Error::SlotProcessingError(e) +impl From for Error { + fn from(e: SystemTimeSlotClockError) -> Error { + Error::SlotClockError(e) } } diff --git a/beacon_node/beacon_chain/src/canonical_head.rs b/beacon_node/beacon_chain/src/canonical_head.rs index a92cff327..14f1adf81 100644 --- a/beacon_node/beacon_chain/src/canonical_head.rs +++ b/beacon_node/beacon_chain/src/canonical_head.rs @@ -1,5 +1,5 @@ use crate::{BeaconChain, CheckPoint, ClientDB, SlotClock}; -use std::sync::RwLockReadGuard; +use parking_lot::RwLockReadGuard; use types::{beacon_state::SlotProcessingError, BeaconBlock, BeaconState, Hash256}; #[derive(Debug, PartialEq)] @@ -20,10 +20,7 @@ where new_beacon_state: BeaconState, new_beacon_state_root: Hash256, ) { - let mut head = self - .canonical_head - .write() - .expect("CRITICAL: CanonicalHead poisioned."); + let mut head = self.canonical_head.write(); head.update( new_beacon_block, new_beacon_block_root, @@ -33,34 +30,18 @@ where } pub fn head(&self) -> RwLockReadGuard { - self.canonical_head - .read() - .expect("CRITICAL: CanonicalHead poisioned.") + self.canonical_head.read() } - pub fn state(&self, slot: u64) -> Result { - let mut state = self - .canonical_head - .read() - .expect("CRITICAL: CanonicalHead poisioned.") - .beacon_state - .clone(); - let previous_block_root = self - .canonical_head - .read() - .expect("CRITICAL: CanonicalHead poisioned.") - .beacon_block_root - .clone(); - - match slot.checked_sub(state.slot) { - None => Err(Error::PastSlot), - Some(distance) => { - for _ in 0..distance { - state.per_slot_processing(previous_block_root.clone(), &self.spec)? - } - Ok(state) - } + pub fn advance_state(&self, slot: u64) -> Result<(), SlotProcessingError> { + let state_slot = self.state.read().slot; + let head_block_root = self.head().beacon_block_root; + for _ in state_slot..slot { + self.state + .write() + .per_slot_processing(head_block_root.clone(), &self.spec)?; } + Ok(()) } } diff --git a/beacon_node/beacon_chain/src/finalized_head.rs b/beacon_node/beacon_chain/src/finalized_head.rs index 28c883b4c..76a20a2c2 100644 --- a/beacon_node/beacon_chain/src/finalized_head.rs +++ b/beacon_node/beacon_chain/src/finalized_head.rs @@ -1,5 +1,5 @@ use crate::{BeaconChain, CheckPoint, ClientDB, SlotClock}; -use std::sync::RwLockReadGuard; +use parking_lot::RwLockReadGuard; use types::{BeaconBlock, BeaconState, Hash256}; impl BeaconChain @@ -14,10 +14,7 @@ where new_beacon_state: BeaconState, new_beacon_state_root: Hash256, ) { - let mut finalized_head = self - .finalized_head - .write() - .expect("CRITICAL: finalized_head poisioned."); + let mut finalized_head = self.finalized_head.write(); finalized_head.update( new_beacon_block, new_beacon_block_root, @@ -27,8 +24,6 @@ where } pub fn finalized_head(&self) -> RwLockReadGuard { - self.finalized_head - .read() - .expect("CRITICAL: finalized_head poisioned.") + self.finalized_head.read() } } diff --git a/beacon_node/beacon_chain/src/info.rs b/beacon_node/beacon_chain/src/info.rs index e23aaef78..34ac64c8e 100644 --- a/beacon_node/beacon_chain/src/info.rs +++ b/beacon_node/beacon_chain/src/info.rs @@ -28,10 +28,7 @@ where } pub fn proposer_slots(&self, validator_index: usize) -> Option { - let slot = self.present_slot()?; - let state = self.state(slot).ok()?; - - if let Some(validator) = state.validator_registry.get(validator_index) { + if let Some(validator) = self.state.read().validator_registry.get(validator_index) { Some(validator.proposer_slots) } else { None @@ -46,35 +43,22 @@ where } pub fn block_proposer(&self, slot: u64) -> Result { - // TODO: fix unwrap - let present_slot = self.present_slot().unwrap(); - // TODO: fix unwrap - let state = self.state(present_slot).unwrap(); - let index = state.get_beacon_proposer_index(slot, &self.spec)?; + let index = self + .state + .read() + .get_beacon_proposer_index(slot, &self.spec)?; Ok(index) } pub fn justified_slot(&self) -> u64 { - // TODO: fix unwrap - let present_slot = self.present_slot().unwrap(); - // TODO: fix unwrap - let state = self.state(present_slot).unwrap(); - state.justified_slot - /* - self.justified_head - .read() - .expect("Justified head poisoned") - .beacon_block - .slot - */ + self.state.read().justified_slot } pub fn validator_attestion_slot_and_shard(&self, validator_index: usize) -> Option<(u64, u64)> { - let present_slot = self.present_slot()?; - let state = self.state(present_slot).ok()?; - - let (slot, shard, _committee) = state + let (slot, shard, _committee) = self + .state + .read() .attestation_slot_and_shard_for_validator(validator_index, &self.spec) .ok()?; Some((slot, shard)) diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 5fdadd34f..5c719dddd 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -10,7 +10,7 @@ pub mod dump; mod finalized_head; mod info; mod lmd_ghost; -mod state_transition; +// mod state_transition; use self::attestation_targets::AttestationTargets; use self::block_graph::BlockGraph; @@ -20,9 +20,10 @@ use db::{ ClientDB, DBError, }; use genesis::{genesis_beacon_block, genesis_beacon_state, GenesisError}; +use parking_lot::RwLock; use slot_clock::SlotClock; use ssz::ssz_encode; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use types::{BeaconBlock, BeaconState, ChainSpec, Hash256}; pub use self::block_processing::Outcome as BlockProcessingOutcome; @@ -79,6 +80,7 @@ pub struct BeaconChain { canonical_head: RwLock, finalized_head: RwLock, justified_head: RwLock, + pub state: RwLock, pub latest_attestation_targets: RwLock, pub spec: ChainSpec, } @@ -137,6 +139,7 @@ where slot_clock, block_graph, attestation_aggregator, + state: RwLock::new(genesis_state.clone()), justified_head, finalized_head, canonical_head, diff --git a/beacon_node/beacon_chain/test_harness/benches/state_transition.rs b/beacon_node/beacon_chain/test_harness/benches/state_transition.rs index d78a17720..15184e10c 100644 --- a/beacon_node/beacon_chain/test_harness/benches/state_transition.rs +++ b/beacon_node/beacon_chain/test_harness/benches/state_transition.rs @@ -1,64 +1,80 @@ use criterion::Criterion; -use criterion::{criterion_group, criterion_main}; +use criterion::{black_box, criterion_group, criterion_main, Benchmark}; +use env_logger::{Builder, Env}; use test_harness::BeaconChainHarness; -use types::ChainSpec; +use types::{ChainSpec, Hash256}; fn mid_epoch_state_transition(c: &mut Criterion) { - let validator_count = 2; + Builder::from_env(Env::default().default_filter_or("debug")).init(); + + let validator_count = 1000; let mut rig = BeaconChainHarness::new(ChainSpec::foundation(), validator_count); - let two_and_half_epochs = (rig.spec.epoch_length * 2) + (rig.spec.epoch_length / 2); + let epoch_depth = (rig.spec.epoch_length * 2) + (rig.spec.epoch_length / 2); - for _ in 0..two_and_half_epochs { + for _ in 0..epoch_depth { rig.advance_chain_with_block(); } - let block = rig.advance_chain_without_block(); - let state = rig.beacon_chain.canonical_head().beacon_state.clone(); + let state = rig.beacon_chain.state.read().clone(); + + assert!((state.slot + 1) % rig.spec.epoch_length != 0); c.bench_function("mid-epoch state transition 10k validators", move |b| { - let block = block.clone(); let state = state.clone(); b.iter(|| { - rig.beacon_chain - .state_transition(state.clone(), &block.clone()) + let mut state = state.clone(); + black_box(state.per_slot_processing(Hash256::zero(), &rig.spec)) }) }); } fn epoch_boundary_state_transition(c: &mut Criterion) { - let validator_count = 10_000; + // Builder::from_env(Env::default().default_filter_or("debug")).init(); + + let validator_count = 10000; let mut rig = BeaconChainHarness::new(ChainSpec::foundation(), validator_count); - let three_epochs = rig.spec.epoch_length * 3; + let epoch_depth = rig.spec.epoch_length * 2; - for _ in 0..(three_epochs - 1) { + for _ in 0..(epoch_depth - 1) { rig.advance_chain_with_block(); } - let state = rig.beacon_chain.canonical_head().beacon_state.clone(); - assert_eq!( - state.slot % rig.spec.epoch_length, - rig.spec.epoch_length - 1, - ); - let block = rig.advance_chain_without_block(); + let state = rig.beacon_chain.state.read().clone(); - c.bench_function("epoch boundary state transition 10k validators", move |b| { - let block = block.clone(); + assert_eq!((state.slot + 1) % rig.spec.epoch_length, 0); + + c.bench( + "routines", + Benchmark::new("routine_1", move |b| { + let state = state.clone(); + b.iter(|| { + let mut state = state.clone(); + black_box(black_box( + state.per_slot_processing(Hash256::zero(), &rig.spec), + )) + }) + }) + .sample_size(5), + ); + + /* + c.bench_function("mid-epoch state transition 10k validators", move |b| { let state = state.clone(); b.iter(|| { - let state = rig - .beacon_chain - .state_transition(state.clone(), &block.clone()) - .unwrap(); - assert_eq!(state.slot % rig.spec.epoch_length, 0); + let mut state = state.clone(); + black_box(black_box( + state.per_slot_processing(Hash256::zero(), &rig.spec), + )) }) }); + */ } criterion_group!( benches, - mid_epoch_state_transition, + // mid_epoch_state_transition, epoch_boundary_state_transition ); criterion_main!(benches); diff --git a/beacon_node/beacon_chain/test_harness/src/harness.rs b/beacon_node/beacon_chain/test_harness/src/harness.rs index 3cdfc8024..7d05c7b89 100644 --- a/beacon_node/beacon_chain/test_harness/src/harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/harness.rs @@ -113,6 +113,7 @@ impl BeaconChainHarness { debug!("Incrementing BeaconChain slot to {}.", slot); self.beacon_chain.slot_clock.set_slot(slot); + self.beacon_chain.advance_state(slot).unwrap(); slot } @@ -125,8 +126,8 @@ impl BeaconChainHarness { let attesting_validators = self .beacon_chain - .state(present_slot) - .unwrap() + .state + .read() .get_crosslink_committees_at_slot(present_slot, &self.spec) .unwrap() .iter() @@ -195,13 +196,27 @@ impl BeaconChainHarness { self.beacon_chain.process_block(block).unwrap(); debug!("...block processed by BeaconChain."); + debug!("Producing free attestations..."); + // Produce new attestations. let free_attestations = self.gather_free_attesations(); + + debug!("Processing free attestations..."); + + free_attestations.par_iter().for_each(|free_attestation| { + self.beacon_chain + .process_free_attestation(free_attestation.clone()) + .unwrap(); + }); + + debug!("Free attestations processed."); + /* for free_attestation in free_attestations { self.beacon_chain .process_free_attestation(free_attestation) .unwrap(); } + */ } pub fn chain_dump(&self) -> Result, DumpError> { diff --git a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs index 9c6ce7456..c3e3191f7 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs @@ -13,10 +13,10 @@ where { fn produce_attestation_data( &self, - slot: u64, + _slot: u64, shard: u64, ) -> Result, NodeError> { - match self.beacon_chain.produce_attestation_data(slot, shard) { + match self.beacon_chain.produce_attestation_data(shard) { Ok(attestation_data) => Ok(Some(attestation_data)), Err(e) => Err(NodeError::RemoteFailure(format!("{:?}", e))), } diff --git a/beacon_node/beacon_chain/test_harness/src/validator/mod.rs b/beacon_node/beacon_chain/test_harness/src/validator/mod.rs index 018ffbc86..adb2cdc45 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/mod.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/mod.rs @@ -2,6 +2,7 @@ use attester::{Attester, Error as AttestationPollError}; use beacon_chain::BeaconChain; use block_producer::{BlockProducer, Error as BlockPollError}; use db::MemoryDB; +use log::trace; use signer::TestSigner; use slot_clock::TestingSlotClock; use std::sync::Arc; diff --git a/beacon_node/beacon_chain/test_harness/tests/chain.rs b/beacon_node/beacon_chain/test_harness/tests/chain.rs index f84157533..ece9bd340 100644 --- a/beacon_node/beacon_chain/test_harness/tests/chain.rs +++ b/beacon_node/beacon_chain/test_harness/tests/chain.rs @@ -24,7 +24,7 @@ fn it_can_build_on_genesis_block() { fn it_can_produce_past_first_epoch_boundary() { Builder::from_env(Env::default().default_filter_or("debug")).init(); - let validator_count = 100; + let validator_count = 128 * 1024; debug!("Starting harness build..."); diff --git a/eth2/types/Cargo.toml b/eth2/types/Cargo.toml index a779c7e7e..77cfb6040 100644 --- a/eth2/types/Cargo.toml +++ b/eth2/types/Cargo.toml @@ -12,6 +12,7 @@ hashing = { path = "../utils/hashing" } honey-badger-split = { path = "../utils/honey-badger-split" } integer-sqrt = "0.1" log = "0.4" +rayon = "1.0" rand = "0.5.5" serde = "1.0" serde_derive = "1.0" diff --git a/eth2/types/src/beacon_state/attestation_validation.rs b/eth2/types/src/beacon_state/attestation_validation.rs index c4a1f0d8a..6e6ade34f 100644 --- a/eth2/types/src/beacon_state/attestation_validation.rs +++ b/eth2/types/src/beacon_state/attestation_validation.rs @@ -61,7 +61,7 @@ impl BeaconState { attestation.data.slot + spec.epoch_length >= self.slot, Error::IncludedTooLate ); - if self.justified_slot >= self.slot - (self.slot % spec.epoch_length) { + if attestation.data.slot >= self.current_epoch_start_slot(spec) { ensure!( attestation.data.justified_slot == self.justified_slot, Error::WrongJustifiedSlot diff --git a/beacon_node/beacon_chain/src/state_transition.rs b/eth2/types/src/beacon_state/block_processing.rs similarity index 66% rename from beacon_node/beacon_chain/src/state_transition.rs rename to eth2/types/src/beacon_state/block_processing.rs index 405da055c..df232d6d1 100644 --- a/beacon_node/beacon_chain/src/state_transition.rs +++ b/eth2/types/src/beacon_state/block_processing.rs @@ -1,19 +1,12 @@ -use super::{BeaconChain, ClientDB, DBError, SlotClock}; +use crate::{ + beacon_state::{AttestationValidationError, CommitteesError, SlotProcessingError}, + readers::BeaconBlockReader, + BeaconBlock, BeaconState, ChainSpec, Exit, Fork, Hash256, PendingAttestation, +}; use bls::{PublicKey, Signature}; use hashing::hash; use log::debug; -use slot_clock::{SystemTimeSlotClockError, TestingSlotClockError}; use ssz::{ssz_encode, TreeHash}; -use types::{ - beacon_state::{AttestationValidationError, CommitteesError, SlotProcessingError}, - readers::BeaconBlockReader, - BeaconBlock, BeaconState, Exit, Fork, Hash256, PendingAttestation, -}; - -// TODO: define elsehwere. -const DOMAIN_PROPOSAL: u64 = 2; -const DOMAIN_EXIT: u64 = 3; -const DOMAIN_RANDAO: u64 = 4; macro_rules! ensure { ($condition: expr, $result: expr) => { @@ -23,6 +16,11 @@ macro_rules! ensure { }; } +// TODO: define elsehwere. +const DOMAIN_PROPOSAL: u64 = 2; +const DOMAIN_EXIT: u64 = 3; +const DOMAIN_RANDAO: u64 = 4; + #[derive(Debug, PartialEq)] pub enum Error { DBError(String), @@ -50,71 +48,50 @@ pub enum Error { BadCustodyReseeds, BadCustodyChallenges, BadCustodyResponses, - SlotClockError(SystemTimeSlotClockError), CommitteesError(CommitteesError), SlotProcessingError(SlotProcessingError), } -impl BeaconChain -where - T: ClientDB, - U: SlotClock, -{ - pub fn state_transition( - &self, - state: BeaconState, +impl BeaconState { + pub fn per_block_processing( + &mut self, block: &BeaconBlock, - ) -> Result { - self.internal_state_transition(state, block, true) + spec: &ChainSpec, + ) -> Result<(), Error> { + self.per_block_processing_signature_optional(block, true, spec) } - pub fn state_transition_without_verifying_block_signature( - &self, - state: BeaconState, + pub fn per_block_processing_without_verifying_block_signature( + &mut self, block: &BeaconBlock, - ) -> Result { - self.internal_state_transition(state, block, false) + spec: &ChainSpec, + ) -> Result<(), Error> { + self.per_block_processing_signature_optional(block, false, spec) } - fn internal_state_transition( - &self, - mut state: BeaconState, + fn per_block_processing_signature_optional( + &mut self, block: &BeaconBlock, verify_block_signature: bool, - ) -> Result { - ensure!(state.slot < block.slot, Error::StateAlreadyTransitioned); - - debug!( - "Starting state transition from slot {} to {}...", - state.slot, block.slot - ); - - for _ in state.slot..block.slot { - state.per_slot_processing(block.parent_root.clone(), &self.spec)?; - } - - /* - * Slot - */ - - ensure!(block.slot() == state.slot, Error::StateSlotMismatch); + spec: &ChainSpec, + ) -> Result<(), Error> { + ensure!(block.slot() == self.slot, Error::StateSlotMismatch); /* * Proposer Signature */ - - let block_proposer_index = state - .get_beacon_proposer_index(block.slot, &self.spec) + let block_proposer_index = self + .get_beacon_proposer_index(block.slot, spec) .map_err(|_| Error::NoBlockProducer)?; - let block_proposer = &state.validator_registry[block_proposer_index]; + let block_proposer = &self.validator_registry[block_proposer_index]; if verify_block_signature { ensure!( bls_verify( &block_proposer.pubkey, - &block.proposal_root(&self.spec)[..], + &block.proposal_root(spec)[..], &block.signature, - get_domain(&state.fork_data, state.slot, DOMAIN_PROPOSAL) + get_domain(&self.fork_data, self.slot, DOMAIN_PROPOSAL) ), Error::BadBlockSignature ); @@ -123,49 +100,42 @@ where /* * RANDAO */ - ensure!( bls_verify( &block_proposer.pubkey, &ssz_encode(&block_proposer.proposer_slots), &block.randao_reveal, - get_domain(&state.fork_data, state.slot, DOMAIN_RANDAO) + get_domain(&self.fork_data, self.slot, DOMAIN_RANDAO) ), Error::BadRandaoSignature ); // TODO: check this is correct. let new_mix = { - let mut mix = state.latest_randao_mixes - [(state.slot % self.spec.latest_randao_mixes_length) as usize] + let mut mix = self.latest_randao_mixes + [(self.slot % spec.latest_randao_mixes_length) as usize] .to_vec(); mix.append(&mut ssz_encode(&block.randao_reveal)); Hash256::from(&hash(&mix)[..]) }; - state.latest_randao_mixes[(state.slot % self.spec.latest_randao_mixes_length) as usize] = - new_mix; + self.latest_randao_mixes[(self.slot % spec.latest_randao_mixes_length) as usize] = new_mix; /* * Eth1 data */ - // TODO: Eth1 data stuff. - - /* - * OPERATIONS - */ + // TODO: Eth1 data processing. /* * Proposer slashings */ - ensure!( - block.body.proposer_slashings.len() as u64 <= self.spec.max_proposer_slashings, + block.body.proposer_slashings.len() as u64 <= spec.max_proposer_slashings, Error::MaxProposerSlashingsExceeded ); for proposer_slashing in &block.body.proposer_slashings { - let proposer = state + let proposer = self .validator_registry .get(proposer_slashing.proposer_index as usize) .ok_or(Error::BadProposerSlashing)?; @@ -183,7 +153,7 @@ where Error::BadProposerSlashing ); ensure!( - proposer.penalized_slot > state.slot, + proposer.penalized_slot > self.slot, Error::BadProposerSlashing ); ensure!( @@ -192,7 +162,7 @@ where &proposer_slashing.proposal_data_1.hash_tree_root(), &proposer_slashing.proposal_signature_1, get_domain( - &state.fork_data, + &self.fork_data, proposer_slashing.proposal_data_1.slot, DOMAIN_PROPOSAL ) @@ -205,34 +175,34 @@ where &proposer_slashing.proposal_data_2.hash_tree_root(), &proposer_slashing.proposal_signature_2, get_domain( - &state.fork_data, + &self.fork_data, proposer_slashing.proposal_data_2.slot, DOMAIN_PROPOSAL ) ), Error::BadProposerSlashing ); - penalize_validator(&state, proposer_slashing.proposer_index as usize); + penalize_validator(&self, proposer_slashing.proposer_index as usize); } /* * Attestations */ ensure!( - block.body.attestations.len() as u64 <= self.spec.max_attestations, + block.body.attestations.len() as u64 <= spec.max_attestations, Error::MaxAttestationsExceeded ); for attestation in &block.body.attestations { - state.validate_attestation(attestation, &self.spec)?; + self.validate_attestation(attestation, spec)?; let pending_attestation = PendingAttestation { data: attestation.data.clone(), aggregation_bitfield: attestation.aggregation_bitfield.clone(), custody_bitfield: attestation.custody_bitfield.clone(), - slot_included: state.slot, + slot_included: self.slot, }; - state.latest_attestations.push(pending_attestation); + self.latest_attestations.push(pending_attestation); } debug!( @@ -244,7 +214,7 @@ where * Deposits */ ensure!( - block.body.deposits.len() as u64 <= self.spec.max_deposits, + block.body.deposits.len() as u64 <= spec.max_deposits, Error::MaxDepositsExceeded ); @@ -255,25 +225,25 @@ where */ ensure!( - block.body.exits.len() as u64 <= self.spec.max_exits, + block.body.exits.len() as u64 <= spec.max_exits, Error::MaxExitsExceeded ); for exit in &block.body.exits { - let validator = state + let validator = self .validator_registry .get(exit.validator_index as usize) .ok_or(Error::BadExit)?; ensure!( - validator.exit_slot > state.slot + self.spec.entry_exit_delay, + validator.exit_slot > self.slot + spec.entry_exit_delay, Error::BadExit ); - ensure!(state.slot >= exit.slot, Error::BadExit); + ensure!(self.slot >= exit.slot, Error::BadExit); let exit_message = { let exit_struct = Exit { slot: exit.slot, validator_index: exit.validator_index, - signature: self.spec.empty_signature.clone(), + signature: spec.empty_signature.clone(), }; exit_struct.hash_tree_root() }; @@ -282,11 +252,11 @@ where &validator.pubkey, &exit_message, &exit.signature, - get_domain(&state.fork_data, exit.slot, DOMAIN_EXIT) + get_domain(&self.fork_data, exit.slot, DOMAIN_EXIT) ), Error::BadProposerSlashing ); - initiate_validator_exit(&state, exit.validator_index); + initiate_validator_exit(&self, exit.validator_index); } /* @@ -307,7 +277,7 @@ where debug!("State transition complete."); - Ok(state) + Ok(()) } } @@ -329,24 +299,6 @@ fn bls_verify(pubkey: &PublicKey, message: &[u8], signature: &Signature, _domain signature.verify(message, pubkey) } -impl From for Error { - fn from(e: DBError) -> Error { - Error::DBError(e.message) - } -} - -impl From for Error { - fn from(_: TestingSlotClockError) -> Error { - unreachable!(); // Testing clock never throws an error. - } -} - -impl From for Error { - fn from(e: SystemTimeSlotClockError) -> Error { - Error::SlotClockError(e) - } -} - impl From for Error { fn from(e: AttestationValidationError) -> Error { Error::InvalidAttestation(e) diff --git a/eth2/types/src/beacon_state/epoch_processing.rs b/eth2/types/src/beacon_state/epoch_processing.rs index 296c2e848..d93cd4fe5 100644 --- a/eth2/types/src/beacon_state/epoch_processing.rs +++ b/eth2/types/src/beacon_state/epoch_processing.rs @@ -7,6 +7,7 @@ use crate::{ }; use integer_sqrt::IntegerSquareRoot; use log::debug; +use rayon::prelude::*; use std::collections::{HashMap, HashSet}; use std::iter::FromIterator; @@ -59,7 +60,7 @@ impl BeaconState { let current_epoch_attestations: Vec<&PendingAttestation> = self .latest_attestations - .iter() + .par_iter() .filter(|a| a.data.slot / spec.epoch_length == self.current_epoch(spec)) .collect(); @@ -77,7 +78,7 @@ impl BeaconState { let current_epoch_boundary_attestations: Vec<&PendingAttestation> = current_epoch_attestations - .iter() + .par_iter() .filter(|a| { match self.get_block_root(self.current_epoch_start_slot(spec), spec) { Some(block_root) => { @@ -112,7 +113,7 @@ impl BeaconState { */ let previous_epoch_attestations: Vec<&PendingAttestation> = self .latest_attestations - .iter() + .par_iter() .filter(|a| { //TODO: ensure these saturating subs are correct. a.data.slot / spec.epoch_length == self.previous_epoch(spec) diff --git a/eth2/types/src/beacon_state/mod.rs b/eth2/types/src/beacon_state/mod.rs index 3972eaa82..f0548914e 100644 --- a/eth2/types/src/beacon_state/mod.rs +++ b/eth2/types/src/beacon_state/mod.rs @@ -12,6 +12,7 @@ use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; mod attestation_participants; mod attestation_validation; +mod block_processing; mod committees; mod epoch_processing; mod shuffling; @@ -20,6 +21,7 @@ mod winning_root; pub use self::attestation_participants::Error as AttestationParticipantsError; pub use self::attestation_validation::Error as AttestationValidationError; +pub use self::block_processing::Error as BlockProcessingError; pub use self::committees::Error as CommitteesError; pub use self::epoch_processing::Error as EpochProcessingError; pub use self::slot_processing::Error as SlotProcessingError; From a86f7fa51b82c0463013596be8c2b4ca618aa3dc Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Feb 2019 15:09:30 +1100 Subject: [PATCH 075/107] Tidy `AttestationAggregator`, add docstrings. --- .../src/attestation_aggregator.rs | 56 +++++++++++++++---- .../src/attestation_processing.rs | 2 +- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_aggregator.rs b/beacon_node/beacon_chain/src/attestation_aggregator.rs index 40b574399..855cb711b 100644 --- a/beacon_node/beacon_chain/src/attestation_aggregator.rs +++ b/beacon_node/beacon_chain/src/attestation_aggregator.rs @@ -6,63 +6,91 @@ use types::{ const PHASE_0_CUSTODY_BIT: bool = false; +/// Provides the functionality to: +/// +/// - Recieve a `FreeAttestation` and aggregate it into an `Attestation` (or create a new if it +/// doesn't exist). +/// - Store all aggregated or created `Attestation`s. +/// - Produce a list of attestations that would be valid for inclusion in some `BeaconState` (and +/// therefore valid for inclusion in a `BeaconBlock`. +/// +/// Note: `Attestations` are stored in memory and never deleted. This is not scalable and must be +/// rectified in a future revision. pub struct AttestationAggregator { store: HashMap, Attestation>, } +/// The outcome of sucessfully processing a `FreeAttestation`. #[derive(Debug, PartialEq)] pub enum ProcessOutcome { - AggregationNotRequired, + /// The free attestation was added to an existing attestation. Aggregated, + /// The free attestation has already been aggregated to an existing attestation. + AggregationNotRequired, + /// The free attestation was transformed into a new attestation. NewAttestationCreated, } #[derive(Debug, PartialEq)] -pub enum ProcessError { +pub enum Error { + /// The supplied `validator_index` is not in the committee for the given `shard` and `slot`. BadValidatorIndex, + /// The given `signature` did not match the `pubkey` in the given + /// `state.validator_registry`. BadSignature, + /// The given `slot` does not match the validators committee assignment. BadSlot, + /// The given `shard` does not match the validators committee assignment. BadShard, + /// There was an error finding the committee for the given `validator_index`. CommitteesError(CommitteesError), } impl AttestationAggregator { + /// Instantiates a new AttestationAggregator with an empty database. pub fn new() -> Self { Self { store: HashMap::new(), } } + /// Accepts some `FreeAttestation`, validates it and either aggregates it upon some existing + /// `Attestation` or produces a new `Attestation`. + /// + /// The "validation" provided is not complete, instead the following points are checked: + /// - The given `validator_index` is in the committee for the given `shard` for the given + /// `slot`. + /// - The signature is verified against that of the validator at `validator_index`. pub fn process_free_attestation( &mut self, state: &BeaconState, free_attestation: &FreeAttestation, spec: &ChainSpec, - ) -> Result { + ) -> Result { let (slot, shard, committee_index) = state.attestation_slot_and_shard_for_validator( free_attestation.validator_index as usize, spec, )?; if free_attestation.data.slot != slot { - return Err(ProcessError::BadSlot); + return Err(Error::BadSlot); } if free_attestation.data.shard != shard { - return Err(ProcessError::BadShard); + return Err(Error::BadShard); } let signable_message = free_attestation.data.signable_message(PHASE_0_CUSTODY_BIT); let validator_pubkey = &state .validator_registry .get(free_attestation.validator_index as usize) - .ok_or_else(|| ProcessError::BadValidatorIndex)? + .ok_or_else(|| Error::BadValidatorIndex)? .pubkey; if !free_attestation .signature .verify(&signable_message, &validator_pubkey) { - return Err(ProcessError::BadSignature); + return Err(Error::BadSignature); } if let Some(existing_attestation) = self.store.get(&signable_message) { @@ -94,8 +122,8 @@ impl AttestationAggregator { /// Returns all known attestations which are: /// - /// a) valid for the given state - /// b) not already in `state.latest_attestations`. + /// - Valid for the given state + /// - Not already in `state.latest_attestations`. pub fn get_attestations_for_state( &self, state: &BeaconState, @@ -124,6 +152,10 @@ impl AttestationAggregator { } } +/// Produces a new `Attestation` where: +/// +/// - `signature` is added to `Attestation.aggregate_signature` +/// - Attestation.aggregation_bitfield[committee_index]` is set to true. fn aggregate_attestation( existing_attestation: &Attestation, signature: &Signature, @@ -150,8 +182,8 @@ fn aggregate_attestation( } } -impl From for ProcessError { - fn from(e: CommitteesError) -> ProcessError { - ProcessError::CommitteesError(e) +impl From for Error { + fn from(e: CommitteesError) -> Error { + Error::CommitteesError(e) } } diff --git a/beacon_node/beacon_chain/src/attestation_processing.rs b/beacon_node/beacon_chain/src/attestation_processing.rs index 1b7e8ace4..bc7e2cda1 100644 --- a/beacon_node/beacon_chain/src/attestation_processing.rs +++ b/beacon_node/beacon_chain/src/attestation_processing.rs @@ -1,5 +1,5 @@ use super::{BeaconChain, ClientDB, SlotClock}; -pub use crate::attestation_aggregator::{ProcessError as AggregatorError, ProcessOutcome}; +pub use crate::attestation_aggregator::{Error as AggregatorError, ProcessOutcome}; use crate::canonical_head::Error as HeadError; use types::FreeAttestation; From 2ed5f694484d0b52ef471bece888b11458fe4a18 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Feb 2019 15:19:50 +1100 Subject: [PATCH 076/107] Tidy attestation_processing, add docstrings. --- .../beacon_chain/src/attestation_processing.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_processing.rs b/beacon_node/beacon_chain/src/attestation_processing.rs index bc7e2cda1..4470c3d22 100644 --- a/beacon_node/beacon_chain/src/attestation_processing.rs +++ b/beacon_node/beacon_chain/src/attestation_processing.rs @@ -1,13 +1,11 @@ use super::{BeaconChain, ClientDB, SlotClock}; pub use crate::attestation_aggregator::{Error as AggregatorError, ProcessOutcome}; -use crate::canonical_head::Error as HeadError; use types::FreeAttestation; #[derive(Debug, PartialEq)] pub enum Error { - PresentSlotUnknown, + /// The free attestation was not processed succesfully. AggregatorError(AggregatorError), - HeadError(HeadError), } impl BeaconChain @@ -15,6 +13,10 @@ where T: ClientDB, U: SlotClock, { + /// Validate a `FreeAttestation` and either: + /// + /// - Create a new `Attestation`. + /// - Aggregate it to an existing `Attestation`. pub fn process_free_attestation( &self, free_attestation: FreeAttestation, @@ -31,9 +33,3 @@ impl From for Error { Error::AggregatorError(e) } } - -impl From for Error { - fn from(e: HeadError) -> Error { - Error::HeadError(e) - } -} From 92753fa24e8d53b868df06bbb3b868d2f126a94c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Feb 2019 15:26:36 +1100 Subject: [PATCH 077/107] Tidy attestation_production and create docstrings --- .../beacon_chain/src/attestation_production.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/beacon_node/beacon_chain/src/attestation_production.rs b/beacon_node/beacon_chain/src/attestation_production.rs index febd2ff9a..40284877e 100644 --- a/beacon_node/beacon_chain/src/attestation_production.rs +++ b/beacon_node/beacon_chain/src/attestation_production.rs @@ -3,9 +3,10 @@ use types::{AttestationData, Hash256}; #[derive(Debug, PartialEq)] pub enum Error { - SlotTooOld, - PresentSlotUnknown, - StateError, + /// The `justified_block_root` is unknown. This is an internal error. + UnknownJustifiedRoot, + /// The `epoch_boundary_root` is unknown. This is an internal error. + UnknownBoundaryRoot, } impl BeaconChain @@ -13,13 +14,14 @@ where T: ClientDB, U: SlotClock, { + /// Produce an `AttestationData` that is valid for the present `slot` and given `shard`. pub fn produce_attestation_data(&self, shard: u64) -> Result { let justified_slot = self.justified_slot(); let justified_block_root = self .state .read() .get_block_root(justified_slot, &self.spec) - .ok_or_else(|| Error::SlotTooOld)? + .ok_or_else(|| Error::UnknownJustifiedRoot)? .clone(); let epoch_boundary_root = self @@ -29,7 +31,7 @@ where self.state.read().current_epoch_start_slot(&self.spec), &self.spec, ) - .ok_or_else(|| Error::SlotTooOld)? + .ok_or_else(|| Error::UnknownBoundaryRoot)? .clone(); Ok(AttestationData { From d50a8b03f80f136f3675a938ec239a5794d09bbc Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Feb 2019 15:37:43 +1100 Subject: [PATCH 078/107] Set BeaconChain `present_slot` to read from state. It used to read from the slot_clock, that has been replaced with `read_slot_clock`. --- .../beacon_chain/src/block_processing.rs | 9 +-------- beacon_node/beacon_chain/src/info.rs | 6 +++++- .../beacon_chain/test_harness/src/harness.rs | 17 +++-------------- 3 files changed, 9 insertions(+), 23 deletions(-) diff --git a/beacon_node/beacon_chain/src/block_processing.rs b/beacon_node/beacon_chain/src/block_processing.rs index 2c637031e..6dd48abda 100644 --- a/beacon_node/beacon_chain/src/block_processing.rs +++ b/beacon_node/beacon_chain/src/block_processing.rs @@ -29,7 +29,6 @@ pub enum Outcome { pub enum Error { DBError(String), UnableToDecodeBlock, - PresentSlotIsNone, SlotClockError(SystemTimeSlotClockError), MissingParentState(Hash256), InvalidParentState(Hash256), @@ -57,25 +56,19 @@ where .ok_or(Error::UnableToDecodeBlock)?; let block_root = block.canonical_root(); - let present_slot = self - .slot_clock - .present_slot()? - .ok_or(Error::PresentSlotIsNone)?; + let present_slot = self.present_slot(); - // Block from future slots (i.e., greater than the present slot) should not be processed. if block.slot() > present_slot { return Ok(Outcome::InvalidBlock(InvalidBlock::FutureSlot)); } let parent_block_root = block.parent_root(); - let parent_block = self .block_store .get_reader(&parent_block_root)? .ok_or(Error::MissingParentBlock(parent_block_root))?; let parent_state_root = parent_block.state_root(); - let parent_state = self .state_store .get_reader(&parent_state_root)? diff --git a/beacon_node/beacon_chain/src/info.rs b/beacon_node/beacon_chain/src/info.rs index 34ac64c8e..70405ab32 100644 --- a/beacon_node/beacon_chain/src/info.rs +++ b/beacon_node/beacon_chain/src/info.rs @@ -35,13 +35,17 @@ where } } - pub fn present_slot(&self) -> Option { + pub fn read_slot_clock(&self) -> Option { match self.slot_clock.present_slot() { Ok(some_slot) => some_slot, _ => None, } } + pub fn present_slot(&self) -> u64 { + self.state.read().slot + } + pub fn block_proposer(&self, slot: u64) -> Result { let index = self .state diff --git a/beacon_node/beacon_chain/test_harness/src/harness.rs b/beacon_node/beacon_chain/test_harness/src/harness.rs index 7d05c7b89..ce88395cb 100644 --- a/beacon_node/beacon_chain/test_harness/src/harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/harness.rs @@ -104,11 +104,7 @@ impl BeaconChainHarness { /// /// Returns the new slot. pub fn increment_beacon_chain_slot(&mut self) -> u64 { - let slot = self - .beacon_chain - .present_slot() - .expect("Unable to determine slot.") - + 1; + let slot = self.beacon_chain.present_slot() + 1; debug!("Incrementing BeaconChain slot to {}.", slot); @@ -122,7 +118,7 @@ impl BeaconChainHarness { /// Note: validators will only produce attestations _once per slot_. So, if you call this twice /// you'll only get attestations on the first run. pub fn gather_free_attesations(&mut self) -> Vec { - let present_slot = self.beacon_chain.present_slot().unwrap(); + let present_slot = self.beacon_chain.present_slot(); let attesting_validators = self .beacon_chain @@ -169,7 +165,7 @@ impl BeaconChainHarness { /// Note: the validator will only produce it _once per slot_. So, if you call this twice you'll /// only get a block once. pub fn produce_block(&mut self) -> BeaconBlock { - let present_slot = self.beacon_chain.present_slot().unwrap(); + let present_slot = self.beacon_chain.present_slot(); let proposer = self.beacon_chain.block_proposer(present_slot).unwrap(); @@ -210,13 +206,6 @@ impl BeaconChainHarness { }); debug!("Free attestations processed."); - /* - for free_attestation in free_attestations { - self.beacon_chain - .process_free_attestation(free_attestation) - .unwrap(); - } - */ } pub fn chain_dump(&self) -> Result, DumpError> { From d4757f2e09f08901e723865c85d050b6c6b65e38 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Feb 2019 15:39:30 +1100 Subject: [PATCH 079/107] Drop test repetitions down to 100 --- beacon_node/beacon_chain/test_harness/tests/chain.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/test_harness/tests/chain.rs b/beacon_node/beacon_chain/test_harness/tests/chain.rs index ece9bd340..f84157533 100644 --- a/beacon_node/beacon_chain/test_harness/tests/chain.rs +++ b/beacon_node/beacon_chain/test_harness/tests/chain.rs @@ -24,7 +24,7 @@ fn it_can_build_on_genesis_block() { fn it_can_produce_past_first_epoch_boundary() { Builder::from_env(Env::default().default_filter_or("debug")).init(); - let validator_count = 128 * 1024; + let validator_count = 100; debug!("Starting harness build..."); From 1e6f85a5eb5d788683bee33af1178f8fadf2a2cf Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Feb 2019 15:59:12 +1100 Subject: [PATCH 080/107] Tidy BlockGraph, use parking_lot, add doc comments --- beacon_node/beacon_chain/src/block_graph.rs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/beacon_node/beacon_chain/src/block_graph.rs b/beacon_node/beacon_chain/src/block_graph.rs index b031c2c10..5af851243 100644 --- a/beacon_node/beacon_chain/src/block_graph.rs +++ b/beacon_node/beacon_chain/src/block_graph.rs @@ -1,12 +1,22 @@ +use parking_lot::{RwLock, RwLockReadGuard}; use std::collections::HashSet; -use std::sync::{RwLock, RwLockReadGuard}; use types::Hash256; +/// Maintains a view of the block DAG, also known as the "blockchain" (except, it tracks multiple +/// chains eminating from a single root instead of just the head of some canonical chain). +/// +/// The BlockGraph does not store the blocks, instead it tracks the block hashes of blocks at the +/// tip of the DAG. It is out of the scope of the object to retrieve blocks. +/// +/// Presently, the DAG root (genesis block) is not tracked. +/// +/// The BlogGraph is thread-safe due to internal RwLocks. pub struct BlockGraph { pub leaves: RwLock>, } impl BlockGraph { + /// Create a new block graph without any leaves. pub fn new() -> Self { Self { leaves: RwLock::new(HashSet::new()), @@ -15,10 +25,7 @@ impl BlockGraph { /// Add a new leaf to the block hash graph. Returns `true` if the leaf was built upon another /// leaf. pub fn add_leaf(&self, parent: &Hash256, leaf: Hash256) -> bool { - let mut leaves = self - .leaves - .write() - .expect("CRITICAL: BlockGraph poisioned."); + let mut leaves = self.leaves.write(); if leaves.contains(parent) { leaves.remove(parent); @@ -30,7 +37,8 @@ impl BlockGraph { } } + /// Returns a read-guarded HashSet of all leaf blocks. pub fn leaves(&self) -> RwLockReadGuard> { - self.leaves.read().expect("CRITICAL: BlockGraph poisioned.") + self.leaves.read() } } From 9d1f98ba8fe49fbaedac1074cb5370e2ef2550c1 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Feb 2019 16:07:59 +1100 Subject: [PATCH 081/107] Delete SlotClock errs from block_processing, tidy. --- .../beacon_chain/src/block_processing.rs | 39 +++++++++++-------- .../src/validator/beacon_node/attester.rs | 2 - .../src/validator/beacon_node/producer.rs | 2 - .../test_harness/src/validator/mod.rs | 1 - 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/beacon_node/beacon_chain/src/block_processing.rs b/beacon_node/beacon_chain/src/block_processing.rs index 6dd48abda..249bd9df7 100644 --- a/beacon_node/beacon_chain/src/block_processing.rs +++ b/beacon_node/beacon_chain/src/block_processing.rs @@ -1,6 +1,5 @@ use super::{BeaconChain, ClientDB, DBError, SlotClock}; use log::debug; -use slot_clock::{SystemTimeSlotClockError, TestingSlotClockError}; use ssz::{ssz_encode, Encodable}; use types::{ beacon_state::{BlockProcessingError, SlotProcessingError}, @@ -10,32 +9,46 @@ use types::{ #[derive(Debug, PartialEq)] pub enum ValidBlock { + /// The block was sucessfully processed. Processed, } #[derive(Debug, PartialEq)] pub enum InvalidBlock { + /// The block slot is greater than the present slot. FutureSlot, + /// The block state_root does not match the generated state. StateRootMismatch, } #[derive(Debug, PartialEq)] pub enum Outcome { + /// The block was sucessfully validated. ValidBlock(ValidBlock), + /// The block was not sucessfully validated. InvalidBlock(InvalidBlock), } #[derive(Debug, PartialEq)] pub enum Error { + /// There was in internal database error. DBError(String), + /// The block SSZ encoding is unreadable. UnableToDecodeBlock, - SlotClockError(SystemTimeSlotClockError), + /// The blocks parent state is not in the database. This is an internal error. MissingParentState(Hash256), + /// The blocks parent state is in the database, but invalid. This is an internal error. InvalidParentState(Hash256), + /// The blocks parent state is in the database, but invalid. This is an internal error. MissingBeaconBlock(Hash256), + /// The parent block is not in the database. The block should not be processed. InvalidBeaconBlock(Hash256), + /// The parent block is not in the database, but invalid. This is an internal error. MissingParentBlock(Hash256), + /// There was an error whilst advancing the parent state to the present slot. This is an + /// internal error. SlotProcessingError(SlotProcessingError), + /// There was an error whilst processing the block against it's state. The block is invalid. PerBlockProcessingError(BlockProcessingError), } @@ -43,8 +56,10 @@ impl BeaconChain where T: ClientDB, U: SlotClock, - Error: From<::Error>, { + /// Accept some block and attempt to add it to block DAG. + /// + /// Will accept blocks from prior slots, however it will reject any block from a future slot. pub fn process_block(&self, block: V) -> Result where V: BeaconBlockReader + Encodable + Sized, @@ -94,12 +109,14 @@ where self.block_store.put(&block_root, &ssz_encode(&block)[..])?; self.state_store.put(&state_root, &ssz_encode(&state)[..])?; + // Update the block DAG. self.block_graph .add_leaf(&parent_block_root, block_root.clone()); // If the parent block was the parent_block, automatically update the canonical head. // - // TODO: this is a first-in-best-dressed scenario that is not ideal -- find a solution. + // TODO: this is a first-in-best-dressed scenario that is not ideal; fork_choice should be + // run instead. if self.head().beacon_block_root == parent_block_root { self.update_canonical_head( block.clone(), @@ -107,10 +124,10 @@ where state.clone(), state_root.clone(), ); + // Update the local state variable. *self.state.write() = state.clone(); } - // The block was sucessfully processed. Ok(Outcome::ValidBlock(ValidBlock::Processed)) } } @@ -132,15 +149,3 @@ impl From for Error { Error::PerBlockProcessingError(e) } } - -impl From for Error { - fn from(_: TestingSlotClockError) -> Error { - unreachable!(); // Testing clock never throws an error. - } -} - -impl From for Error { - fn from(e: SystemTimeSlotClockError) -> Error { - Error::SlotClockError(e) - } -} diff --git a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs index c3e3191f7..bcfb929ea 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs @@ -1,6 +1,5 @@ use super::BenchingBeaconNode; use attester::{BeaconNode as AttesterBeaconNode, BeaconNodeError as NodeError, PublishOutcome}; -use beacon_chain::block_processing::Error as ProcessingError; use beacon_chain::block_production::Error as BlockProductionError; use db::ClientDB; use slot_clock::SlotClock; @@ -9,7 +8,6 @@ use types::{AttestationData, FreeAttestation}; impl AttesterBeaconNode for BenchingBeaconNode where BlockProductionError: From<::Error>, - ProcessingError: From<::Error>, { fn produce_attestation_data( &self, diff --git a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/producer.rs b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/producer.rs index 70ca4ff0c..9d0b6b91f 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/producer.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/producer.rs @@ -1,5 +1,4 @@ use super::BenchingBeaconNode; -use beacon_chain::block_processing::Error as ProcessingError; use beacon_chain::block_production::Error as BlockProductionError; use block_producer::{ BeaconNode as BeaconBlockNode, BeaconNodeError as BeaconBlockNodeError, PublishOutcome, @@ -11,7 +10,6 @@ use types::{BeaconBlock, PublicKey, Signature}; impl BeaconBlockNode for BenchingBeaconNode where BlockProductionError: From<::Error>, - ProcessingError: From<::Error>, { /// Requests the `proposer_nonce` from the `BeaconChain`. fn proposer_nonce(&self, pubkey: &PublicKey) -> Result { diff --git a/beacon_node/beacon_chain/test_harness/src/validator/mod.rs b/beacon_node/beacon_chain/test_harness/src/validator/mod.rs index adb2cdc45..018ffbc86 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/mod.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/mod.rs @@ -2,7 +2,6 @@ use attester::{Attester, Error as AttestationPollError}; use beacon_chain::BeaconChain; use block_producer::{BlockProducer, Error as BlockPollError}; use db::MemoryDB; -use log::trace; use signer::TestSigner; use slot_clock::TestingSlotClock; use std::sync::Arc; From db230475d7bd7b7511e28b2221a7054511b81905 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Feb 2019 16:21:18 +1100 Subject: [PATCH 082/107] Remove SlotClock error from block production, tidy. --- .../beacon_chain/src/block_production.rs | 25 ++++--------------- .../src/validator/beacon_node/attester.rs | 6 +---- .../src/validator/beacon_node/producer.rs | 6 +---- .../src/validator/direct_duties.rs | 12 +++------ 4 files changed, 10 insertions(+), 39 deletions(-) diff --git a/beacon_node/beacon_chain/src/block_production.rs b/beacon_node/beacon_chain/src/block_production.rs index 6c1fa10c4..eecf1af06 100644 --- a/beacon_node/beacon_chain/src/block_production.rs +++ b/beacon_node/beacon_chain/src/block_production.rs @@ -1,20 +1,16 @@ use super::{BeaconChain, ClientDB, DBError, SlotClock}; use bls::Signature; use log::debug; -use slot_clock::{SystemTimeSlotClockError, TestingSlotClockError}; use types::{ beacon_state::{BlockProcessingError, SlotProcessingError}, - readers::{BeaconBlockReader, BeaconStateReader}, BeaconBlock, BeaconBlockBody, BeaconState, Eth1Data, Hash256, }; #[derive(Debug, PartialEq)] pub enum Error { DBError(String), - PresentSlotIsNone, SlotProcessingError(SlotProcessingError), PerBlockProcessingError(BlockProcessingError), - SlotClockError(SystemTimeSlotClockError), } impl BeaconChain @@ -22,13 +18,14 @@ where T: ClientDB, U: SlotClock, { + /// Produce a new block at the present slot. + /// + /// The produced block will not be inheriently valid, it must be signed by a block producer. + /// Block signing is out of the scope of this function and should be done by a separate program. pub fn produce_block( &self, randao_reveal: Signature, - ) -> Result<(BeaconBlock, BeaconState), Error> - where - Error: From<::Error>, - { + ) -> Result<(BeaconBlock, BeaconState), Error> { debug!("Starting block production..."); let mut state = self.state.read().clone(); @@ -103,15 +100,3 @@ impl From for Error { Error::PerBlockProcessingError(e) } } - -impl From for Error { - fn from(_: TestingSlotClockError) -> Error { - unreachable!(); // Testing clock never throws an error. - } -} - -impl From for Error { - fn from(e: SystemTimeSlotClockError) -> Error { - Error::SlotClockError(e) - } -} diff --git a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs index bcfb929ea..19225b671 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs @@ -1,14 +1,10 @@ use super::BenchingBeaconNode; use attester::{BeaconNode as AttesterBeaconNode, BeaconNodeError as NodeError, PublishOutcome}; -use beacon_chain::block_production::Error as BlockProductionError; use db::ClientDB; use slot_clock::SlotClock; use types::{AttestationData, FreeAttestation}; -impl AttesterBeaconNode for BenchingBeaconNode -where - BlockProductionError: From<::Error>, -{ +impl AttesterBeaconNode for BenchingBeaconNode { fn produce_attestation_data( &self, _slot: u64, diff --git a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/producer.rs b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/producer.rs index 9d0b6b91f..861acd25e 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/producer.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/producer.rs @@ -1,5 +1,4 @@ use super::BenchingBeaconNode; -use beacon_chain::block_production::Error as BlockProductionError; use block_producer::{ BeaconNode as BeaconBlockNode, BeaconNodeError as BeaconBlockNodeError, PublishOutcome, }; @@ -7,10 +6,7 @@ use db::ClientDB; use slot_clock::SlotClock; use types::{BeaconBlock, PublicKey, Signature}; -impl BeaconBlockNode for BenchingBeaconNode -where - BlockProductionError: From<::Error>, -{ +impl BeaconBlockNode for BenchingBeaconNode { /// Requests the `proposer_nonce` from the `BeaconChain`. fn proposer_nonce(&self, pubkey: &PublicKey) -> Result { let validator_index = self diff --git a/beacon_node/beacon_chain/test_harness/src/validator/direct_duties.rs b/beacon_node/beacon_chain/test_harness/src/validator/direct_duties.rs index eddc0d177..943bf6399 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/direct_duties.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/direct_duties.rs @@ -1,7 +1,7 @@ use attester::{ DutiesReader as AttesterDutiesReader, DutiesReaderError as AttesterDutiesReaderError, }; -use beacon_chain::{block_production::Error as BlockProductionError, BeaconChain}; +use beacon_chain::BeaconChain; use block_producer::{ DutiesReader as ProducerDutiesReader, DutiesReaderError as ProducerDutiesReaderError, }; @@ -24,10 +24,7 @@ impl DirectDuties { } } -impl ProducerDutiesReader for DirectDuties -where - BlockProductionError: From<::Error>, -{ +impl ProducerDutiesReader for DirectDuties { fn is_block_production_slot(&self, slot: u64) -> Result { let validator_index = self .beacon_chain @@ -42,10 +39,7 @@ where } } -impl AttesterDutiesReader for DirectDuties -where - BlockProductionError: From<::Error>, -{ +impl AttesterDutiesReader for DirectDuties { fn validator_index(&self) -> Option { match self.beacon_chain.validator_index(&self.pubkey) { Some(index) => Some(index as u64), From 942ef4b002838842344344d440a701c9a61af378 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Feb 2019 16:30:42 +1100 Subject: [PATCH 083/107] Move `advance_slot` to its own file. --- .../beacon_chain/src/canonical_head.rs | 12 --------- beacon_node/beacon_chain/src/lib.rs | 2 +- beacon_node/beacon_chain/src/state.rs | 25 +++++++++++++++++++ 3 files changed, 26 insertions(+), 13 deletions(-) create mode 100644 beacon_node/beacon_chain/src/state.rs diff --git a/beacon_node/beacon_chain/src/canonical_head.rs b/beacon_node/beacon_chain/src/canonical_head.rs index 14f1adf81..35262961e 100644 --- a/beacon_node/beacon_chain/src/canonical_head.rs +++ b/beacon_node/beacon_chain/src/canonical_head.rs @@ -4,7 +4,6 @@ use types::{beacon_state::SlotProcessingError, BeaconBlock, BeaconState, Hash256 #[derive(Debug, PartialEq)] pub enum Error { - PastSlot, SlotProcessingError(SlotProcessingError), } @@ -32,17 +31,6 @@ where pub fn head(&self) -> RwLockReadGuard { self.canonical_head.read() } - - pub fn advance_state(&self, slot: u64) -> Result<(), SlotProcessingError> { - let state_slot = self.state.read().slot; - let head_block_root = self.head().beacon_block_root; - for _ in state_slot..slot { - self.state - .write() - .per_slot_processing(head_block_root.clone(), &self.spec)?; - } - Ok(()) - } } impl From for Error { diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 5c719dddd..314f18dc9 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -10,7 +10,7 @@ pub mod dump; mod finalized_head; mod info; mod lmd_ghost; -// mod state_transition; +mod state; use self::attestation_targets::AttestationTargets; use self::block_graph::BlockGraph; diff --git a/beacon_node/beacon_chain/src/state.rs b/beacon_node/beacon_chain/src/state.rs new file mode 100644 index 000000000..33e50ce35 --- /dev/null +++ b/beacon_node/beacon_chain/src/state.rs @@ -0,0 +1,25 @@ +use crate::{BeaconChain, ClientDB, SlotClock}; +use types::beacon_state::SlotProcessingError; + +impl BeaconChain +where + T: ClientDB, + U: SlotClock, +{ + /// Advance the `self.state` `BeaconState` to the supplied slot. + /// + /// This will perform per_slot and per_epoch processing as required. + /// + /// The `previous_block_root` will be set to the root of the current head block (as determined + /// by the fork-choice rule). + pub fn advance_state(&self, slot: u64) -> Result<(), SlotProcessingError> { + let state_slot = self.state.read().slot; + let head_block_root = self.head().beacon_block_root; + for _ in state_slot..slot { + self.state + .write() + .per_slot_processing(head_block_root.clone(), &self.spec)?; + } + Ok(()) + } +} From 4d062d77f92e5ec67a4082f0dc7ce5930ea0c7e4 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Feb 2019 16:55:37 +1100 Subject: [PATCH 084/107] Move `CheckPoint` into file, tidy canonical_head --- .../beacon_chain/src/canonical_head.rs | 7 ++++ beacon_node/beacon_chain/src/checkpoint.rs | 41 +++++++++++++++++++ beacon_node/beacon_chain/src/lib.rs | 40 ++---------------- beacon_node/beacon_chain/src/state.rs | 4 ++ 4 files changed, 55 insertions(+), 37 deletions(-) create mode 100644 beacon_node/beacon_chain/src/checkpoint.rs diff --git a/beacon_node/beacon_chain/src/canonical_head.rs b/beacon_node/beacon_chain/src/canonical_head.rs index 35262961e..32f2dc000 100644 --- a/beacon_node/beacon_chain/src/canonical_head.rs +++ b/beacon_node/beacon_chain/src/canonical_head.rs @@ -12,6 +12,7 @@ where T: ClientDB, U: SlotClock, { + /// Update the canonical head to some new values. pub fn update_canonical_head( &self, new_beacon_block: BeaconBlock, @@ -28,6 +29,12 @@ where ); } + /// Returns a read-lock guarded `CheckPoint` struct for reading the head (as chosen by the + /// fork-choice rule). + /// + /// It is important to note that the `beacon_state` returned may not match the present slot. It + /// is the state as it was when the head block was recieved, which could be some slots prior to + /// now. pub fn head(&self) -> RwLockReadGuard { self.canonical_head.read() } diff --git a/beacon_node/beacon_chain/src/checkpoint.rs b/beacon_node/beacon_chain/src/checkpoint.rs new file mode 100644 index 000000000..b2e0f9baa --- /dev/null +++ b/beacon_node/beacon_chain/src/checkpoint.rs @@ -0,0 +1,41 @@ +use types::{BeaconBlock, BeaconState, Hash256}; + +/// Represents some block and it's associated state. Generally, this will be used for tracking the +/// head, justified head and finalized head. +pub struct CheckPoint { + pub beacon_block: BeaconBlock, + pub beacon_block_root: Hash256, + pub beacon_state: BeaconState, + pub beacon_state_root: Hash256, +} + +impl CheckPoint { + /// Create a new checkpoint. + pub fn new( + beacon_block: BeaconBlock, + beacon_block_root: Hash256, + beacon_state: BeaconState, + beacon_state_root: Hash256, + ) -> Self { + Self { + beacon_block, + beacon_block_root, + beacon_state, + beacon_state_root, + } + } + + /// Update all fields of the checkpoint. + pub fn update( + &mut self, + beacon_block: BeaconBlock, + beacon_block_root: Hash256, + beacon_state: BeaconState, + beacon_state_root: Hash256, + ) { + self.beacon_block = beacon_block; + self.beacon_block_root = beacon_block_root; + self.beacon_state = beacon_state; + self.beacon_state_root = beacon_state_root; + } +} diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 314f18dc9..ddbb2732c 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -6,6 +6,7 @@ mod block_graph; pub mod block_processing; pub mod block_production; mod canonical_head; +mod checkpoint; pub mod dump; mod finalized_head; mod info; @@ -14,6 +15,7 @@ mod state; use self::attestation_targets::AttestationTargets; use self::block_graph::BlockGraph; +use self::checkpoint::CheckPoint; use attestation_aggregator::AttestationAggregator; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, @@ -24,7 +26,7 @@ use parking_lot::RwLock; use slot_clock::SlotClock; use ssz::ssz_encode; use std::sync::Arc; -use types::{BeaconBlock, BeaconState, ChainSpec, Hash256}; +use types::{BeaconState, ChainSpec, Hash256}; pub use self::block_processing::Outcome as BlockProcessingOutcome; @@ -35,42 +37,6 @@ pub enum BeaconChainError { DBError(String), } -pub struct CheckPoint { - pub beacon_block: BeaconBlock, - pub beacon_block_root: Hash256, - pub beacon_state: BeaconState, - pub beacon_state_root: Hash256, -} - -impl CheckPoint { - pub fn new( - beacon_block: BeaconBlock, - beacon_block_root: Hash256, - beacon_state: BeaconState, - beacon_state_root: Hash256, - ) -> Self { - Self { - beacon_block, - beacon_block_root, - beacon_state, - beacon_state_root, - } - } - - pub fn update( - &mut self, - beacon_block: BeaconBlock, - beacon_block_root: Hash256, - beacon_state: BeaconState, - beacon_state_root: Hash256, - ) { - self.beacon_block = beacon_block; - self.beacon_block_root = beacon_block_root; - self.beacon_state = beacon_state; - self.beacon_state_root = beacon_state_root; - } -} - pub struct BeaconChain { pub block_store: Arc>, pub state_store: Arc>, diff --git a/beacon_node/beacon_chain/src/state.rs b/beacon_node/beacon_chain/src/state.rs index 33e50ce35..db1119c8a 100644 --- a/beacon_node/beacon_chain/src/state.rs +++ b/beacon_node/beacon_chain/src/state.rs @@ -12,6 +12,10 @@ where /// /// The `previous_block_root` will be set to the root of the current head block (as determined /// by the fork-choice rule). + /// + /// It is important to note that this is _not_ the state corresponding to the canonical head + /// block, instead it is that state which may or may not have had additional per slot/epoch + /// processing applied to it. pub fn advance_state(&self, slot: u64) -> Result<(), SlotProcessingError> { let state_slot = self.state.read().slot; let head_block_root = self.head().beacon_block_root; From b99e4ed9f4278bbb1b8b581a0c712733f4d8efaf Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Feb 2019 17:04:25 +1100 Subject: [PATCH 085/107] Tidy slot_dump, remove `SlotDump`, use `CheckPoint` --- beacon_node/beacon_chain/src/checkpoint.rs | 2 ++ beacon_node/beacon_chain/src/dump.rs | 28 ++++++++----------- beacon_node/beacon_chain/src/lib.rs | 2 +- .../beacon_chain/test_harness/src/harness.rs | 6 ++-- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/beacon_node/beacon_chain/src/checkpoint.rs b/beacon_node/beacon_chain/src/checkpoint.rs index b2e0f9baa..bef97d2ed 100644 --- a/beacon_node/beacon_chain/src/checkpoint.rs +++ b/beacon_node/beacon_chain/src/checkpoint.rs @@ -1,7 +1,9 @@ +use serde_derive::Serialize; use types::{BeaconBlock, BeaconState, Hash256}; /// Represents some block and it's associated state. Generally, this will be used for tracking the /// head, justified head and finalized head. +#[derive(PartialEq, Clone, Serialize)] pub struct CheckPoint { pub beacon_block: BeaconBlock, pub beacon_block_root: Hash256, diff --git a/beacon_node/beacon_chain/src/dump.rs b/beacon_node/beacon_chain/src/dump.rs index 613534328..b2b031504 100644 --- a/beacon_node/beacon_chain/src/dump.rs +++ b/beacon_node/beacon_chain/src/dump.rs @@ -1,18 +1,11 @@ -use super::{BeaconChain, ClientDB, DBError, SlotClock}; -use serde_derive::Serialize; -use types::{BeaconBlock, BeaconState, Hash256}; - -#[derive(Debug, Clone, Serialize)] -pub struct SlotDump { - pub beacon_block: BeaconBlock, - pub beacon_block_root: Hash256, - pub beacon_state: BeaconState, - pub beacon_state_root: Hash256, -} +use super::{BeaconChain, CheckPoint, ClientDB, DBError, SlotClock}; +use types::Hash256; #[derive(Debug, Clone)] pub enum Error { + /// There was an error reading from the database. This is an internal error. DBError(String), + /// There is a missing (or invalid) block in the database. This is an internal error. MissingBlock(Hash256), } @@ -21,10 +14,14 @@ where T: ClientDB, U: SlotClock, { - pub fn chain_dump(&self) -> Result, Error> { + /// Dumps the entire canonical chain, from the head to genesis to a vector for analysis. + /// + /// This could be a very expensive operation and should only be done in testing/analysis + /// activities. + pub fn chain_dump(&self) -> Result, Error> { let mut dump = vec![]; - let mut last_slot = SlotDump { + let mut last_slot = CheckPoint { beacon_block: self.head().beacon_block.clone(), beacon_block_root: self.head().beacon_block_root, beacon_state: self.head().beacon_state.clone(), @@ -37,8 +34,7 @@ where let beacon_block_root = last_slot.beacon_block.parent_root; if beacon_block_root == self.spec.zero_hash { - // Genesis has been reached. - break; + break; // Genesis has been reached. } let beacon_block = self @@ -51,7 +47,7 @@ where .get_deserialized(&beacon_state_root)? .ok_or_else(|| Error::MissingBlock(beacon_state_root))?; - let slot = SlotDump { + let slot = CheckPoint { beacon_block, beacon_block_root, beacon_state, diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index ddbb2732c..459e5c509 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -15,7 +15,6 @@ mod state; use self::attestation_targets::AttestationTargets; use self::block_graph::BlockGraph; -use self::checkpoint::CheckPoint; use attestation_aggregator::AttestationAggregator; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, @@ -29,6 +28,7 @@ use std::sync::Arc; use types::{BeaconState, ChainSpec, Hash256}; pub use self::block_processing::Outcome as BlockProcessingOutcome; +pub use self::checkpoint::CheckPoint; #[derive(Debug, PartialEq)] pub enum BeaconChainError { diff --git a/beacon_node/beacon_chain/test_harness/src/harness.rs b/beacon_node/beacon_chain/test_harness/src/harness.rs index ce88395cb..f55fb7666 100644 --- a/beacon_node/beacon_chain/test_harness/src/harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/harness.rs @@ -1,6 +1,6 @@ use super::TestValidator; -pub use beacon_chain::dump::{Error as DumpError, SlotDump}; use beacon_chain::BeaconChain; +pub use beacon_chain::{dump::Error as DumpError, CheckPoint}; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, MemoryDB, @@ -208,11 +208,11 @@ impl BeaconChainHarness { debug!("Free attestations processed."); } - pub fn chain_dump(&self) -> Result, DumpError> { + pub fn chain_dump(&self) -> Result, DumpError> { self.beacon_chain.chain_dump() } - pub fn dump_to_file(&self, filename: String, chain_dump: &Vec) { + pub fn dump_to_file(&self, filename: String, chain_dump: &Vec) { let json = serde_json::to_string(chain_dump).unwrap(); let mut file = File::create(filename).unwrap(); file.write_all(json.as_bytes()) From a71e1031ced86fa901b44761e44122d0608cf6d5 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Feb 2019 17:07:59 +1100 Subject: [PATCH 086/107] Add docstrings to finalized_head --- beacon_node/beacon_chain/src/finalized_head.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/beacon_node/beacon_chain/src/finalized_head.rs b/beacon_node/beacon_chain/src/finalized_head.rs index 76a20a2c2..ac2917fdb 100644 --- a/beacon_node/beacon_chain/src/finalized_head.rs +++ b/beacon_node/beacon_chain/src/finalized_head.rs @@ -7,6 +7,7 @@ where T: ClientDB, U: SlotClock, { + /// Update the justified head to some new values. pub fn update_finalized_head( &self, new_beacon_block: BeaconBlock, @@ -23,6 +24,8 @@ where ); } + /// Returns a read-lock guarded `CheckPoint` struct for reading the justified head (as chosen, + /// indirectly, by the fork-choice rule). pub fn finalized_head(&self) -> RwLockReadGuard { self.finalized_head.read() } From 865919e39814b8f87871492a4911a7aa209e1041 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Feb 2019 17:50:19 +1100 Subject: [PATCH 087/107] Improve attester errors, move info -> helpers - Ensured one can distingush between a committee error and an invalid validator index when using `validator_attesation_slot_and_shard`. - Renamed the `info.rs` file to `getters.rs`, for clarity. --- .../src/attestation_aggregator.rs | 10 +- beacon_node/beacon_chain/src/getters.rs | 100 ++++++++++++++++++ beacon_node/beacon_chain/src/info.rs | 76 ------------- beacon_node/beacon_chain/src/lib.rs | 2 +- .../src/validator/direct_duties.rs | 9 +- .../types/src/beacon_state/slot_processing.rs | 7 +- 6 files changed, 116 insertions(+), 88 deletions(-) create mode 100644 beacon_node/beacon_chain/src/getters.rs delete mode 100644 beacon_node/beacon_chain/src/info.rs diff --git a/beacon_node/beacon_chain/src/attestation_aggregator.rs b/beacon_node/beacon_chain/src/attestation_aggregator.rs index 855cb711b..dd08b22ea 100644 --- a/beacon_node/beacon_chain/src/attestation_aggregator.rs +++ b/beacon_node/beacon_chain/src/attestation_aggregator.rs @@ -67,10 +67,12 @@ impl AttestationAggregator { free_attestation: &FreeAttestation, spec: &ChainSpec, ) -> Result { - let (slot, shard, committee_index) = state.attestation_slot_and_shard_for_validator( - free_attestation.validator_index as usize, - spec, - )?; + let (slot, shard, committee_index) = state + .attestation_slot_and_shard_for_validator( + free_attestation.validator_index as usize, + spec, + )? + .ok_or_else(|| Error::BadValidatorIndex)?; if free_attestation.data.slot != slot { return Err(Error::BadSlot); diff --git a/beacon_node/beacon_chain/src/getters.rs b/beacon_node/beacon_chain/src/getters.rs new file mode 100644 index 000000000..cba519743 --- /dev/null +++ b/beacon_node/beacon_chain/src/getters.rs @@ -0,0 +1,100 @@ +use super::{BeaconChain, ClientDB, SlotClock}; +use types::{beacon_state::CommitteesError, PublicKey}; + +impl BeaconChain +where + T: ClientDB, + U: SlotClock, +{ + /// Returns the the validator index (if any) for the given public key. + /// + /// Information is retrieved from the present `beacon_state.validator_registry`. + pub fn validator_index(&self, pubkey: &PublicKey) -> Option { + for (i, validator) in self + .head() + .beacon_state + .validator_registry + .iter() + .enumerate() + { + if validator.pubkey == *pubkey { + return Some(i); + } + } + None + } + + /// Returns the number of slots the validator has been required to propose. + /// + /// Returns `None` if the `validator_index` is invalid. + /// + /// Information is retrieved from the present `beacon_state.validator_registry`. + pub fn proposer_slots(&self, validator_index: usize) -> Option { + if let Some(validator) = self.state.read().validator_registry.get(validator_index) { + Some(validator.proposer_slots) + } else { + None + } + } + + /// Reads the slot clock, returns `None` if the slot is unavailable. + /// + /// The slot might be unavailable due to an error with the system clock, or if the present time + /// is before genesis (i.e., a negative slot). + /// + /// This is distinct to `present_slot`, which simply reads the latest state. If a + /// call to `read_slot_clock` results in a higher slot than a call to `present_slot`, + /// `self.state` should undergo per slot processing. + pub fn read_slot_clock(&self) -> Option { + match self.slot_clock.present_slot() { + Ok(some_slot) => some_slot, + _ => None, + } + } + + /// Returns slot of the present state. + /// + /// This is distinct to `read_slot_clock`, which reads from the actual system clock. If + /// `self.state` has not been transitioned it is possible for the system clock to be on a + /// different slot to what is returned from this call. + pub fn present_slot(&self) -> u64 { + self.state.read().slot + } + + /// Returns the block proposer for a given slot. + /// + /// Information is read from the present `beacon_state` shuffling, so only information from the + /// present and prior epoch is available. + pub fn block_proposer(&self, slot: u64) -> Result { + let index = self + .state + .read() + .get_beacon_proposer_index(slot, &self.spec)?; + + Ok(index) + } + + /// Returns the justified slot for the present state. + pub fn justified_slot(&self) -> u64 { + self.state.read().justified_slot + } + + /// Returns the attestation slot and shard for a given validator index. + /// + /// Information is read from the current state, so only information from the present and prior + /// epoch is available. + pub fn validator_attestion_slot_and_shard( + &self, + validator_index: usize, + ) -> Result, CommitteesError> { + if let Some((slot, shard, _committee)) = self + .state + .read() + .attestation_slot_and_shard_for_validator(validator_index, &self.spec)? + { + Ok(Some((slot, shard))) + } else { + Ok(None) + } + } +} diff --git a/beacon_node/beacon_chain/src/info.rs b/beacon_node/beacon_chain/src/info.rs deleted file mode 100644 index 70405ab32..000000000 --- a/beacon_node/beacon_chain/src/info.rs +++ /dev/null @@ -1,76 +0,0 @@ -use super::{BeaconChain, ClientDB, SlotClock}; -use types::{beacon_state::CommitteesError, PublicKey}; - -#[derive(Debug, PartialEq)] -pub enum Error { - SlotClockError, - CommitteesError(CommitteesError), -} - -impl BeaconChain -where - T: ClientDB, - U: SlotClock, -{ - pub fn validator_index(&self, pubkey: &PublicKey) -> Option { - for (i, validator) in self - .head() - .beacon_state - .validator_registry - .iter() - .enumerate() - { - if validator.pubkey == *pubkey { - return Some(i); - } - } - None - } - - pub fn proposer_slots(&self, validator_index: usize) -> Option { - if let Some(validator) = self.state.read().validator_registry.get(validator_index) { - Some(validator.proposer_slots) - } else { - None - } - } - - pub fn read_slot_clock(&self) -> Option { - match self.slot_clock.present_slot() { - Ok(some_slot) => some_slot, - _ => None, - } - } - - pub fn present_slot(&self) -> u64 { - self.state.read().slot - } - - pub fn block_proposer(&self, slot: u64) -> Result { - let index = self - .state - .read() - .get_beacon_proposer_index(slot, &self.spec)?; - - Ok(index) - } - - pub fn justified_slot(&self) -> u64 { - self.state.read().justified_slot - } - - pub fn validator_attestion_slot_and_shard(&self, validator_index: usize) -> Option<(u64, u64)> { - let (slot, shard, _committee) = self - .state - .read() - .attestation_slot_and_shard_for_validator(validator_index, &self.spec) - .ok()?; - Some((slot, shard)) - } -} - -impl From for Error { - fn from(e: CommitteesError) -> Error { - Error::CommitteesError(e) - } -} diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 459e5c509..e0ce5d8a7 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -9,7 +9,7 @@ mod canonical_head; mod checkpoint; pub mod dump; mod finalized_head; -mod info; +mod getters; mod lmd_ghost; mod state; diff --git a/beacon_node/beacon_chain/test_harness/src/validator/direct_duties.rs b/beacon_node/beacon_chain/test_harness/src/validator/direct_duties.rs index 943bf6399..9eda378a0 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/direct_duties.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/direct_duties.rs @@ -53,9 +53,12 @@ impl AttesterDutiesReader for DirectDuties { .beacon_chain .validator_attestion_slot_and_shard(validator_index as usize) { - Some((attest_slot, attest_shard)) if attest_slot == slot => Ok(Some(attest_shard)), - Some(_) => Ok(None), - None => Err(AttesterDutiesReaderError::UnknownEpoch), + Ok(Some((attest_slot, attest_shard))) if attest_slot == slot => { + Ok(Some(attest_shard)) + } + Ok(Some(_)) => Ok(None), + Ok(None) => Err(AttesterDutiesReaderError::UnknownEpoch), + Err(_) => panic!("Error when getting validator attestation shard."), } } else { Err(AttesterDutiesReaderError::UnknownValidator) diff --git a/eth2/types/src/beacon_state/slot_processing.rs b/eth2/types/src/beacon_state/slot_processing.rs index 4693bd069..76135210f 100644 --- a/eth2/types/src/beacon_state/slot_processing.rs +++ b/eth2/types/src/beacon_state/slot_processing.rs @@ -2,7 +2,6 @@ use crate::{ beacon_state::{CommitteesError, EpochProcessingError}, BeaconState, ChainSpec, Hash256, }; -use log::debug; #[derive(Debug, PartialEq)] pub enum Error { @@ -43,17 +42,17 @@ impl BeaconState { &self, validator_index: usize, spec: &ChainSpec, - ) -> Result<(u64, u64, u64), CommitteesError> { + ) -> Result, CommitteesError> { let mut result = None; for slot in self.get_current_epoch_boundaries(spec.epoch_length) { for (committee, shard) in self.get_crosslink_committees_at_slot(slot, spec)? { if let Some(committee_index) = committee.iter().position(|&i| i == validator_index) { - result = Some(Ok((slot, shard, committee_index as u64))); + result = Some((slot, shard, committee_index as u64)); } } } - result.unwrap() + Ok(result) } } From 5ca53190b5408675f39250a83f285f80c537c67c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Feb 2019 18:18:39 +1100 Subject: [PATCH 088/107] Add comments to beacon chain fork choice --- beacon_node/beacon_chain/src/lib.rs | 1 + beacon_node/beacon_chain/src/lmd_ghost.rs | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index e0ce5d8a7..68c0c3c9c 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -56,6 +56,7 @@ where T: ClientDB, U: SlotClock, { + /// Instantiate a new Beacon Chain, from genesis. pub fn genesis( state_store: Arc>, block_store: Arc>, diff --git a/beacon_node/beacon_chain/src/lmd_ghost.rs b/beacon_node/beacon_chain/src/lmd_ghost.rs index 6116834bb..176d4a686 100644 --- a/beacon_node/beacon_chain/src/lmd_ghost.rs +++ b/beacon_node/beacon_chain/src/lmd_ghost.rs @@ -12,11 +12,6 @@ use types::{ Hash256, }; -#[derive(Debug, PartialEq)] -pub enum Outcome { - Something, -} - #[derive(Debug, PartialEq)] pub enum Error { DBError(String), @@ -32,6 +27,7 @@ where U: SlotClock, Error: From<::Error>, { + /// Run the fork-choice rule on the current chain, updating the canonical head, if required. pub fn fork_choice(&self) -> Result<(), Error> { let present_head = &self.finalized_head().beacon_block_root; @@ -56,6 +52,7 @@ where Ok(()) } + /// A very inefficient implementation of LMD ghost. pub fn slow_lmd_ghost(&self, start_hash: &Hash256) -> Result { let start = self .block_store @@ -114,6 +111,9 @@ where } } +/// Get the total number of votes for some given block root. +/// +/// The vote count is incrememented each time an attestation target votes for a block root. fn get_vote_count( block_store: &Arc>, attestation_targets: &[Hash256], From efc0902abb710382d6133981ec7a78f3c4f3e183 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Feb 2019 18:20:42 +1100 Subject: [PATCH 089/107] Run cargo fmt --- eth2/types/src/beacon_state/winning_root.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth2/types/src/beacon_state/winning_root.rs b/eth2/types/src/beacon_state/winning_root.rs index 20fdca09b..1b891af73 100644 --- a/eth2/types/src/beacon_state/winning_root.rs +++ b/eth2/types/src/beacon_state/winning_root.rs @@ -100,7 +100,7 @@ impl BeaconState { } impl From for Error { - fn from (e: AttestationParticipantsError) -> Error { + fn from(e: AttestationParticipantsError) -> Error { Error::AttestationParticipantsError(e) } } From 4cc88c8cc72f066ec6d1255c5d873bbd96add905 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Feb 2019 18:22:43 +1100 Subject: [PATCH 090/107] Tidy benches --- .../test_harness/benches/state_transition.rs | 20 ++++--------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/benches/state_transition.rs b/beacon_node/beacon_chain/test_harness/benches/state_transition.rs index 15184e10c..013ecfd1e 100644 --- a/beacon_node/beacon_chain/test_harness/benches/state_transition.rs +++ b/beacon_node/beacon_chain/test_harness/benches/state_transition.rs @@ -1,11 +1,11 @@ use criterion::Criterion; use criterion::{black_box, criterion_group, criterion_main, Benchmark}; -use env_logger::{Builder, Env}; +// use env_logger::{Builder, Env}; use test_harness::BeaconChainHarness; use types::{ChainSpec, Hash256}; fn mid_epoch_state_transition(c: &mut Criterion) { - Builder::from_env(Env::default().default_filter_or("debug")).init(); + // Builder::from_env(Env::default().default_filter_or("debug")).init(); let validator_count = 1000; let mut rig = BeaconChainHarness::new(ChainSpec::foundation(), validator_count); @@ -56,25 +56,13 @@ fn epoch_boundary_state_transition(c: &mut Criterion) { )) }) }) - .sample_size(5), + .sample_size(5), // sample size is low because function is sloooow. ); - - /* - c.bench_function("mid-epoch state transition 10k validators", move |b| { - let state = state.clone(); - b.iter(|| { - let mut state = state.clone(); - black_box(black_box( - state.per_slot_processing(Hash256::zero(), &rig.spec), - )) - }) - }); - */ } criterion_group!( benches, - // mid_epoch_state_transition, + mid_epoch_state_transition, epoch_boundary_state_transition ); criterion_main!(benches); From cb85fbcdb2b634d3a3956af42a6b5c423a05ddec Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 1 Feb 2019 18:48:37 +1100 Subject: [PATCH 091/107] Tidy and add docstring to chain test harness. --- .../beacon_chain/test_harness/src/harness.rs | 12 ++++++++ .../src/validator/beacon_node/attester.rs | 4 +-- .../src/validator/beacon_node/mod.rs | 12 ++++++-- .../src/validator/beacon_node/producer.rs | 4 +-- .../src/validator/direct_duties.rs | 2 ++ .../test_harness/src/validator/mod.rs | 30 +++++++++++++++---- .../test_harness/src/validator/signer.rs | 3 +- 7 files changed, 54 insertions(+), 13 deletions(-) diff --git a/beacon_node/beacon_chain/test_harness/src/harness.rs b/beacon_node/beacon_chain/test_harness/src/harness.rs index f55fb7666..7bc86a7e1 100644 --- a/beacon_node/beacon_chain/test_harness/src/harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/harness.rs @@ -15,6 +15,12 @@ use std::iter::FromIterator; use std::sync::Arc; use types::{BeaconBlock, ChainSpec, FreeAttestation, Keypair, Validator}; +/// The beacon chain harness simulates a single beacon node with `validator_count` validators connected +/// to it. Each validator is provided a borrow to the beacon chain, where it may read +/// information and submit blocks/attesations for processing. +/// +/// This test harness is useful for testing validator and internal state transition logic. It +/// is not useful for testing that multiple beacon nodes can reach consensus. pub struct BeaconChainHarness { pub db: Arc, pub beacon_chain: Arc>, @@ -25,6 +31,10 @@ pub struct BeaconChainHarness { } impl BeaconChainHarness { + /// Create a new harness with: + /// + /// - A keypair, `BlockProducer` and `Attester` for each validator. + /// - A new BeaconChain struct where the given validators are in the genesis. pub fn new(mut spec: ChainSpec, validator_count: usize) -> Self { let db = Arc::new(MemoryDB::open()); let block_store = Arc::new(BeaconBlockStore::new(db.clone())); @@ -208,10 +218,12 @@ impl BeaconChainHarness { debug!("Free attestations processed."); } + /// Dump all blocks and states from the canonical beacon chain. pub fn chain_dump(&self) -> Result, DumpError> { self.beacon_chain.chain_dump() } + /// Write the output of `chain_dump` to a JSON file. pub fn dump_to_file(&self, filename: String, chain_dump: &Vec) { let json = serde_json::to_string(chain_dump).unwrap(); let mut file = File::create(filename).unwrap(); diff --git a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs index 19225b671..e7198461e 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs @@ -1,10 +1,10 @@ -use super::BenchingBeaconNode; +use super::DirectBeaconNode; use attester::{BeaconNode as AttesterBeaconNode, BeaconNodeError as NodeError, PublishOutcome}; use db::ClientDB; use slot_clock::SlotClock; use types::{AttestationData, FreeAttestation}; -impl AttesterBeaconNode for BenchingBeaconNode { +impl AttesterBeaconNode for DirectBeaconNode { fn produce_attestation_data( &self, _slot: u64, diff --git a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/mod.rs b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/mod.rs index caed2bc5f..90f8943fc 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/mod.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/mod.rs @@ -8,13 +8,19 @@ use types::{BeaconBlock, FreeAttestation}; mod attester; mod producer; -pub struct BenchingBeaconNode { +/// Connect directly to a borrowed `BeaconChain` instance so an attester/producer can request/submit +/// blocks/attestations. +/// +/// `BeaconBlock`s and `FreeAttestation`s are not actually published to the `BeaconChain`, instead +/// they are stored inside this struct. This is to allow one to benchmark the submission of the +/// block/attestation directly, or modify it before submission. +pub struct DirectBeaconNode { beacon_chain: Arc>, published_blocks: RwLock>, published_attestations: RwLock>, } -impl BenchingBeaconNode { +impl DirectBeaconNode { pub fn new(beacon_chain: Arc>) -> Self { Self { beacon_chain, @@ -23,10 +29,12 @@ impl BenchingBeaconNode { } } + /// Get the last published block (if any). pub fn last_published_block(&self) -> Option { Some(self.published_blocks.read().last()?.clone()) } + /// Get the last published attestation (if any). pub fn last_published_free_attestation(&self) -> Option { Some(self.published_attestations.read().last()?.clone()) } diff --git a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/producer.rs b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/producer.rs index 861acd25e..36c82f6cc 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/producer.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/producer.rs @@ -1,4 +1,4 @@ -use super::BenchingBeaconNode; +use super::DirectBeaconNode; use block_producer::{ BeaconNode as BeaconBlockNode, BeaconNodeError as BeaconBlockNodeError, PublishOutcome, }; @@ -6,7 +6,7 @@ use db::ClientDB; use slot_clock::SlotClock; use types::{BeaconBlock, PublicKey, Signature}; -impl BeaconBlockNode for BenchingBeaconNode { +impl BeaconBlockNode for DirectBeaconNode { /// Requests the `proposer_nonce` from the `BeaconChain`. fn proposer_nonce(&self, pubkey: &PublicKey) -> Result { let validator_index = self diff --git a/beacon_node/beacon_chain/test_harness/src/validator/direct_duties.rs b/beacon_node/beacon_chain/test_harness/src/validator/direct_duties.rs index 9eda378a0..e724b3e55 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/direct_duties.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/direct_duties.rs @@ -10,6 +10,8 @@ use slot_clock::SlotClock; use std::sync::Arc; use types::PublicKey; +/// Connects directly to a borrowed `BeaconChain` and reads attester/proposer duties directly from +/// it. pub struct DirectDuties { beacon_chain: Arc>, pubkey: PublicKey, diff --git a/beacon_node/beacon_chain/test_harness/src/validator/mod.rs b/beacon_node/beacon_chain/test_harness/src/validator/mod.rs index 018ffbc86..4eae0bb3a 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/mod.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/mod.rs @@ -10,7 +10,7 @@ mod beacon_node; mod direct_duties; mod signer; -pub use self::beacon_node::BenchingBeaconNode; +pub use self::beacon_node::DirectBeaconNode; pub use self::direct_duties::DirectDuties; pub use attester::PollOutcome as AttestationPollOutcome; pub use block_producer::PollOutcome as BlockPollOutcome; @@ -27,28 +27,37 @@ pub enum AttestationProduceError { PollError(AttestationPollError), } +/// A `BlockProducer` and `Attester` which sign using a common keypair. +/// +/// The test validator connects directly to a borrowed `BeaconChain` struct. It is useful for +/// testing that the core proposer and attester logic is functioning. Also for supporting beacon +/// chain tests. pub struct TestValidator { pub block_producer: BlockProducer< TestingSlotClock, - BenchingBeaconNode, + DirectBeaconNode, DirectDuties, TestSigner, >, pub attester: Attester< TestingSlotClock, - BenchingBeaconNode, + DirectBeaconNode, DirectDuties, TestSigner, >, pub spec: Arc, pub epoch_map: Arc>, pub keypair: Keypair, - pub beacon_node: Arc>, + pub beacon_node: Arc>, pub slot_clock: Arc, pub signer: Arc, } impl TestValidator { + /// Create a new TestValidator that signs with the given keypair, operates per the given spec and connects to the + /// supplied beacon node. + /// + /// A `BlockProducer` and `Attester` is created.. pub fn new( keypair: Keypair, beacon_chain: Arc>, @@ -56,7 +65,7 @@ impl TestValidator { ) -> Self { let slot_clock = Arc::new(TestingSlotClock::new(spec.genesis_slot)); let signer = Arc::new(TestSigner::new(keypair.clone())); - let beacon_node = Arc::new(BenchingBeaconNode::new(beacon_chain.clone())); + let beacon_node = Arc::new(DirectBeaconNode::new(beacon_chain.clone())); let epoch_map = Arc::new(DirectDuties::new(keypair.pk.clone(), beacon_chain.clone())); let block_producer = BlockProducer::new( @@ -87,8 +96,11 @@ impl TestValidator { } } + /// Run the `poll` function on the `BlockProducer` and produce a block. + /// + /// An error is returned if the producer refuses to produce. pub fn produce_block(&mut self) -> Result { - // Using `BenchingBeaconNode`, the validator will always return sucessufully if it tries to + // Using `DirectBeaconNode`, the validator will always return sucessufully if it tries to // publish a block. match self.block_producer.poll() { Ok(BlockPollOutcome::BlockProduced(_)) => {} @@ -101,6 +113,9 @@ impl TestValidator { .expect("Unable to obtain produced block.")) } + /// Run the `poll` function on the `Attester` and produce a `FreeAttestation`. + /// + /// An error is returned if the attester refuses to attest. pub fn produce_free_attestation(&mut self) -> Result { match self.attester.poll() { Ok(AttestationPollOutcome::AttestationProduced(_)) => {} @@ -113,6 +128,9 @@ impl TestValidator { .expect("Unable to obtain produced attestation.")) } + /// Set the validators slot clock to the specified slot. + /// + /// The validators slot clock will always read this value until it is set to something else. pub fn set_slot(&mut self, slot: u64) { self.slot_clock.set_slot(slot) } diff --git a/beacon_node/beacon_chain/test_harness/src/validator/signer.rs b/beacon_node/beacon_chain/test_harness/src/validator/signer.rs index 572a870ff..6e63c7f99 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/signer.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/signer.rs @@ -3,7 +3,7 @@ use block_producer::Signer as BlockProposerSigner; use std::sync::RwLock; use types::{Keypair, Signature}; -/// A test-only struct used to simulate a Beacon Node. +/// A test-only struct used to perform signing for a proposer or attester. pub struct TestSigner { keypair: Keypair, should_sign: RwLock, @@ -24,6 +24,7 @@ impl TestSigner { *self.should_sign.write().unwrap() = enabled; } + /// Sign some message. fn bls_sign(&self, message: &[u8]) -> Option { Some(Signature::new(message, &self.keypair.sk)) } From dcb507ca18fa96792570c2ebae94efaa448d6968 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 5 Feb 2019 03:51:24 +1100 Subject: [PATCH 092/107] Remove unused errors from genesis crate --- eth2/genesis/src/beacon_state.rs | 42 +++++++------------------------- eth2/genesis/src/lib.rs | 2 +- 2 files changed, 10 insertions(+), 34 deletions(-) diff --git a/eth2/genesis/src/beacon_state.rs b/eth2/genesis/src/beacon_state.rs index 7d7cc7665..4ccb32e54 100644 --- a/eth2/genesis/src/beacon_state.rs +++ b/eth2/genesis/src/beacon_state.rs @@ -1,30 +1,12 @@ use types::{BeaconState, ChainSpec, Crosslink, Fork}; -use validator_shuffling::{shard_and_committees_for_cycle, ValidatorAssignmentError}; - -#[derive(Debug, PartialEq)] -pub enum Error { - NoValidators, - ValidationAssignmentError(ValidatorAssignmentError), - NotImplemented, -} - -pub fn genesis_beacon_state(spec: &ChainSpec) -> Result { - /* - * Assign the validators to shards, using all zeros as the seed. - */ - let _shard_and_committee_for_slots = { - let mut a = shard_and_committees_for_cycle(&[0; 32], &spec.initial_validators, 0, &spec)?; - let mut b = a.clone(); - a.append(&mut b); - a - }; +pub fn genesis_beacon_state(spec: &ChainSpec) -> BeaconState { let initial_crosslink = Crosslink { slot: spec.genesis_slot, shard_block_root: spec.zero_hash, }; - Ok(BeaconState { + BeaconState { /* * Misc */ @@ -81,12 +63,6 @@ pub fn genesis_beacon_state(spec: &ChainSpec) -> Result { */ latest_eth1_data: spec.intial_eth1_data.clone(), eth1_data_votes: vec![], - }) -} - -impl From for Error { - fn from(e: ValidatorAssignmentError) -> Error { - Error::ValidationAssignmentError(e) } } @@ -99,7 +75,7 @@ mod tests { fn test_genesis_state() { let spec = ChainSpec::foundation(); - let state = genesis_beacon_state(&spec).unwrap(); + let state = genesis_beacon_state(&spec); assert_eq!( state.validator_registry.len(), @@ -111,7 +87,7 @@ mod tests { fn test_genesis_state_misc() { let spec = ChainSpec::foundation(); - let state = genesis_beacon_state(&spec).unwrap(); + let state = genesis_beacon_state(&spec); assert_eq!(state.slot, 0); assert_eq!(state.genesis_time, spec.genesis_time); @@ -124,7 +100,7 @@ mod tests { fn test_genesis_state_validators() { let spec = ChainSpec::foundation(); - let state = genesis_beacon_state(&spec).unwrap(); + let state = genesis_beacon_state(&spec); assert_eq!(state.validator_registry, spec.initial_validators); assert_eq!(state.validator_balances, spec.initial_balances); @@ -137,7 +113,7 @@ mod tests { fn test_genesis_state_randomness_committees() { let spec = ChainSpec::foundation(); - let state = genesis_beacon_state(&spec).unwrap(); + let state = genesis_beacon_state(&spec); // Array of size 8,192 each being zero_hash assert_eq!(state.latest_randao_mixes.len(), 8_192); @@ -166,7 +142,7 @@ mod tests { fn test_genesis_state_finanilty() { let spec = ChainSpec::foundation(); - let state = genesis_beacon_state(&spec).unwrap(); + let state = genesis_beacon_state(&spec); assert_eq!(state.previous_justified_slot, 0); assert_eq!(state.justified_slot, 0); @@ -178,7 +154,7 @@ mod tests { fn test_genesis_state_recent_state() { let spec = ChainSpec::foundation(); - let state = genesis_beacon_state(&spec).unwrap(); + let state = genesis_beacon_state(&spec); // Test latest_crosslinks assert_eq!(state.latest_crosslinks.len(), 1_024); @@ -210,7 +186,7 @@ mod tests { fn test_genesis_state_deposit_root() { let spec = ChainSpec::foundation(); - let state = genesis_beacon_state(&spec).unwrap(); + let state = genesis_beacon_state(&spec); assert_eq!(&state.latest_eth1_data, &spec.intial_eth1_data); assert!(state.eth1_data_votes.is_empty()); diff --git a/eth2/genesis/src/lib.rs b/eth2/genesis/src/lib.rs index 003e66959..295bdbd3c 100644 --- a/eth2/genesis/src/lib.rs +++ b/eth2/genesis/src/lib.rs @@ -2,4 +2,4 @@ mod beacon_block; mod beacon_state; pub use crate::beacon_block::genesis_beacon_block; -pub use crate::beacon_state::{genesis_beacon_state, Error as GenesisError}; +pub use crate::beacon_state::genesis_beacon_state; From bd1cfeeba92460f0dbe8cacdbcda480bb723c683 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 5 Feb 2019 03:52:34 +1100 Subject: [PATCH 093/107] Change bls import method for validator --- eth2/types/src/validator.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eth2/types/src/validator.rs b/eth2/types/src/validator.rs index 708300d9d..5c7d0ad30 100644 --- a/eth2/types/src/validator.rs +++ b/eth2/types/src/validator.rs @@ -1,6 +1,5 @@ use super::Hash256; -use crate::test_utils::TestRandom; -use bls::PublicKey; +use crate::{test_utils::TestRandom, PublicKey}; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; From c1ed5cd2d8131123f6d3099cd3f895d8c08fbaa6 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 5 Feb 2019 03:52:47 +1100 Subject: [PATCH 094/107] Refactor beacon_chain. Moves all the `BeaconChain` impls into the same file that the struct is defined in. --- .../src/attestation_aggregator.rs | 90 ++- .../src/attestation_processing.rs | 35 -- .../src/attestation_production.rs | 48 -- .../beacon_chain/src/attestation_targets.rs | 21 - beacon_node/beacon_chain/src/beacon_chain.rs | 586 ++++++++++++++++++ .../beacon_chain/src/block_processing.rs | 151 ----- .../beacon_chain/src/block_production.rs | 102 --- .../beacon_chain/src/canonical_head.rs | 47 -- beacon_node/beacon_chain/src/dump.rs | 69 --- .../beacon_chain/src/finalized_head.rs | 32 - beacon_node/beacon_chain/src/getters.rs | 100 --- beacon_node/beacon_chain/src/lib.rs | 124 +--- beacon_node/beacon_chain/src/lmd_ghost.rs | 4 +- beacon_node/beacon_chain/src/state.rs | 29 - 14 files changed, 648 insertions(+), 790 deletions(-) delete mode 100644 beacon_node/beacon_chain/src/attestation_processing.rs delete mode 100644 beacon_node/beacon_chain/src/attestation_production.rs create mode 100644 beacon_node/beacon_chain/src/beacon_chain.rs delete mode 100644 beacon_node/beacon_chain/src/block_processing.rs delete mode 100644 beacon_node/beacon_chain/src/block_production.rs delete mode 100644 beacon_node/beacon_chain/src/canonical_head.rs delete mode 100644 beacon_node/beacon_chain/src/dump.rs delete mode 100644 beacon_node/beacon_chain/src/finalized_head.rs delete mode 100644 beacon_node/beacon_chain/src/getters.rs delete mode 100644 beacon_node/beacon_chain/src/state.rs diff --git a/beacon_node/beacon_chain/src/attestation_aggregator.rs b/beacon_node/beacon_chain/src/attestation_aggregator.rs index dd08b22ea..0f6aa388e 100644 --- a/beacon_node/beacon_chain/src/attestation_aggregator.rs +++ b/beacon_node/beacon_chain/src/attestation_aggregator.rs @@ -20,19 +20,18 @@ pub struct AttestationAggregator { store: HashMap, Attestation>, } -/// The outcome of sucessfully processing a `FreeAttestation`. -#[derive(Debug, PartialEq)] -pub enum ProcessOutcome { +pub struct Outcome { + pub valid: bool, + pub message: Message, +} + +pub enum Message { /// The free attestation was added to an existing attestation. Aggregated, /// The free attestation has already been aggregated to an existing attestation. AggregationNotRequired, /// The free attestation was transformed into a new attestation. NewAttestationCreated, -} - -#[derive(Debug, PartialEq)] -pub enum Error { /// The supplied `validator_index` is not in the committee for the given `shard` and `slot`. BadValidatorIndex, /// The given `signature` did not match the `pubkey` in the given @@ -42,8 +41,20 @@ pub enum Error { BadSlot, /// The given `shard` does not match the validators committee assignment. BadShard, - /// There was an error finding the committee for the given `validator_index`. - CommitteesError(CommitteesError), +} + +macro_rules! some_or_invalid { + ($expression: expr, $error: expr) => { + match $expression { + Some(x) => x, + None => { + return Ok(Outcome { + valid: false, + message: $error, + }); + } + } + }; } impl AttestationAggregator { @@ -66,33 +77,45 @@ impl AttestationAggregator { state: &BeaconState, free_attestation: &FreeAttestation, spec: &ChainSpec, - ) -> Result { - let (slot, shard, committee_index) = state - .attestation_slot_and_shard_for_validator( + ) -> Result { + let (slot, shard, committee_index) = some_or_invalid!( + state.attestation_slot_and_shard_for_validator( free_attestation.validator_index as usize, spec, - )? - .ok_or_else(|| Error::BadValidatorIndex)?; + )?, + Message::BadValidatorIndex + ); if free_attestation.data.slot != slot { - return Err(Error::BadSlot); + return Ok(Outcome { + valid: false, + message: Message::BadSlot, + }); } if free_attestation.data.shard != shard { - return Err(Error::BadShard); + return Ok(Outcome { + valid: false, + message: Message::BadShard, + }); } let signable_message = free_attestation.data.signable_message(PHASE_0_CUSTODY_BIT); - let validator_pubkey = &state - .validator_registry - .get(free_attestation.validator_index as usize) - .ok_or_else(|| Error::BadValidatorIndex)? - .pubkey; + + let validator_record = some_or_invalid!( + state + .validator_registry + .get(free_attestation.validator_index as usize), + Message::BadValidatorIndex + ); if !free_attestation .signature - .verify(&signable_message, &validator_pubkey) + .verify(&signable_message, &validator_record.pubkey) { - return Err(Error::BadSignature); + return Ok(Outcome { + valid: false, + message: Message::BadSignature, + }); } if let Some(existing_attestation) = self.store.get(&signable_message) { @@ -102,9 +125,15 @@ impl AttestationAggregator { committee_index as usize, ) { self.store.insert(signable_message, updated_attestation); - Ok(ProcessOutcome::Aggregated) + Ok(Outcome { + valid: true, + message: Message::Aggregated, + }) } else { - Ok(ProcessOutcome::AggregationNotRequired) + Ok(Outcome { + valid: true, + message: Message::AggregationNotRequired, + }) } } else { let mut aggregate_signature = AggregateSignature::new(); @@ -118,7 +147,10 @@ impl AttestationAggregator { aggregate_signature, }; self.store.insert(signable_message, new_attestation); - Ok(ProcessOutcome::NewAttestationCreated) + Ok(Outcome { + valid: true, + message: Message::NewAttestationCreated, + }) } } @@ -183,9 +215,3 @@ fn aggregate_attestation( }) } } - -impl From for Error { - fn from(e: CommitteesError) -> Error { - Error::CommitteesError(e) - } -} diff --git a/beacon_node/beacon_chain/src/attestation_processing.rs b/beacon_node/beacon_chain/src/attestation_processing.rs deleted file mode 100644 index 4470c3d22..000000000 --- a/beacon_node/beacon_chain/src/attestation_processing.rs +++ /dev/null @@ -1,35 +0,0 @@ -use super::{BeaconChain, ClientDB, SlotClock}; -pub use crate::attestation_aggregator::{Error as AggregatorError, ProcessOutcome}; -use types::FreeAttestation; - -#[derive(Debug, PartialEq)] -pub enum Error { - /// The free attestation was not processed succesfully. - AggregatorError(AggregatorError), -} - -impl BeaconChain -where - T: ClientDB, - U: SlotClock, -{ - /// Validate a `FreeAttestation` and either: - /// - /// - Create a new `Attestation`. - /// - Aggregate it to an existing `Attestation`. - pub fn process_free_attestation( - &self, - free_attestation: FreeAttestation, - ) -> Result { - self.attestation_aggregator - .write() - .process_free_attestation(&self.state.read(), &free_attestation, &self.spec) - .map_err(|e| e.into()) - } -} - -impl From for Error { - fn from(e: AggregatorError) -> Error { - Error::AggregatorError(e) - } -} diff --git a/beacon_node/beacon_chain/src/attestation_production.rs b/beacon_node/beacon_chain/src/attestation_production.rs deleted file mode 100644 index 40284877e..000000000 --- a/beacon_node/beacon_chain/src/attestation_production.rs +++ /dev/null @@ -1,48 +0,0 @@ -use super::{BeaconChain, ClientDB, SlotClock}; -use types::{AttestationData, Hash256}; - -#[derive(Debug, PartialEq)] -pub enum Error { - /// The `justified_block_root` is unknown. This is an internal error. - UnknownJustifiedRoot, - /// The `epoch_boundary_root` is unknown. This is an internal error. - UnknownBoundaryRoot, -} - -impl BeaconChain -where - T: ClientDB, - U: SlotClock, -{ - /// Produce an `AttestationData` that is valid for the present `slot` and given `shard`. - pub fn produce_attestation_data(&self, shard: u64) -> Result { - let justified_slot = self.justified_slot(); - let justified_block_root = self - .state - .read() - .get_block_root(justified_slot, &self.spec) - .ok_or_else(|| Error::UnknownJustifiedRoot)? - .clone(); - - let epoch_boundary_root = self - .state - .read() - .get_block_root( - self.state.read().current_epoch_start_slot(&self.spec), - &self.spec, - ) - .ok_or_else(|| Error::UnknownBoundaryRoot)? - .clone(); - - Ok(AttestationData { - slot: self.state.read().slot, - shard, - beacon_block_root: self.head().beacon_block_root.clone(), - epoch_boundary_root, - shard_block_root: Hash256::zero(), - latest_crosslink_root: Hash256::zero(), - justified_slot, - justified_block_root, - }) - } -} diff --git a/beacon_node/beacon_chain/src/attestation_targets.rs b/beacon_node/beacon_chain/src/attestation_targets.rs index 62f0bbdfa..6585e4a47 100644 --- a/beacon_node/beacon_chain/src/attestation_targets.rs +++ b/beacon_node/beacon_chain/src/attestation_targets.rs @@ -1,4 +1,3 @@ -use crate::{BeaconChain, ClientDB, SlotClock}; use std::collections::HashMap; use types::Hash256; @@ -21,23 +20,3 @@ impl AttestationTargets { self.map.insert(validator_index, block_hash) } } - -impl BeaconChain -where - T: ClientDB, - U: SlotClock, -{ - pub fn insert_latest_attestation_target(&self, validator_index: u64, block_root: Hash256) { - let mut targets = self.latest_attestation_targets.write(); - targets.insert(validator_index, block_root); - } - - pub fn get_latest_attestation_target(&self, validator_index: u64) -> Option { - let targets = self.latest_attestation_targets.read(); - - match targets.get(validator_index) { - Some(hash) => Some(hash.clone()), - None => None, - } - } -} diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs new file mode 100644 index 000000000..436d5a246 --- /dev/null +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -0,0 +1,586 @@ +use db::{ + stores::{BeaconBlockStore, BeaconStateStore}, + ClientDB, DBError, +}; +use genesis::{genesis_beacon_block, genesis_beacon_state}; +use log::{debug, trace}; +use parking_lot::{RwLock, RwLockReadGuard}; +use slot_clock::SlotClock; +use ssz::ssz_encode; +use std::sync::Arc; +use types::{ + beacon_state::{BlockProcessingError, CommitteesError, SlotProcessingError}, + readers::{BeaconBlockReader, BeaconStateReader}, + AttestationData, BeaconBlock, BeaconBlockBody, BeaconState, ChainSpec, Eth1Data, + FreeAttestation, Hash256, PublicKey, Signature, +}; + +use crate::attestation_aggregator::{AttestationAggregator, Outcome as AggregationOutcome}; +use crate::attestation_targets::AttestationTargets; +use crate::block_graph::BlockGraph; +use crate::checkpoint::CheckPoint; + +#[derive(Debug, PartialEq)] +pub enum Error { + InsufficientValidators, + BadRecentBlockRoots, + CommitteesError(CommitteesError), + DBInconsistent(String), + DBError(String), +} + +#[derive(Debug, PartialEq)] +pub enum ValidBlock { + /// The block was sucessfully processed. + Processed, +} + +#[derive(Debug, PartialEq)] +pub enum InvalidBlock { + /// The block slot is greater than the present slot. + FutureSlot, + /// The block state_root does not match the generated state. + StateRootMismatch, + /// The blocks parent_root is unknown. + ParentUnknown, + /// There was an error whilst advancing the parent state to the present slot. This condition + /// should not occur, it likely represents an internal error. + SlotProcessingError(SlotProcessingError), + /// The block could not be applied to the state, it is invalid. + PerBlockProcessingError(BlockProcessingError), +} + +#[derive(Debug, PartialEq)] +pub enum BlockProcessingOutcome { + /// The block was sucessfully validated. + ValidBlock(ValidBlock), + /// The block was not sucessfully validated. + InvalidBlock(InvalidBlock), +} + +pub struct BeaconChain { + pub block_store: Arc>, + pub state_store: Arc>, + pub slot_clock: U, + pub block_graph: BlockGraph, + pub attestation_aggregator: RwLock, + canonical_head: RwLock, + finalized_head: RwLock, + justified_head: RwLock, + pub state: RwLock, + pub latest_attestation_targets: RwLock, + pub spec: ChainSpec, +} + +impl BeaconChain +where + T: ClientDB, + U: SlotClock, +{ + /// Instantiate a new Beacon Chain, from genesis. + pub fn genesis( + state_store: Arc>, + block_store: Arc>, + slot_clock: U, + spec: ChainSpec, + ) -> Result { + if spec.initial_validators.is_empty() { + return Err(Error::InsufficientValidators); + } + + let genesis_state = genesis_beacon_state(&spec); + let state_root = genesis_state.canonical_root(); + state_store.put(&state_root, &ssz_encode(&genesis_state)[..])?; + + let genesis_block = genesis_beacon_block(state_root, &spec); + let block_root = genesis_block.canonical_root(); + block_store.put(&block_root, &ssz_encode(&genesis_block)[..])?; + + let block_graph = BlockGraph::new(); + block_graph.add_leaf(&Hash256::zero(), block_root.clone()); + + let finalized_head = RwLock::new(CheckPoint::new( + genesis_block.clone(), + block_root.clone(), + genesis_state.clone(), + state_root.clone(), + )); + let justified_head = RwLock::new(CheckPoint::new( + genesis_block.clone(), + block_root.clone(), + genesis_state.clone(), + state_root.clone(), + )); + let canonical_head = RwLock::new(CheckPoint::new( + genesis_block.clone(), + block_root.clone(), + genesis_state.clone(), + state_root.clone(), + )); + let attestation_aggregator = RwLock::new(AttestationAggregator::new()); + + let latest_attestation_targets = RwLock::new(AttestationTargets::new()); + + Ok(Self { + block_store, + state_store, + slot_clock, + block_graph, + attestation_aggregator, + state: RwLock::new(genesis_state.clone()), + justified_head, + finalized_head, + canonical_head, + latest_attestation_targets, + spec: spec, + }) + } + + /// Update the canonical head to some new values. + pub fn update_canonical_head( + &self, + new_beacon_block: BeaconBlock, + new_beacon_block_root: Hash256, + new_beacon_state: BeaconState, + new_beacon_state_root: Hash256, + ) { + let mut head = self.canonical_head.write(); + head.update( + new_beacon_block, + new_beacon_block_root, + new_beacon_state, + new_beacon_state_root, + ); + } + + /// Returns a read-lock guarded `CheckPoint` struct for reading the head (as chosen by the + /// fork-choice rule). + /// + /// It is important to note that the `beacon_state` returned may not match the present slot. It + /// is the state as it was when the head block was recieved, which could be some slots prior to + /// now. + pub fn head(&self) -> RwLockReadGuard { + self.canonical_head.read() + } + + /// Update the justified head to some new values. + pub fn update_finalized_head( + &self, + new_beacon_block: BeaconBlock, + new_beacon_block_root: Hash256, + new_beacon_state: BeaconState, + new_beacon_state_root: Hash256, + ) { + let mut finalized_head = self.finalized_head.write(); + finalized_head.update( + new_beacon_block, + new_beacon_block_root, + new_beacon_state, + new_beacon_state_root, + ); + } + + /// Returns a read-lock guarded `CheckPoint` struct for reading the justified head (as chosen, + /// indirectly, by the fork-choice rule). + pub fn finalized_head(&self) -> RwLockReadGuard { + self.finalized_head.read() + } + + /// Advance the `self.state` `BeaconState` to the supplied slot. + /// + /// This will perform per_slot and per_epoch processing as required. + /// + /// The `previous_block_root` will be set to the root of the current head block (as determined + /// by the fork-choice rule). + /// + /// It is important to note that this is _not_ the state corresponding to the canonical head + /// block, instead it is that state which may or may not have had additional per slot/epoch + /// processing applied to it. + pub fn advance_state(&self, slot: u64) -> Result<(), SlotProcessingError> { + let state_slot = self.state.read().slot; + let head_block_root = self.head().beacon_block_root; + for _ in state_slot..slot { + self.state + .write() + .per_slot_processing(head_block_root.clone(), &self.spec)?; + } + Ok(()) + } + + /// Returns the the validator index (if any) for the given public key. + /// + /// Information is retrieved from the present `beacon_state.validator_registry`. + pub fn validator_index(&self, pubkey: &PublicKey) -> Option { + for (i, validator) in self + .head() + .beacon_state + .validator_registry + .iter() + .enumerate() + { + if validator.pubkey == *pubkey { + return Some(i); + } + } + None + } + + /// Returns the number of slots the validator has been required to propose. + /// + /// Returns `None` if the `validator_index` is invalid. + /// + /// Information is retrieved from the present `beacon_state.validator_registry`. + pub fn proposer_slots(&self, validator_index: usize) -> Option { + if let Some(validator) = self.state.read().validator_registry.get(validator_index) { + Some(validator.proposer_slots) + } else { + None + } + } + + /// Reads the slot clock, returns `None` if the slot is unavailable. + /// + /// The slot might be unavailable due to an error with the system clock, or if the present time + /// is before genesis (i.e., a negative slot). + /// + /// This is distinct to `present_slot`, which simply reads the latest state. If a + /// call to `read_slot_clock` results in a higher slot than a call to `present_slot`, + /// `self.state` should undergo per slot processing. + pub fn read_slot_clock(&self) -> Option { + match self.slot_clock.present_slot() { + Ok(some_slot) => some_slot, + _ => None, + } + } + + /// Returns slot of the present state. + /// + /// This is distinct to `read_slot_clock`, which reads from the actual system clock. If + /// `self.state` has not been transitioned it is possible for the system clock to be on a + /// different slot to what is returned from this call. + pub fn present_slot(&self) -> u64 { + self.state.read().slot + } + + /// Returns the block proposer for a given slot. + /// + /// Information is read from the present `beacon_state` shuffling, so only information from the + /// present and prior epoch is available. + pub fn block_proposer(&self, slot: u64) -> Result { + let index = self + .state + .read() + .get_beacon_proposer_index(slot, &self.spec)?; + + Ok(index) + } + + /// Returns the justified slot for the present state. + pub fn justified_slot(&self) -> u64 { + self.state.read().justified_slot + } + + /// Returns the attestation slot and shard for a given validator index. + /// + /// Information is read from the current state, so only information from the present and prior + /// epoch is available. + pub fn validator_attestion_slot_and_shard( + &self, + validator_index: usize, + ) -> Result, CommitteesError> { + if let Some((slot, shard, _committee)) = self + .state + .read() + .attestation_slot_and_shard_for_validator(validator_index, &self.spec)? + { + Ok(Some((slot, shard))) + } else { + Ok(None) + } + } + + /// Produce an `AttestationData` that is valid for the present `slot` and given `shard`. + pub fn produce_attestation_data(&self, shard: u64) -> Result { + let justified_slot = self.justified_slot(); + let justified_block_root = self + .state + .read() + .get_block_root(justified_slot, &self.spec) + .ok_or_else(|| Error::BadRecentBlockRoots)? + .clone(); + + let epoch_boundary_root = self + .state + .read() + .get_block_root( + self.state.read().current_epoch_start_slot(&self.spec), + &self.spec, + ) + .ok_or_else(|| Error::BadRecentBlockRoots)? + .clone(); + + Ok(AttestationData { + slot: self.state.read().slot, + shard, + beacon_block_root: self.head().beacon_block_root.clone(), + epoch_boundary_root, + shard_block_root: Hash256::zero(), + latest_crosslink_root: Hash256::zero(), + justified_slot, + justified_block_root, + }) + } + + /// Validate a `FreeAttestation` and either: + /// + /// - Create a new `Attestation`. + /// - Aggregate it to an existing `Attestation`. + pub fn process_free_attestation( + &self, + free_attestation: FreeAttestation, + ) -> Result { + self.attestation_aggregator + .write() + .process_free_attestation(&self.state.read(), &free_attestation, &self.spec) + .map_err(|e| e.into()) + } + + /// Set the latest attestation target for some validator. + pub fn insert_latest_attestation_target(&self, validator_index: u64, block_root: Hash256) { + let mut targets = self.latest_attestation_targets.write(); + targets.insert(validator_index, block_root); + } + + /// Get the latest attestation target for some validator. + pub fn get_latest_attestation_target(&self, validator_index: u64) -> Option { + let targets = self.latest_attestation_targets.read(); + + match targets.get(validator_index) { + Some(hash) => Some(hash.clone()), + None => None, + } + } + + /// Dumps the entire canonical chain, from the head to genesis to a vector for analysis. + /// + /// This could be a very expensive operation and should only be done in testing/analysis + /// activities. + pub fn chain_dump(&self) -> Result, Error> { + let mut dump = vec![]; + + let mut last_slot = CheckPoint { + beacon_block: self.head().beacon_block.clone(), + beacon_block_root: self.head().beacon_block_root, + beacon_state: self.head().beacon_state.clone(), + beacon_state_root: self.head().beacon_state_root, + }; + + dump.push(last_slot.clone()); + + loop { + let beacon_block_root = last_slot.beacon_block.parent_root; + + if beacon_block_root == self.spec.zero_hash { + break; // Genesis has been reached. + } + + let beacon_block = self + .block_store + .get_deserialized(&beacon_block_root)? + .ok_or_else(|| { + Error::DBInconsistent(format!("Missing block {}", beacon_block_root)) + })?; + let beacon_state_root = beacon_block.state_root; + let beacon_state = self + .state_store + .get_deserialized(&beacon_state_root)? + .ok_or_else(|| { + Error::DBInconsistent(format!("Missing state {}", beacon_state_root)) + })?; + + let slot = CheckPoint { + beacon_block, + beacon_block_root, + beacon_state, + beacon_state_root, + }; + + dump.push(slot.clone()); + last_slot = slot; + } + + Ok(dump) + } + + /// Accept some block and attempt to add it to block DAG. + /// + /// Will accept blocks from prior slots, however it will reject any block from a future slot. + pub fn process_block(&self, block: BeaconBlock) -> Result { + debug!("Processing block with slot {}...", block.slot()); + + let block_root = block.canonical_root(); + + let present_slot = self.present_slot(); + + if block.slot > present_slot { + return Ok(BlockProcessingOutcome::InvalidBlock( + InvalidBlock::FutureSlot, + )); + } + + // Load the blocks parent block from the database, returning invalid if that block is not + // found. + let parent_block_root = block.parent_root; + let parent_block = match self.block_store.get_reader(&parent_block_root)? { + Some(parent_root) => parent_root, + None => { + return Ok(BlockProcessingOutcome::InvalidBlock( + InvalidBlock::ParentUnknown, + )); + } + }; + + // Load the parent blocks state from the database, returning an error if it is not found. + // It is an error because if know the parent block we should also know the parent state. + let parent_state_root = parent_block.state_root(); + let parent_state = self + .state_store + .get_reader(&parent_state_root)? + .ok_or(Error::DBInconsistent(format!( + "Missing state {}", + parent_state_root + )))? + .into_beacon_state() + .ok_or(Error::DBInconsistent(format!( + "State SSZ invalid {}", + parent_state_root + )))?; + + // TODO: check the block proposer signature BEFORE doing a state transition. This will + // significantly lower exposure surface to DoS attacks. + + // Transition the parent state to the present slot. + let mut state = parent_state; + for _ in state.slot..present_slot { + if let Err(e) = state.per_slot_processing(parent_block_root.clone(), &self.spec) { + return Ok(BlockProcessingOutcome::InvalidBlock( + InvalidBlock::SlotProcessingError(e), + )); + } + } + + // Apply the recieved block to its parent state (which has been transitioned into this + // slot). + if let Err(e) = state.per_block_processing(&block, &self.spec) { + return Ok(BlockProcessingOutcome::InvalidBlock( + InvalidBlock::PerBlockProcessingError(e), + )); + } + + let state_root = state.canonical_root(); + + if block.state_root != state_root { + return Ok(BlockProcessingOutcome::InvalidBlock( + InvalidBlock::StateRootMismatch, + )); + } + + // Store the block and state. + self.block_store.put(&block_root, &ssz_encode(&block)[..])?; + self.state_store.put(&state_root, &ssz_encode(&state)[..])?; + + // Update the block DAG. + self.block_graph + .add_leaf(&parent_block_root, block_root.clone()); + + // If the parent block was the parent_block, automatically update the canonical head. + // + // TODO: this is a first-in-best-dressed scenario that is not ideal; fork_choice should be + // run instead. + if self.head().beacon_block_root == parent_block_root { + self.update_canonical_head( + block.clone(), + block_root.clone(), + state.clone(), + state_root.clone(), + ); + // Update the local state variable. + *self.state.write() = state.clone(); + } + + Ok(BlockProcessingOutcome::ValidBlock(ValidBlock::Processed)) + } + + /// Produce a new block at the present slot. + /// + /// The produced block will not be inheriently valid, it must be signed by a block producer. + /// Block signing is out of the scope of this function and should be done by a separate program. + pub fn produce_block(&self, randao_reveal: Signature) -> Option<(BeaconBlock, BeaconState)> { + debug!("Producing block at slot {}...", self.state.read().slot); + + let mut state = self.state.read().clone(); + + trace!("Finding attestations for new block..."); + + let attestations = self + .attestation_aggregator + .read() + .get_attestations_for_state(&state, &self.spec); + + trace!( + "Inserting {} attestation(s) into new block.", + attestations.len() + ); + + let parent_root = state + .get_block_root(state.slot.saturating_sub(1), &self.spec)? + .clone(); + + let mut block = BeaconBlock { + slot: state.slot, + parent_root, + state_root: Hash256::zero(), // Updated after the state is calculated. + randao_reveal: randao_reveal, + eth1_data: Eth1Data { + // TODO: replace with real data + deposit_root: Hash256::zero(), + block_hash: Hash256::zero(), + }, + signature: self.spec.empty_signature.clone(), // To be completed by a validator. + body: BeaconBlockBody { + proposer_slashings: vec![], + casper_slashings: vec![], + attestations: attestations, + custody_reseeds: vec![], + custody_challenges: vec![], + custody_responses: vec![], + deposits: vec![], + exits: vec![], + }, + }; + + state + .per_block_processing_without_verifying_block_signature(&block, &self.spec) + .ok()?; + + let state_root = state.canonical_root(); + + block.state_root = state_root; + + trace!("Block produced."); + + Some((block, state)) + } +} + +impl From for Error { + fn from(e: DBError) -> Error { + Error::DBError(e.message) + } +} + +impl From for Error { + fn from(e: CommitteesError) -> Error { + Error::CommitteesError(e) + } +} diff --git a/beacon_node/beacon_chain/src/block_processing.rs b/beacon_node/beacon_chain/src/block_processing.rs deleted file mode 100644 index 249bd9df7..000000000 --- a/beacon_node/beacon_chain/src/block_processing.rs +++ /dev/null @@ -1,151 +0,0 @@ -use super::{BeaconChain, ClientDB, DBError, SlotClock}; -use log::debug; -use ssz::{ssz_encode, Encodable}; -use types::{ - beacon_state::{BlockProcessingError, SlotProcessingError}, - readers::{BeaconBlockReader, BeaconStateReader}, - Hash256, -}; - -#[derive(Debug, PartialEq)] -pub enum ValidBlock { - /// The block was sucessfully processed. - Processed, -} - -#[derive(Debug, PartialEq)] -pub enum InvalidBlock { - /// The block slot is greater than the present slot. - FutureSlot, - /// The block state_root does not match the generated state. - StateRootMismatch, -} - -#[derive(Debug, PartialEq)] -pub enum Outcome { - /// The block was sucessfully validated. - ValidBlock(ValidBlock), - /// The block was not sucessfully validated. - InvalidBlock(InvalidBlock), -} - -#[derive(Debug, PartialEq)] -pub enum Error { - /// There was in internal database error. - DBError(String), - /// The block SSZ encoding is unreadable. - UnableToDecodeBlock, - /// The blocks parent state is not in the database. This is an internal error. - MissingParentState(Hash256), - /// The blocks parent state is in the database, but invalid. This is an internal error. - InvalidParentState(Hash256), - /// The blocks parent state is in the database, but invalid. This is an internal error. - MissingBeaconBlock(Hash256), - /// The parent block is not in the database. The block should not be processed. - InvalidBeaconBlock(Hash256), - /// The parent block is not in the database, but invalid. This is an internal error. - MissingParentBlock(Hash256), - /// There was an error whilst advancing the parent state to the present slot. This is an - /// internal error. - SlotProcessingError(SlotProcessingError), - /// There was an error whilst processing the block against it's state. The block is invalid. - PerBlockProcessingError(BlockProcessingError), -} - -impl BeaconChain -where - T: ClientDB, - U: SlotClock, -{ - /// Accept some block and attempt to add it to block DAG. - /// - /// Will accept blocks from prior slots, however it will reject any block from a future slot. - pub fn process_block(&self, block: V) -> Result - where - V: BeaconBlockReader + Encodable + Sized, - { - debug!("Processing block with slot {}...", block.slot()); - - let block = block - .into_beacon_block() - .ok_or(Error::UnableToDecodeBlock)?; - let block_root = block.canonical_root(); - - let present_slot = self.present_slot(); - - if block.slot() > present_slot { - return Ok(Outcome::InvalidBlock(InvalidBlock::FutureSlot)); - } - - let parent_block_root = block.parent_root(); - let parent_block = self - .block_store - .get_reader(&parent_block_root)? - .ok_or(Error::MissingParentBlock(parent_block_root))?; - - let parent_state_root = parent_block.state_root(); - let parent_state = self - .state_store - .get_reader(&parent_state_root)? - .ok_or(Error::MissingParentState(parent_state_root))? - .into_beacon_state() - .ok_or(Error::InvalidParentState(parent_state_root))?; - - let mut state = parent_state; - - for _ in state.slot..present_slot { - state.per_slot_processing(parent_block_root.clone(), &self.spec)?; - } - - state.per_block_processing(&block, &self.spec)?; - - let state_root = state.canonical_root(); - - if block.state_root != state_root { - return Ok(Outcome::InvalidBlock(InvalidBlock::StateRootMismatch)); - } - - // Store the block and state. - self.block_store.put(&block_root, &ssz_encode(&block)[..])?; - self.state_store.put(&state_root, &ssz_encode(&state)[..])?; - - // Update the block DAG. - self.block_graph - .add_leaf(&parent_block_root, block_root.clone()); - - // If the parent block was the parent_block, automatically update the canonical head. - // - // TODO: this is a first-in-best-dressed scenario that is not ideal; fork_choice should be - // run instead. - if self.head().beacon_block_root == parent_block_root { - self.update_canonical_head( - block.clone(), - block_root.clone(), - state.clone(), - state_root.clone(), - ); - // Update the local state variable. - *self.state.write() = state.clone(); - } - - Ok(Outcome::ValidBlock(ValidBlock::Processed)) - } -} - -impl From for Error { - fn from(e: DBError) -> Error { - Error::DBError(e.message) - } -} - -impl From for Error { - fn from(e: SlotProcessingError) -> Error { - Error::SlotProcessingError(e) - } -} - -impl From for Error { - fn from(e: BlockProcessingError) -> Error { - Error::PerBlockProcessingError(e) - } -} diff --git a/beacon_node/beacon_chain/src/block_production.rs b/beacon_node/beacon_chain/src/block_production.rs deleted file mode 100644 index eecf1af06..000000000 --- a/beacon_node/beacon_chain/src/block_production.rs +++ /dev/null @@ -1,102 +0,0 @@ -use super::{BeaconChain, ClientDB, DBError, SlotClock}; -use bls::Signature; -use log::debug; -use types::{ - beacon_state::{BlockProcessingError, SlotProcessingError}, - BeaconBlock, BeaconBlockBody, BeaconState, Eth1Data, Hash256, -}; - -#[derive(Debug, PartialEq)] -pub enum Error { - DBError(String), - SlotProcessingError(SlotProcessingError), - PerBlockProcessingError(BlockProcessingError), -} - -impl BeaconChain -where - T: ClientDB, - U: SlotClock, -{ - /// Produce a new block at the present slot. - /// - /// The produced block will not be inheriently valid, it must be signed by a block producer. - /// Block signing is out of the scope of this function and should be done by a separate program. - pub fn produce_block( - &self, - randao_reveal: Signature, - ) -> Result<(BeaconBlock, BeaconState), Error> { - debug!("Starting block production..."); - - let mut state = self.state.read().clone(); - - debug!("Finding attesatations for new block..."); - - let attestations = self - .attestation_aggregator - .read() - .get_attestations_for_state(&state, &self.spec); - - debug!( - "Inserting {} attestation(s) into new block.", - attestations.len() - ); - - let parent_root = state - .get_block_root(state.slot.saturating_sub(1), &self.spec) - // TODO: fix unwrap - .unwrap() - .clone(); - - let mut block = BeaconBlock { - slot: state.slot, - parent_root, - state_root: Hash256::zero(), // Updated after the state is calculated. - randao_reveal: randao_reveal, - eth1_data: Eth1Data { - // TODO: replace with real data - deposit_root: Hash256::zero(), - block_hash: Hash256::zero(), - }, - signature: self.spec.empty_signature.clone(), // To be completed by a validator. - body: BeaconBlockBody { - proposer_slashings: vec![], - casper_slashings: vec![], - attestations: attestations, - custody_reseeds: vec![], - custody_challenges: vec![], - custody_responses: vec![], - deposits: vec![], - exits: vec![], - }, - }; - - state.per_block_processing_without_verifying_block_signature(&block, &self.spec)?; - - let state_root = state.canonical_root(); - - block.state_root = state_root; - - debug!("Block produced."); - - Ok((block, state)) - } -} - -impl From for Error { - fn from(e: DBError) -> Error { - Error::DBError(e.message) - } -} - -impl From for Error { - fn from(e: SlotProcessingError) -> Error { - Error::SlotProcessingError(e) - } -} - -impl From for Error { - fn from(e: BlockProcessingError) -> Error { - Error::PerBlockProcessingError(e) - } -} diff --git a/beacon_node/beacon_chain/src/canonical_head.rs b/beacon_node/beacon_chain/src/canonical_head.rs deleted file mode 100644 index 32f2dc000..000000000 --- a/beacon_node/beacon_chain/src/canonical_head.rs +++ /dev/null @@ -1,47 +0,0 @@ -use crate::{BeaconChain, CheckPoint, ClientDB, SlotClock}; -use parking_lot::RwLockReadGuard; -use types::{beacon_state::SlotProcessingError, BeaconBlock, BeaconState, Hash256}; - -#[derive(Debug, PartialEq)] -pub enum Error { - SlotProcessingError(SlotProcessingError), -} - -impl BeaconChain -where - T: ClientDB, - U: SlotClock, -{ - /// Update the canonical head to some new values. - pub fn update_canonical_head( - &self, - new_beacon_block: BeaconBlock, - new_beacon_block_root: Hash256, - new_beacon_state: BeaconState, - new_beacon_state_root: Hash256, - ) { - let mut head = self.canonical_head.write(); - head.update( - new_beacon_block, - new_beacon_block_root, - new_beacon_state, - new_beacon_state_root, - ); - } - - /// Returns a read-lock guarded `CheckPoint` struct for reading the head (as chosen by the - /// fork-choice rule). - /// - /// It is important to note that the `beacon_state` returned may not match the present slot. It - /// is the state as it was when the head block was recieved, which could be some slots prior to - /// now. - pub fn head(&self) -> RwLockReadGuard { - self.canonical_head.read() - } -} - -impl From for Error { - fn from(e: SlotProcessingError) -> Error { - Error::SlotProcessingError(e) - } -} diff --git a/beacon_node/beacon_chain/src/dump.rs b/beacon_node/beacon_chain/src/dump.rs deleted file mode 100644 index b2b031504..000000000 --- a/beacon_node/beacon_chain/src/dump.rs +++ /dev/null @@ -1,69 +0,0 @@ -use super::{BeaconChain, CheckPoint, ClientDB, DBError, SlotClock}; -use types::Hash256; - -#[derive(Debug, Clone)] -pub enum Error { - /// There was an error reading from the database. This is an internal error. - DBError(String), - /// There is a missing (or invalid) block in the database. This is an internal error. - MissingBlock(Hash256), -} - -impl BeaconChain -where - T: ClientDB, - U: SlotClock, -{ - /// Dumps the entire canonical chain, from the head to genesis to a vector for analysis. - /// - /// This could be a very expensive operation and should only be done in testing/analysis - /// activities. - pub fn chain_dump(&self) -> Result, Error> { - let mut dump = vec![]; - - let mut last_slot = CheckPoint { - beacon_block: self.head().beacon_block.clone(), - beacon_block_root: self.head().beacon_block_root, - beacon_state: self.head().beacon_state.clone(), - beacon_state_root: self.head().beacon_state_root, - }; - - dump.push(last_slot.clone()); - - loop { - let beacon_block_root = last_slot.beacon_block.parent_root; - - if beacon_block_root == self.spec.zero_hash { - break; // Genesis has been reached. - } - - let beacon_block = self - .block_store - .get_deserialized(&beacon_block_root)? - .ok_or_else(|| Error::MissingBlock(beacon_block_root))?; - let beacon_state_root = beacon_block.state_root; - let beacon_state = self - .state_store - .get_deserialized(&beacon_state_root)? - .ok_or_else(|| Error::MissingBlock(beacon_state_root))?; - - let slot = CheckPoint { - beacon_block, - beacon_block_root, - beacon_state, - beacon_state_root, - }; - - dump.push(slot.clone()); - last_slot = slot; - } - - Ok(dump) - } -} - -impl From for Error { - fn from(e: DBError) -> Error { - Error::DBError(e.message) - } -} diff --git a/beacon_node/beacon_chain/src/finalized_head.rs b/beacon_node/beacon_chain/src/finalized_head.rs deleted file mode 100644 index ac2917fdb..000000000 --- a/beacon_node/beacon_chain/src/finalized_head.rs +++ /dev/null @@ -1,32 +0,0 @@ -use crate::{BeaconChain, CheckPoint, ClientDB, SlotClock}; -use parking_lot::RwLockReadGuard; -use types::{BeaconBlock, BeaconState, Hash256}; - -impl BeaconChain -where - T: ClientDB, - U: SlotClock, -{ - /// Update the justified head to some new values. - pub fn update_finalized_head( - &self, - new_beacon_block: BeaconBlock, - new_beacon_block_root: Hash256, - new_beacon_state: BeaconState, - new_beacon_state_root: Hash256, - ) { - let mut finalized_head = self.finalized_head.write(); - finalized_head.update( - new_beacon_block, - new_beacon_block_root, - new_beacon_state, - new_beacon_state_root, - ); - } - - /// Returns a read-lock guarded `CheckPoint` struct for reading the justified head (as chosen, - /// indirectly, by the fork-choice rule). - pub fn finalized_head(&self) -> RwLockReadGuard { - self.finalized_head.read() - } -} diff --git a/beacon_node/beacon_chain/src/getters.rs b/beacon_node/beacon_chain/src/getters.rs deleted file mode 100644 index cba519743..000000000 --- a/beacon_node/beacon_chain/src/getters.rs +++ /dev/null @@ -1,100 +0,0 @@ -use super::{BeaconChain, ClientDB, SlotClock}; -use types::{beacon_state::CommitteesError, PublicKey}; - -impl BeaconChain -where - T: ClientDB, - U: SlotClock, -{ - /// Returns the the validator index (if any) for the given public key. - /// - /// Information is retrieved from the present `beacon_state.validator_registry`. - pub fn validator_index(&self, pubkey: &PublicKey) -> Option { - for (i, validator) in self - .head() - .beacon_state - .validator_registry - .iter() - .enumerate() - { - if validator.pubkey == *pubkey { - return Some(i); - } - } - None - } - - /// Returns the number of slots the validator has been required to propose. - /// - /// Returns `None` if the `validator_index` is invalid. - /// - /// Information is retrieved from the present `beacon_state.validator_registry`. - pub fn proposer_slots(&self, validator_index: usize) -> Option { - if let Some(validator) = self.state.read().validator_registry.get(validator_index) { - Some(validator.proposer_slots) - } else { - None - } - } - - /// Reads the slot clock, returns `None` if the slot is unavailable. - /// - /// The slot might be unavailable due to an error with the system clock, or if the present time - /// is before genesis (i.e., a negative slot). - /// - /// This is distinct to `present_slot`, which simply reads the latest state. If a - /// call to `read_slot_clock` results in a higher slot than a call to `present_slot`, - /// `self.state` should undergo per slot processing. - pub fn read_slot_clock(&self) -> Option { - match self.slot_clock.present_slot() { - Ok(some_slot) => some_slot, - _ => None, - } - } - - /// Returns slot of the present state. - /// - /// This is distinct to `read_slot_clock`, which reads from the actual system clock. If - /// `self.state` has not been transitioned it is possible for the system clock to be on a - /// different slot to what is returned from this call. - pub fn present_slot(&self) -> u64 { - self.state.read().slot - } - - /// Returns the block proposer for a given slot. - /// - /// Information is read from the present `beacon_state` shuffling, so only information from the - /// present and prior epoch is available. - pub fn block_proposer(&self, slot: u64) -> Result { - let index = self - .state - .read() - .get_beacon_proposer_index(slot, &self.spec)?; - - Ok(index) - } - - /// Returns the justified slot for the present state. - pub fn justified_slot(&self) -> u64 { - self.state.read().justified_slot - } - - /// Returns the attestation slot and shard for a given validator index. - /// - /// Information is read from the current state, so only information from the present and prior - /// epoch is available. - pub fn validator_attestion_slot_and_shard( - &self, - validator_index: usize, - ) -> Result, CommitteesError> { - if let Some((slot, shard, _committee)) = self - .state - .read() - .attestation_slot_and_shard_for_validator(validator_index, &self.spec)? - { - Ok(Some((slot, shard))) - } else { - Ok(None) - } - } -} diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 68c0c3c9c..ef7273f36 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -1,129 +1,9 @@ mod attestation_aggregator; -pub mod attestation_processing; -mod attestation_production; mod attestation_targets; +mod beacon_chain; mod block_graph; -pub mod block_processing; -pub mod block_production; -mod canonical_head; mod checkpoint; -pub mod dump; -mod finalized_head; -mod getters; mod lmd_ghost; -mod state; -use self::attestation_targets::AttestationTargets; -use self::block_graph::BlockGraph; -use attestation_aggregator::AttestationAggregator; -use db::{ - stores::{BeaconBlockStore, BeaconStateStore}, - ClientDB, DBError, -}; -use genesis::{genesis_beacon_block, genesis_beacon_state, GenesisError}; -use parking_lot::RwLock; -use slot_clock::SlotClock; -use ssz::ssz_encode; -use std::sync::Arc; -use types::{BeaconState, ChainSpec, Hash256}; - -pub use self::block_processing::Outcome as BlockProcessingOutcome; +pub use self::beacon_chain::{BeaconChain, Error}; pub use self::checkpoint::CheckPoint; - -#[derive(Debug, PartialEq)] -pub enum BeaconChainError { - InsufficientValidators, - GenesisError(GenesisError), - DBError(String), -} - -pub struct BeaconChain { - pub block_store: Arc>, - pub state_store: Arc>, - pub slot_clock: U, - pub block_graph: BlockGraph, - pub attestation_aggregator: RwLock, - canonical_head: RwLock, - finalized_head: RwLock, - justified_head: RwLock, - pub state: RwLock, - pub latest_attestation_targets: RwLock, - pub spec: ChainSpec, -} - -impl BeaconChain -where - T: ClientDB, - U: SlotClock, -{ - /// Instantiate a new Beacon Chain, from genesis. - pub fn genesis( - state_store: Arc>, - block_store: Arc>, - slot_clock: U, - spec: ChainSpec, - ) -> Result { - if spec.initial_validators.is_empty() { - return Err(BeaconChainError::InsufficientValidators); - } - - let genesis_state = genesis_beacon_state(&spec)?; - let state_root = genesis_state.canonical_root(); - state_store.put(&state_root, &ssz_encode(&genesis_state)[..])?; - - let genesis_block = genesis_beacon_block(state_root, &spec); - let block_root = genesis_block.canonical_root(); - block_store.put(&block_root, &ssz_encode(&genesis_block)[..])?; - - let block_graph = BlockGraph::new(); - block_graph.add_leaf(&Hash256::zero(), block_root.clone()); - - let finalized_head = RwLock::new(CheckPoint::new( - genesis_block.clone(), - block_root.clone(), - genesis_state.clone(), - state_root.clone(), - )); - let justified_head = RwLock::new(CheckPoint::new( - genesis_block.clone(), - block_root.clone(), - genesis_state.clone(), - state_root.clone(), - )); - let canonical_head = RwLock::new(CheckPoint::new( - genesis_block.clone(), - block_root.clone(), - genesis_state.clone(), - state_root.clone(), - )); - let attestation_aggregator = RwLock::new(AttestationAggregator::new()); - - let latest_attestation_targets = RwLock::new(AttestationTargets::new()); - - Ok(Self { - block_store, - state_store, - slot_clock, - block_graph, - attestation_aggregator, - state: RwLock::new(genesis_state.clone()), - justified_head, - finalized_head, - canonical_head, - latest_attestation_targets, - spec: spec, - }) - } -} - -impl From for BeaconChainError { - fn from(e: DBError) -> BeaconChainError { - BeaconChainError::DBError(e.message) - } -} - -impl From for BeaconChainError { - fn from(e: GenesisError) -> BeaconChainError { - BeaconChainError::GenesisError(e) - } -} diff --git a/beacon_node/beacon_chain/src/lmd_ghost.rs b/beacon_node/beacon_chain/src/lmd_ghost.rs index 176d4a686..4d0a68c37 100644 --- a/beacon_node/beacon_chain/src/lmd_ghost.rs +++ b/beacon_node/beacon_chain/src/lmd_ghost.rs @@ -1,9 +1,9 @@ -use super::{BeaconChain, SlotClock}; +use crate::BeaconChain; use db::{ stores::{BeaconBlockAtSlotError, BeaconBlockStore}, ClientDB, DBError, }; -use slot_clock::TestingSlotClockError; +use slot_clock::{SlotClock, TestingSlotClockError}; use std::collections::HashSet; use std::sync::Arc; use types::{ diff --git a/beacon_node/beacon_chain/src/state.rs b/beacon_node/beacon_chain/src/state.rs deleted file mode 100644 index db1119c8a..000000000 --- a/beacon_node/beacon_chain/src/state.rs +++ /dev/null @@ -1,29 +0,0 @@ -use crate::{BeaconChain, ClientDB, SlotClock}; -use types::beacon_state::SlotProcessingError; - -impl BeaconChain -where - T: ClientDB, - U: SlotClock, -{ - /// Advance the `self.state` `BeaconState` to the supplied slot. - /// - /// This will perform per_slot and per_epoch processing as required. - /// - /// The `previous_block_root` will be set to the root of the current head block (as determined - /// by the fork-choice rule). - /// - /// It is important to note that this is _not_ the state corresponding to the canonical head - /// block, instead it is that state which may or may not have had additional per slot/epoch - /// processing applied to it. - pub fn advance_state(&self, slot: u64) -> Result<(), SlotProcessingError> { - let state_slot = self.state.read().slot; - let head_block_root = self.head().beacon_block_root; - for _ in state_slot..slot { - self.state - .write() - .per_slot_processing(head_block_root.clone(), &self.spec)?; - } - Ok(()) - } -} From d83bafae10591af16cfa1b06a23a193cc42a8271 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 5 Feb 2019 05:26:47 +1100 Subject: [PATCH 095/107] Fix compiler errors from BeaconChain refactor. I accidentally forgot to include this in the last commit, my bad! --- beacon_node/beacon_chain/src/beacon_chain.rs | 2 +- beacon_node/beacon_chain/test_harness/src/harness.rs | 4 ++-- .../test_harness/src/validator/beacon_node/producer.rs | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 436d5a246..9eaa7d7c1 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -415,7 +415,7 @@ where /// Accept some block and attempt to add it to block DAG. /// /// Will accept blocks from prior slots, however it will reject any block from a future slot. - pub fn process_block(&self, block: BeaconBlock) -> Result { + pub fn process_block(&self, block: BeaconBlock) -> Result { debug!("Processing block with slot {}...", block.slot()); let block_root = block.canonical_root(); diff --git a/beacon_node/beacon_chain/test_harness/src/harness.rs b/beacon_node/beacon_chain/test_harness/src/harness.rs index 7bc86a7e1..12ed40755 100644 --- a/beacon_node/beacon_chain/test_harness/src/harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/harness.rs @@ -1,6 +1,6 @@ use super::TestValidator; use beacon_chain::BeaconChain; -pub use beacon_chain::{dump::Error as DumpError, CheckPoint}; +pub use beacon_chain::{CheckPoint, Error as BeaconChainError}; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, MemoryDB, @@ -219,7 +219,7 @@ impl BeaconChainHarness { } /// Dump all blocks and states from the canonical beacon chain. - pub fn chain_dump(&self) -> Result, DumpError> { + pub fn chain_dump(&self) -> Result, BeaconChainError> { self.beacon_chain.chain_dump() } diff --git a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/producer.rs b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/producer.rs index 36c82f6cc..d188d8554 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/producer.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/producer.rs @@ -30,7 +30,9 @@ impl BeaconBlockNode for DirectBeaconNode { let (block, _state) = self .beacon_chain .produce_block(randao_reveal.clone()) - .map_err(|e| BeaconBlockNodeError::RemoteFailure(format!("{:?}", e)))?; + .ok_or_else(|| { + BeaconBlockNodeError::RemoteFailure(format!("Did not produce block.")) + })?; if block.slot == slot { Ok(Some(block)) From 4008ec9b25c56ca29e22140dc549d7b7f251d20c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 5 Feb 2019 05:49:53 +1100 Subject: [PATCH 096/107] Re-org test_harness files for consistency --- .../src/validator/beacon_node/attester.rs | 26 ---- .../src/validator/beacon_node/mod.rs | 41 ------ .../src/validator/beacon_node/producer.rs | 58 -------- .../src/validator/direct_beacon_node.rs | 121 ++++++++++++++++ .../test_harness/src/validator/mod.rs | 137 +----------------- .../test_harness/src/validator/validator.rs | 133 +++++++++++++++++ 6 files changed, 257 insertions(+), 259 deletions(-) delete mode 100644 beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs delete mode 100644 beacon_node/beacon_chain/test_harness/src/validator/beacon_node/mod.rs delete mode 100644 beacon_node/beacon_chain/test_harness/src/validator/beacon_node/producer.rs create mode 100644 beacon_node/beacon_chain/test_harness/src/validator/direct_beacon_node.rs create mode 100644 beacon_node/beacon_chain/test_harness/src/validator/validator.rs diff --git a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs deleted file mode 100644 index e7198461e..000000000 --- a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/attester.rs +++ /dev/null @@ -1,26 +0,0 @@ -use super::DirectBeaconNode; -use attester::{BeaconNode as AttesterBeaconNode, BeaconNodeError as NodeError, PublishOutcome}; -use db::ClientDB; -use slot_clock::SlotClock; -use types::{AttestationData, FreeAttestation}; - -impl AttesterBeaconNode for DirectBeaconNode { - fn produce_attestation_data( - &self, - _slot: u64, - shard: u64, - ) -> Result, NodeError> { - match self.beacon_chain.produce_attestation_data(shard) { - Ok(attestation_data) => Ok(Some(attestation_data)), - Err(e) => Err(NodeError::RemoteFailure(format!("{:?}", e))), - } - } - - fn publish_attestation_data( - &self, - free_attestation: FreeAttestation, - ) -> Result { - self.published_attestations.write().push(free_attestation); - Ok(PublishOutcome::ValidAttestation) - } -} diff --git a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/mod.rs b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/mod.rs deleted file mode 100644 index 90f8943fc..000000000 --- a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/mod.rs +++ /dev/null @@ -1,41 +0,0 @@ -use beacon_chain::BeaconChain; -use db::ClientDB; -use parking_lot::RwLock; -use slot_clock::SlotClock; -use std::sync::Arc; -use types::{BeaconBlock, FreeAttestation}; - -mod attester; -mod producer; - -/// Connect directly to a borrowed `BeaconChain` instance so an attester/producer can request/submit -/// blocks/attestations. -/// -/// `BeaconBlock`s and `FreeAttestation`s are not actually published to the `BeaconChain`, instead -/// they are stored inside this struct. This is to allow one to benchmark the submission of the -/// block/attestation directly, or modify it before submission. -pub struct DirectBeaconNode { - beacon_chain: Arc>, - published_blocks: RwLock>, - published_attestations: RwLock>, -} - -impl DirectBeaconNode { - pub fn new(beacon_chain: Arc>) -> Self { - Self { - beacon_chain, - published_blocks: RwLock::new(vec![]), - published_attestations: RwLock::new(vec![]), - } - } - - /// Get the last published block (if any). - pub fn last_published_block(&self) -> Option { - Some(self.published_blocks.read().last()?.clone()) - } - - /// Get the last published attestation (if any). - pub fn last_published_free_attestation(&self) -> Option { - Some(self.published_attestations.read().last()?.clone()) - } -} diff --git a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/producer.rs b/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/producer.rs deleted file mode 100644 index d188d8554..000000000 --- a/beacon_node/beacon_chain/test_harness/src/validator/beacon_node/producer.rs +++ /dev/null @@ -1,58 +0,0 @@ -use super::DirectBeaconNode; -use block_producer::{ - BeaconNode as BeaconBlockNode, BeaconNodeError as BeaconBlockNodeError, PublishOutcome, -}; -use db::ClientDB; -use slot_clock::SlotClock; -use types::{BeaconBlock, PublicKey, Signature}; - -impl BeaconBlockNode for DirectBeaconNode { - /// Requests the `proposer_nonce` from the `BeaconChain`. - fn proposer_nonce(&self, pubkey: &PublicKey) -> Result { - let validator_index = self - .beacon_chain - .validator_index(pubkey) - .ok_or_else(|| BeaconBlockNodeError::RemoteFailure("pubkey unknown.".to_string()))?; - - self.beacon_chain - .proposer_slots(validator_index) - .ok_or_else(|| { - BeaconBlockNodeError::RemoteFailure("validator_index unknown.".to_string()) - }) - } - - /// Requests a new `BeaconBlock from the `BeaconChain`. - fn produce_beacon_block( - &self, - slot: u64, - randao_reveal: &Signature, - ) -> Result, BeaconBlockNodeError> { - let (block, _state) = self - .beacon_chain - .produce_block(randao_reveal.clone()) - .ok_or_else(|| { - BeaconBlockNodeError::RemoteFailure(format!("Did not produce block.")) - })?; - - if block.slot == slot { - Ok(Some(block)) - } else { - Err(BeaconBlockNodeError::RemoteFailure( - "Unable to produce at non-current slot.".to_string(), - )) - } - } - - /// A block is not _actually_ published to the `BeaconChain`, instead it is stored in the - /// `published_block_vec` and a successful `ValidBlock` is returned to the caller. - /// - /// The block may be retrieved and then applied to the `BeaconChain` manually, potentially in a - /// benchmarking scenario. - fn publish_beacon_block( - &self, - block: BeaconBlock, - ) -> Result { - self.published_blocks.write().push(block); - Ok(PublishOutcome::ValidBlock) - } -} diff --git a/beacon_node/beacon_chain/test_harness/src/validator/direct_beacon_node.rs b/beacon_node/beacon_chain/test_harness/src/validator/direct_beacon_node.rs new file mode 100644 index 000000000..ed71f28d3 --- /dev/null +++ b/beacon_node/beacon_chain/test_harness/src/validator/direct_beacon_node.rs @@ -0,0 +1,121 @@ +use attester::{ + BeaconNode as AttesterBeaconNode, BeaconNodeError as NodeError, + PublishOutcome as AttestationPublishOutcome, +}; +use beacon_chain::BeaconChain; +use block_producer::{ + BeaconNode as BeaconBlockNode, BeaconNodeError as BeaconBlockNodeError, + PublishOutcome as BlockPublishOutcome, +}; +use db::ClientDB; +use parking_lot::RwLock; +use slot_clock::SlotClock; +use std::sync::Arc; +use types::{AttestationData, BeaconBlock, FreeAttestation, PublicKey, Signature}; + +// mod attester; +// mod producer; + +/// Connect directly to a borrowed `BeaconChain` instance so an attester/producer can request/submit +/// blocks/attestations. +/// +/// `BeaconBlock`s and `FreeAttestation`s are not actually published to the `BeaconChain`, instead +/// they are stored inside this struct. This is to allow one to benchmark the submission of the +/// block/attestation directly, or modify it before submission. +pub struct DirectBeaconNode { + beacon_chain: Arc>, + published_blocks: RwLock>, + published_attestations: RwLock>, +} + +impl DirectBeaconNode { + pub fn new(beacon_chain: Arc>) -> Self { + Self { + beacon_chain, + published_blocks: RwLock::new(vec![]), + published_attestations: RwLock::new(vec![]), + } + } + + /// Get the last published block (if any). + pub fn last_published_block(&self) -> Option { + Some(self.published_blocks.read().last()?.clone()) + } + + /// Get the last published attestation (if any). + pub fn last_published_free_attestation(&self) -> Option { + Some(self.published_attestations.read().last()?.clone()) + } +} + +impl AttesterBeaconNode for DirectBeaconNode { + fn produce_attestation_data( + &self, + _slot: u64, + shard: u64, + ) -> Result, NodeError> { + match self.beacon_chain.produce_attestation_data(shard) { + Ok(attestation_data) => Ok(Some(attestation_data)), + Err(e) => Err(NodeError::RemoteFailure(format!("{:?}", e))), + } + } + + fn publish_attestation_data( + &self, + free_attestation: FreeAttestation, + ) -> Result { + self.published_attestations.write().push(free_attestation); + Ok(AttestationPublishOutcome::ValidAttestation) + } +} + +impl BeaconBlockNode for DirectBeaconNode { + /// Requests the `proposer_nonce` from the `BeaconChain`. + fn proposer_nonce(&self, pubkey: &PublicKey) -> Result { + let validator_index = self + .beacon_chain + .validator_index(pubkey) + .ok_or_else(|| BeaconBlockNodeError::RemoteFailure("pubkey unknown.".to_string()))?; + + self.beacon_chain + .proposer_slots(validator_index) + .ok_or_else(|| { + BeaconBlockNodeError::RemoteFailure("validator_index unknown.".to_string()) + }) + } + + /// Requests a new `BeaconBlock from the `BeaconChain`. + fn produce_beacon_block( + &self, + slot: u64, + randao_reveal: &Signature, + ) -> Result, BeaconBlockNodeError> { + let (block, _state) = self + .beacon_chain + .produce_block(randao_reveal.clone()) + .ok_or_else(|| { + BeaconBlockNodeError::RemoteFailure(format!("Did not produce block.")) + })?; + + if block.slot == slot { + Ok(Some(block)) + } else { + Err(BeaconBlockNodeError::RemoteFailure( + "Unable to produce at non-current slot.".to_string(), + )) + } + } + + /// A block is not _actually_ published to the `BeaconChain`, instead it is stored in the + /// `published_block_vec` and a successful `ValidBlock` is returned to the caller. + /// + /// The block may be retrieved and then applied to the `BeaconChain` manually, potentially in a + /// benchmarking scenario. + fn publish_beacon_block( + &self, + block: BeaconBlock, + ) -> Result { + self.published_blocks.write().push(block); + Ok(BlockPublishOutcome::ValidBlock) + } +} diff --git a/beacon_node/beacon_chain/test_harness/src/validator/mod.rs b/beacon_node/beacon_chain/test_harness/src/validator/mod.rs index 4eae0bb3a..3a0d53395 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/mod.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator/mod.rs @@ -1,137 +1,6 @@ -use attester::{Attester, Error as AttestationPollError}; -use beacon_chain::BeaconChain; -use block_producer::{BlockProducer, Error as BlockPollError}; -use db::MemoryDB; -use signer::TestSigner; -use slot_clock::TestingSlotClock; -use std::sync::Arc; -use types::{BeaconBlock, ChainSpec, FreeAttestation, Keypair}; -mod beacon_node; +mod direct_beacon_node; mod direct_duties; mod signer; +mod validator; -pub use self::beacon_node::DirectBeaconNode; -pub use self::direct_duties::DirectDuties; -pub use attester::PollOutcome as AttestationPollOutcome; -pub use block_producer::PollOutcome as BlockPollOutcome; - -#[derive(Debug, PartialEq)] -pub enum BlockProduceError { - DidNotProduce(BlockPollOutcome), - PollError(BlockPollError), -} - -#[derive(Debug, PartialEq)] -pub enum AttestationProduceError { - DidNotProduce(AttestationPollOutcome), - PollError(AttestationPollError), -} - -/// A `BlockProducer` and `Attester` which sign using a common keypair. -/// -/// The test validator connects directly to a borrowed `BeaconChain` struct. It is useful for -/// testing that the core proposer and attester logic is functioning. Also for supporting beacon -/// chain tests. -pub struct TestValidator { - pub block_producer: BlockProducer< - TestingSlotClock, - DirectBeaconNode, - DirectDuties, - TestSigner, - >, - pub attester: Attester< - TestingSlotClock, - DirectBeaconNode, - DirectDuties, - TestSigner, - >, - pub spec: Arc, - pub epoch_map: Arc>, - pub keypair: Keypair, - pub beacon_node: Arc>, - pub slot_clock: Arc, - pub signer: Arc, -} - -impl TestValidator { - /// Create a new TestValidator that signs with the given keypair, operates per the given spec and connects to the - /// supplied beacon node. - /// - /// A `BlockProducer` and `Attester` is created.. - pub fn new( - keypair: Keypair, - beacon_chain: Arc>, - spec: Arc, - ) -> Self { - let slot_clock = Arc::new(TestingSlotClock::new(spec.genesis_slot)); - let signer = Arc::new(TestSigner::new(keypair.clone())); - let beacon_node = Arc::new(DirectBeaconNode::new(beacon_chain.clone())); - let epoch_map = Arc::new(DirectDuties::new(keypair.pk.clone(), beacon_chain.clone())); - - let block_producer = BlockProducer::new( - spec.clone(), - keypair.pk.clone(), - epoch_map.clone(), - slot_clock.clone(), - beacon_node.clone(), - signer.clone(), - ); - - let attester = Attester::new( - epoch_map.clone(), - slot_clock.clone(), - beacon_node.clone(), - signer.clone(), - ); - - Self { - block_producer, - attester, - spec, - epoch_map, - keypair, - beacon_node, - slot_clock, - signer, - } - } - - /// Run the `poll` function on the `BlockProducer` and produce a block. - /// - /// An error is returned if the producer refuses to produce. - pub fn produce_block(&mut self) -> Result { - // Using `DirectBeaconNode`, the validator will always return sucessufully if it tries to - // publish a block. - match self.block_producer.poll() { - Ok(BlockPollOutcome::BlockProduced(_)) => {} - Ok(outcome) => return Err(BlockProduceError::DidNotProduce(outcome)), - Err(error) => return Err(BlockProduceError::PollError(error)), - }; - Ok(self - .beacon_node - .last_published_block() - .expect("Unable to obtain produced block.")) - } - - /// Run the `poll` function on the `Attester` and produce a `FreeAttestation`. - /// - /// An error is returned if the attester refuses to attest. - pub fn produce_free_attestation(&mut self) -> Result { - match self.attester.poll() { - Ok(AttestationPollOutcome::AttestationProduced(_)) => {} - Ok(outcome) => return Err(AttestationProduceError::DidNotProduce(outcome)), - Err(error) => return Err(AttestationProduceError::PollError(error)), - }; - Ok(self - .beacon_node - .last_published_free_attestation() - .expect("Unable to obtain produced attestation.")) - } - - /// Set the validators slot clock to the specified slot. - /// - /// The validators slot clock will always read this value until it is set to something else. - pub fn set_slot(&mut self, slot: u64) { - self.slot_clock.set_slot(slot) - } -} +pub use self::validator::TestValidator; diff --git a/beacon_node/beacon_chain/test_harness/src/validator/validator.rs b/beacon_node/beacon_chain/test_harness/src/validator/validator.rs new file mode 100644 index 000000000..d33a0412b --- /dev/null +++ b/beacon_node/beacon_chain/test_harness/src/validator/validator.rs @@ -0,0 +1,133 @@ +use super::direct_beacon_node::DirectBeaconNode; +use super::direct_duties::DirectDuties; +use super::signer::TestSigner; +use attester::PollOutcome as AttestationPollOutcome; +use attester::{Attester, Error as AttestationPollError}; +use beacon_chain::BeaconChain; +use block_producer::PollOutcome as BlockPollOutcome; +use block_producer::{BlockProducer, Error as BlockPollError}; +use db::MemoryDB; +use slot_clock::TestingSlotClock; +use std::sync::Arc; +use types::{BeaconBlock, ChainSpec, FreeAttestation, Keypair}; + +#[derive(Debug, PartialEq)] +pub enum BlockProduceError { + DidNotProduce(BlockPollOutcome), + PollError(BlockPollError), +} + +#[derive(Debug, PartialEq)] +pub enum AttestationProduceError { + DidNotProduce(AttestationPollOutcome), + PollError(AttestationPollError), +} + +/// A `BlockProducer` and `Attester` which sign using a common keypair. +/// +/// The test validator connects directly to a borrowed `BeaconChain` struct. It is useful for +/// testing that the core proposer and attester logic is functioning. Also for supporting beacon +/// chain tests. +pub struct TestValidator { + pub block_producer: BlockProducer< + TestingSlotClock, + DirectBeaconNode, + DirectDuties, + TestSigner, + >, + pub attester: Attester< + TestingSlotClock, + DirectBeaconNode, + DirectDuties, + TestSigner, + >, + pub spec: Arc, + pub epoch_map: Arc>, + pub keypair: Keypair, + pub beacon_node: Arc>, + pub slot_clock: Arc, + pub signer: Arc, +} + +impl TestValidator { + /// Create a new TestValidator that signs with the given keypair, operates per the given spec and connects to the + /// supplied beacon node. + /// + /// A `BlockProducer` and `Attester` is created.. + pub fn new( + keypair: Keypair, + beacon_chain: Arc>, + spec: Arc, + ) -> Self { + let slot_clock = Arc::new(TestingSlotClock::new(spec.genesis_slot)); + let signer = Arc::new(TestSigner::new(keypair.clone())); + let beacon_node = Arc::new(DirectBeaconNode::new(beacon_chain.clone())); + let epoch_map = Arc::new(DirectDuties::new(keypair.pk.clone(), beacon_chain.clone())); + + let block_producer = BlockProducer::new( + spec.clone(), + keypair.pk.clone(), + epoch_map.clone(), + slot_clock.clone(), + beacon_node.clone(), + signer.clone(), + ); + + let attester = Attester::new( + epoch_map.clone(), + slot_clock.clone(), + beacon_node.clone(), + signer.clone(), + ); + + Self { + block_producer, + attester, + spec, + epoch_map, + keypair, + beacon_node, + slot_clock, + signer, + } + } + + /// Run the `poll` function on the `BlockProducer` and produce a block. + /// + /// An error is returned if the producer refuses to produce. + pub fn produce_block(&mut self) -> Result { + // Using `DirectBeaconNode`, the validator will always return sucessufully if it tries to + // publish a block. + match self.block_producer.poll() { + Ok(BlockPollOutcome::BlockProduced(_)) => {} + Ok(outcome) => return Err(BlockProduceError::DidNotProduce(outcome)), + Err(error) => return Err(BlockProduceError::PollError(error)), + }; + Ok(self + .beacon_node + .last_published_block() + .expect("Unable to obtain produced block.")) + } + + /// Run the `poll` function on the `Attester` and produce a `FreeAttestation`. + /// + /// An error is returned if the attester refuses to attest. + pub fn produce_free_attestation(&mut self) -> Result { + match self.attester.poll() { + Ok(AttestationPollOutcome::AttestationProduced(_)) => {} + Ok(outcome) => return Err(AttestationProduceError::DidNotProduce(outcome)), + Err(error) => return Err(AttestationProduceError::PollError(error)), + }; + Ok(self + .beacon_node + .last_published_free_attestation() + .expect("Unable to obtain produced attestation.")) + } + + /// Set the validators slot clock to the specified slot. + /// + /// The validators slot clock will always read this value until it is set to something else. + pub fn set_slot(&mut self, slot: u64) { + self.slot_clock.set_slot(slot) + } +} From ade2460377b60f602d304359290d1fbb64c5a246 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 5 Feb 2019 12:25:42 +1100 Subject: [PATCH 097/107] Move all beacon_state fns into single file. --- eth2/types/src/beacon_state.rs | 1878 +++++++++++++++++ .../beacon_state/attestation_participants.rs | 71 - .../beacon_state/attestation_validation.rs | 130 -- .../src/beacon_state/block_processing.rs | 318 --- eth2/types/src/beacon_state/committees.rs | 148 -- .../src/beacon_state/epoch_processing.rs | 799 ------- eth2/types/src/beacon_state/mod.rs | 281 --- eth2/types/src/beacon_state/shuffling.rs | 45 - .../types/src/beacon_state/slot_processing.rs | 73 - eth2/types/src/beacon_state/winning_root.rs | 106 - 10 files changed, 1878 insertions(+), 1971 deletions(-) create mode 100644 eth2/types/src/beacon_state.rs delete mode 100644 eth2/types/src/beacon_state/attestation_participants.rs delete mode 100644 eth2/types/src/beacon_state/attestation_validation.rs delete mode 100644 eth2/types/src/beacon_state/block_processing.rs delete mode 100644 eth2/types/src/beacon_state/committees.rs delete mode 100644 eth2/types/src/beacon_state/epoch_processing.rs delete mode 100644 eth2/types/src/beacon_state/mod.rs delete mode 100644 eth2/types/src/beacon_state/shuffling.rs delete mode 100644 eth2/types/src/beacon_state/slot_processing.rs delete mode 100644 eth2/types/src/beacon_state/winning_root.rs diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs new file mode 100644 index 000000000..39787ab32 --- /dev/null +++ b/eth2/types/src/beacon_state.rs @@ -0,0 +1,1878 @@ +use crate::test_utils::TestRandom; +use crate::{ + validator::StatusFlags, validator_registry::get_active_validator_indices, AggregatePublicKey, + Attestation, AttestationData, BeaconBlock, Bitfield, ChainSpec, Crosslink, Eth1Data, + Eth1DataVote, Exit, Fork, Hash256, PendingAttestation, PublicKey, Signature, Validator, +}; +use bls::bls_verify_aggregate; +use honey_badger_split::SplitExt; +use integer_sqrt::IntegerSquareRoot; +use log::debug; +use rand::RngCore; +use rayon::prelude::*; +use serde_derive::Serialize; +use ssz::ssz_encode; +use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; +use std::collections::{HashMap, HashSet}; +use std::iter::FromIterator; +use std::ops::Range; +use vec_shuffle::shuffle; + +// TODO: define elsehwere. +const DOMAIN_PROPOSAL: u64 = 2; +const DOMAIN_EXIT: u64 = 3; +const DOMAIN_RANDAO: u64 = 4; +const PHASE_0_CUSTODY_BIT: bool = false; +const DOMAIN_ATTESTATION: u64 = 1; + +pub enum Error { + InsufficientValidators, + BadBlockSignature, + InvalidEpoch(u64, Range), + CommitteesError(CommitteesError), +} + +#[derive(Debug, PartialEq)] +pub enum BlockProcessingError { + DBError(String), + StateAlreadyTransitioned, + PresentSlotIsNone, + UnableToDecodeBlock, + MissingParentState(Hash256), + InvalidParentState(Hash256), + MissingBeaconBlock(Hash256), + InvalidBeaconBlock(Hash256), + MissingParentBlock(Hash256), + NoBlockProducer, + StateSlotMismatch, + BadBlockSignature, + BadRandaoSignature, + MaxProposerSlashingsExceeded, + BadProposerSlashing, + MaxAttestationsExceeded, + InvalidAttestation(AttestationValidationError), + NoBlockRoot, + MaxDepositsExceeded, + MaxExitsExceeded, + BadExit, + BadCustodyReseeds, + BadCustodyChallenges, + BadCustodyResponses, + CommitteesError(CommitteesError), + SlotProcessingError(SlotProcessingError), +} + +#[derive(Debug, PartialEq)] +pub enum EpochError { + UnableToDetermineProducer, + NoBlockRoots, + BaseRewardQuotientIsZero, + CommitteesError(CommitteesError), + AttestationParticipantsError(AttestationParticipantsError), + InclusionError(InclusionError), + WinningRootError(WinningRootError), +} + +#[derive(Debug, PartialEq)] +pub enum WinningRootError { + NoWinningRoot, + AttestationParticipantsError(AttestationParticipantsError), +} + +#[derive(Debug, PartialEq)] +pub enum CommitteesError { + InvalidEpoch(u64, Range), + InsufficientNumberOfValidators, +} + +#[derive(Debug, PartialEq)] +pub enum InclusionError { + NoIncludedAttestations, + AttestationParticipantsError(AttestationParticipantsError), +} + +#[derive(Debug, PartialEq)] +pub enum AttestationParticipantsError { + NoCommitteeForShard, + NoCommittees, + BadBitfieldLength, + CommitteesError(CommitteesError), +} + +#[derive(Debug, PartialEq)] +pub enum SlotProcessingError { + CommitteesError(CommitteesError), + EpochProcessingError(EpochError), +} + +#[derive(Debug, PartialEq)] +pub enum AttestationValidationError { + IncludedTooEarly, + IncludedTooLate, + WrongJustifiedSlot, + WrongJustifiedRoot, + BadLatestCrosslinkRoot, + BadSignature, + ShardBlockRootNotZero, + NoBlockRoot, + AttestationParticipantsError(AttestationParticipantsError), +} + +#[derive(Clone)] +pub struct WinningRoot { + pub shard_block_root: Hash256, + pub attesting_validator_indices: Vec, + pub total_balance: u64, + pub total_attesting_balance: u64, +} + +macro_rules! ensure { + ($condition: expr, $result: expr) => { + if !$condition { + return Err($result); + } + }; +} + +macro_rules! safe_add_assign { + ($a: expr, $b: expr) => { + $a = $a.saturating_add($b); + }; +} +macro_rules! safe_sub_assign { + ($a: expr, $b: expr) => { + $a = $a.saturating_sub($b); + }; +} + +// Custody will not be added to the specs until Phase 1 (Sharding Phase) so dummy class used. +type CustodyChallenge = usize; + +#[derive(Debug, PartialEq, Clone, Default, Serialize)] +pub struct BeaconState { + // Misc + pub slot: u64, + pub genesis_time: u64, + pub fork_data: Fork, + + // Validator registry + pub validator_registry: Vec, + pub validator_balances: Vec, + pub validator_registry_update_slot: u64, + pub validator_registry_exit_count: u64, + pub validator_registry_delta_chain_tip: Hash256, + + // Randomness and committees + pub latest_randao_mixes: Vec, + pub latest_vdf_outputs: Vec, + pub previous_epoch_start_shard: u64, + pub current_epoch_start_shard: u64, + pub previous_epoch_calculation_slot: u64, + pub current_epoch_calculation_slot: u64, + pub previous_epoch_seed: Hash256, + pub current_epoch_seed: Hash256, + + // Custody challenges + pub custody_challenges: Vec, + + // Finality + pub previous_justified_slot: u64, + pub justified_slot: u64, + pub justification_bitfield: u64, + pub finalized_slot: u64, + + // Recent state + pub latest_crosslinks: Vec, + pub latest_block_roots: Vec, + pub latest_penalized_balances: Vec, + pub latest_attestations: Vec, + pub batched_block_roots: Vec, + + // Ethereum 1.0 chain data + pub latest_eth1_data: Eth1Data, + pub eth1_data_votes: Vec, +} + +impl BeaconState { + pub fn canonical_root(&self) -> Hash256 { + Hash256::from(&self.hash_tree_root()[..]) + } + + pub fn current_epoch(&self, spec: &ChainSpec) -> u64 { + self.slot / spec.epoch_length + } + + pub fn previous_epoch(&self, spec: &ChainSpec) -> u64 { + self.current_epoch(spec).saturating_sub(1) + } + + pub fn current_epoch_start_slot(&self, spec: &ChainSpec) -> u64 { + self.current_epoch(spec) * spec.epoch_length + } + + pub fn previous_epoch_start_slot(&self, spec: &ChainSpec) -> u64 { + self.previous_epoch(spec) * spec.epoch_length + } + + /// Returns the number of committees per slot. + /// + /// Note: this is _not_ the committee size. + pub fn get_committee_count_per_slot( + &self, + active_validator_count: usize, + spec: &ChainSpec, + ) -> u64 { + std::cmp::max( + 1, + std::cmp::min( + spec.shard_count / spec.epoch_length, + active_validator_count as u64 / spec.epoch_length / spec.target_committee_size, + ), + ) + } + + /// Returns the start slot and end slot of the current epoch containing `self.slot`. + pub fn get_current_epoch_boundaries(&self, epoch_length: u64) -> Range { + let slot_in_epoch = self.slot % epoch_length; + let start = self.slot - slot_in_epoch; + let end = self.slot + (epoch_length - slot_in_epoch); + start..end + } + + /// Returns the start slot and end slot of the current epoch containing `self.slot`. + pub fn get_previous_epoch_boundaries(&self, spec: &ChainSpec) -> Range { + let current_epoch = self.slot / spec.epoch_length; + let previous_epoch = current_epoch.saturating_sub(1); + let start = previous_epoch * spec.epoch_length; + let end = start + spec.epoch_length; + start..end + } + + fn get_previous_epoch_committee_count_per_slot(&self, spec: &ChainSpec) -> u64 { + let previous_active_validators = get_active_validator_indices( + &self.validator_registry, + self.previous_epoch_calculation_slot, + ); + self.get_committee_count_per_slot(previous_active_validators.len(), spec) as u64 + } + + pub fn get_current_epoch_committee_count_per_slot(&self, spec: &ChainSpec) -> u64 { + let current_active_validators = get_active_validator_indices( + &self.validator_registry, + self.current_epoch_calculation_slot, + ); + self.get_committee_count_per_slot(current_active_validators.len(), spec) + } + + pub fn get_crosslink_committees_at_slot( + &self, + slot: u64, + spec: &ChainSpec, + ) -> Result, u64)>, CommitteesError> { + let epoch = slot / spec.epoch_length; + let current_epoch = self.slot / spec.epoch_length; + let previous_epoch = if current_epoch == spec.genesis_slot { + current_epoch + } else { + current_epoch.saturating_sub(1) + }; + let next_epoch = current_epoch + 1; + + ensure!( + (previous_epoch <= epoch) & (epoch < next_epoch), + CommitteesError::InvalidEpoch(slot, previous_epoch..current_epoch) + ); + + let offset = slot % spec.epoch_length; + + let (committees_per_slot, shuffling, slot_start_shard) = if epoch < current_epoch { + let committees_per_slot = self.get_previous_epoch_committee_count_per_slot(spec); + let shuffling = self.get_shuffling( + self.previous_epoch_seed, + self.previous_epoch_calculation_slot, + spec, + ); + let slot_start_shard = + (self.previous_epoch_start_shard + committees_per_slot * offset) % spec.shard_count; + (committees_per_slot, shuffling, slot_start_shard) + } else { + let committees_per_slot = self.get_current_epoch_committee_count_per_slot(spec); + let shuffling = self.get_shuffling( + self.current_epoch_seed, + self.current_epoch_calculation_slot, + spec, + ); + let slot_start_shard = + (self.current_epoch_start_shard + committees_per_slot * offset) % spec.shard_count; + (committees_per_slot, shuffling, slot_start_shard) + }; + + let mut crosslinks_at_slot = vec![]; + for i in 0..committees_per_slot { + let tuple = ( + shuffling[(committees_per_slot * offset + i) as usize].clone(), + (slot_start_shard + i) % spec.shard_count, + ); + crosslinks_at_slot.push(tuple) + } + Ok(crosslinks_at_slot) + } + + pub fn per_slot_processing( + &mut self, + previous_block_root: Hash256, + spec: &ChainSpec, + ) -> Result<(), SlotProcessingError> { + if (self.slot + 1) % spec.epoch_length == 0 { + self.per_epoch_processing(spec)?; + } + + self.slot += 1; + + let block_proposer = self.get_beacon_proposer_index(self.slot, spec)?; + + self.validator_registry[block_proposer].proposer_slots += 1; + self.latest_randao_mixes[(self.slot % spec.latest_randao_mixes_length) as usize] = + self.latest_randao_mixes[((self.slot - 1) % spec.latest_randao_mixes_length) as usize]; + + // Block roots. + self.latest_block_roots[((self.slot - 1) % spec.latest_block_roots_length) as usize] = + previous_block_root; + + if self.slot % spec.latest_block_roots_length == 0 { + let root = merkle_root(&self.latest_block_roots[..]); + self.batched_block_roots.push(root); + } + Ok(()) + } + + pub fn attestation_slot_and_shard_for_validator( + &self, + validator_index: usize, + spec: &ChainSpec, + ) -> Result, CommitteesError> { + let mut result = None; + for slot in self.get_current_epoch_boundaries(spec.epoch_length) { + for (committee, shard) in self.get_crosslink_committees_at_slot(slot, spec)? { + if let Some(committee_index) = committee.iter().position(|&i| i == validator_index) + { + result = Some((slot, shard, committee_index as u64)); + } + } + } + Ok(result) + } + + pub fn per_block_processing( + &mut self, + block: &BeaconBlock, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError> { + self.per_block_processing_signature_optional(block, true, spec) + } + + pub fn per_block_processing_without_verifying_block_signature( + &mut self, + block: &BeaconBlock, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError> { + self.per_block_processing_signature_optional(block, false, spec) + } + + fn per_block_processing_signature_optional( + &mut self, + block: &BeaconBlock, + verify_block_signature: bool, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError> { + ensure!( + block.slot == self.slot, + BlockProcessingError::StateSlotMismatch + ); + + /* + * Proposer Signature + */ + let block_proposer_index = self + .get_beacon_proposer_index(block.slot, spec) + .map_err(|_| BlockProcessingError::NoBlockProducer)?; + let block_proposer = &self.validator_registry[block_proposer_index]; + + if verify_block_signature { + ensure!( + bls_verify( + &block_proposer.pubkey, + &block.proposal_root(spec)[..], + &block.signature, + get_domain(&self.fork_data, self.slot, DOMAIN_PROPOSAL) + ), + BlockProcessingError::BadBlockSignature + ); + } + + /* + * RANDAO + */ + ensure!( + bls_verify( + &block_proposer.pubkey, + &ssz_encode(&block_proposer.proposer_slots), + &block.randao_reveal, + get_domain(&self.fork_data, self.slot, DOMAIN_RANDAO) + ), + BlockProcessingError::BadRandaoSignature + ); + + // TODO: check this is correct. + let new_mix = { + let mut mix = self.latest_randao_mixes + [(self.slot % spec.latest_randao_mixes_length) as usize] + .to_vec(); + mix.append(&mut ssz_encode(&block.randao_reveal)); + Hash256::from(&hash(&mix)[..]) + }; + + self.latest_randao_mixes[(self.slot % spec.latest_randao_mixes_length) as usize] = new_mix; + + /* + * Eth1 data + */ + + // TODO: Eth1 data processing. + + /* + * Proposer slashings + */ + ensure!( + block.body.proposer_slashings.len() as u64 <= spec.max_proposer_slashings, + BlockProcessingError::MaxProposerSlashingsExceeded + ); + for proposer_slashing in &block.body.proposer_slashings { + let proposer = self + .validator_registry + .get(proposer_slashing.proposer_index as usize) + .ok_or(BlockProcessingError::BadProposerSlashing)?; + ensure!( + proposer_slashing.proposal_data_1.slot == proposer_slashing.proposal_data_2.slot, + BlockProcessingError::BadProposerSlashing + ); + ensure!( + proposer_slashing.proposal_data_1.shard == proposer_slashing.proposal_data_2.shard, + BlockProcessingError::BadProposerSlashing + ); + ensure!( + proposer_slashing.proposal_data_1.block_root + != proposer_slashing.proposal_data_2.block_root, + BlockProcessingError::BadProposerSlashing + ); + ensure!( + proposer.penalized_slot > self.slot, + BlockProcessingError::BadProposerSlashing + ); + ensure!( + bls_verify( + &proposer.pubkey, + &proposer_slashing.proposal_data_1.hash_tree_root(), + &proposer_slashing.proposal_signature_1, + get_domain( + &self.fork_data, + proposer_slashing.proposal_data_1.slot, + DOMAIN_PROPOSAL + ) + ), + BlockProcessingError::BadProposerSlashing + ); + ensure!( + bls_verify( + &proposer.pubkey, + &proposer_slashing.proposal_data_2.hash_tree_root(), + &proposer_slashing.proposal_signature_2, + get_domain( + &self.fork_data, + proposer_slashing.proposal_data_2.slot, + DOMAIN_PROPOSAL + ) + ), + BlockProcessingError::BadProposerSlashing + ); + penalize_validator(&self, proposer_slashing.proposer_index as usize); + } + + /* + * Attestations + */ + ensure!( + block.body.attestations.len() as u64 <= spec.max_attestations, + BlockProcessingError::MaxAttestationsExceeded + ); + + for attestation in &block.body.attestations { + self.validate_attestation(attestation, spec)?; + + let pending_attestation = PendingAttestation { + data: attestation.data.clone(), + aggregation_bitfield: attestation.aggregation_bitfield.clone(), + custody_bitfield: attestation.custody_bitfield.clone(), + slot_included: self.slot, + }; + self.latest_attestations.push(pending_attestation); + } + + debug!( + "{} attestations verified & processed.", + block.body.attestations.len() + ); + + /* + * Deposits + */ + ensure!( + block.body.deposits.len() as u64 <= spec.max_deposits, + BlockProcessingError::MaxDepositsExceeded + ); + + // TODO: process deposits. + + /* + * Exits + */ + + ensure!( + block.body.exits.len() as u64 <= spec.max_exits, + BlockProcessingError::MaxExitsExceeded + ); + + for exit in &block.body.exits { + let validator = self + .validator_registry + .get(exit.validator_index as usize) + .ok_or(BlockProcessingError::BadExit)?; + ensure!( + validator.exit_slot > self.slot + spec.entry_exit_delay, + BlockProcessingError::BadExit + ); + ensure!(self.slot >= exit.slot, BlockProcessingError::BadExit); + let exit_message = { + let exit_struct = Exit { + slot: exit.slot, + validator_index: exit.validator_index, + signature: spec.empty_signature.clone(), + }; + exit_struct.hash_tree_root() + }; + ensure!( + bls_verify( + &validator.pubkey, + &exit_message, + &exit.signature, + get_domain(&self.fork_data, exit.slot, DOMAIN_EXIT) + ), + BlockProcessingError::BadProposerSlashing + ); + initiate_validator_exit(&self, exit.validator_index); + } + + /* + * Custody + */ + ensure!( + block.body.custody_reseeds.is_empty(), + BlockProcessingError::BadCustodyReseeds + ); + ensure!( + block.body.custody_challenges.is_empty(), + BlockProcessingError::BadCustodyChallenges + ); + ensure!( + block.body.custody_responses.is_empty(), + BlockProcessingError::BadCustodyResponses + ); + + debug!("State transition complete."); + + Ok(()) + } + + pub fn get_shuffling(&self, seed: Hash256, slot: u64, spec: &ChainSpec) -> Vec> { + let slot = slot - (slot % spec.epoch_length); + + let active_validator_indices = get_active_validator_indices(&self.validator_registry, slot); + + let committees_per_slot = + self.get_committee_count_per_slot(active_validator_indices.len(), spec); + + // TODO: check that Hash256 matches 'int_to_bytes32'. + let seed = seed ^ Hash256::from(slot); + let shuffled_active_validator_indices = + shuffle(&seed, active_validator_indices).expect("Max validator count exceed!"); + + shuffled_active_validator_indices + .honey_badger_split((committees_per_slot * spec.epoch_length) as usize) + .filter_map(|slice: &[usize]| Some(slice.to_vec())) + .collect() + } + + /// Returns the beacon proposer index for the `slot`. + /// If the state does not contain an index for a beacon proposer at the requested `slot`, then `None` is returned. + pub fn get_beacon_proposer_index( + &self, + slot: u64, + spec: &ChainSpec, + ) -> Result { + let committees = self.get_crosslink_committees_at_slot(slot, spec)?; + committees + .first() + .ok_or(CommitteesError::InsufficientNumberOfValidators) + .and_then(|(first_committee, _)| { + let index = (slot as usize) + .checked_rem(first_committee.len()) + .ok_or(CommitteesError::InsufficientNumberOfValidators)?; + // NOTE: next index will not panic as we have already returned if this is the case + Ok(first_committee[index]) + }) + } + + pub fn per_epoch_processing(&mut self, spec: &ChainSpec) -> Result<(), EpochError> { + debug!( + "Starting per-epoch processing on epoch {}...", + self.current_epoch(spec) + ); + /* + * All Validators + */ + let active_validator_indices = + get_active_validator_indices(&self.validator_registry, self.slot); + let total_balance = self.get_effective_balances(&active_validator_indices[..], spec); + + debug!( + "{} validators with a total balance of {} wei.", + active_validator_indices.len(), + total_balance + ); + + let current_epoch_attestations: Vec<&PendingAttestation> = self + .latest_attestations + .par_iter() + .filter(|a| a.data.slot / spec.epoch_length == self.current_epoch(spec)) + .collect(); + + debug!( + "Current epoch attestations: {}", + current_epoch_attestations.len() + ); + + /* + * Validators attesting during the current epoch. + */ + if self.latest_block_roots.is_empty() { + return Err(EpochError::NoBlockRoots); + } + + let current_epoch_boundary_attestations: Vec<&PendingAttestation> = + current_epoch_attestations + .par_iter() + .filter(|a| { + match self.get_block_root(self.current_epoch_start_slot(spec), spec) { + Some(block_root) => { + (a.data.epoch_boundary_root == *block_root) + && (a.data.justified_slot == self.justified_slot) + } + // Protected by a check that latest_block_roots isn't empty. + // + // TODO: provide detailed reasoning. + None => unreachable!(), + } + }) + .cloned() + .collect(); + + let current_epoch_boundary_attester_indices = self + .get_attestation_participants_union(¤t_epoch_boundary_attestations[..], spec)?; + let current_epoch_boundary_attesting_balance = + self.get_effective_balances(¤t_epoch_boundary_attester_indices[..], spec); + + debug!( + "Current epoch boundary attesters: {}", + current_epoch_boundary_attester_indices.len() + ); + + /* + * Validators attesting during the previous epoch + */ + + /* + * Validators that made an attestation during the previous epoch + */ + let previous_epoch_attestations: Vec<&PendingAttestation> = self + .latest_attestations + .par_iter() + .filter(|a| { + //TODO: ensure these saturating subs are correct. + a.data.slot / spec.epoch_length == self.previous_epoch(spec) + }) + .collect(); + + debug!( + "previous epoch attestations: {}", + previous_epoch_attestations.len() + ); + + let previous_epoch_attester_indices = + self.get_attestation_participants_union(&previous_epoch_attestations[..], spec)?; + + /* + * Validators targetting the previous justified slot + */ + let previous_epoch_justified_attestations: Vec<&PendingAttestation> = { + let mut a: Vec<&PendingAttestation> = current_epoch_attestations + .iter() + .filter(|a| a.data.justified_slot == self.previous_justified_slot) + .cloned() + .collect(); + let mut b: Vec<&PendingAttestation> = previous_epoch_attestations + .iter() + .filter(|a| a.data.justified_slot == self.previous_justified_slot) + .cloned() + .collect(); + a.append(&mut b); + a + }; + + let previous_epoch_justified_attester_indices = self + .get_attestation_participants_union(&previous_epoch_justified_attestations[..], spec)?; + let previous_epoch_justified_attesting_balance = + self.get_effective_balances(&previous_epoch_justified_attester_indices[..], spec); + + /* + * Validators justifying the epoch boundary block at the start of the previous epoch + */ + let previous_epoch_boundary_attestations: Vec<&PendingAttestation> = + previous_epoch_justified_attestations + .iter() + .filter(|a| { + match self.get_block_root(self.previous_epoch_start_slot(spec), spec) { + Some(block_root) => a.data.epoch_boundary_root == *block_root, + // Protected by a check that latest_block_roots isn't empty. + // + // TODO: provide detailed reasoning. + None => unreachable!(), + } + }) + .cloned() + .collect(); + + let previous_epoch_boundary_attester_indices = self + .get_attestation_participants_union(&previous_epoch_boundary_attestations[..], spec)?; + let previous_epoch_boundary_attesting_balance = + self.get_effective_balances(&previous_epoch_boundary_attester_indices[..], spec); + + /* + * Validators attesting to the expected beacon chain head during the previous epoch. + */ + let previous_epoch_head_attestations: Vec<&PendingAttestation> = + previous_epoch_attestations + .iter() + .filter(|a| { + match self.get_block_root(a.data.slot, spec) { + Some(block_root) => a.data.beacon_block_root == *block_root, + // Protected by a check that latest_block_roots isn't empty. + // + // TODO: provide detailed reasoning. + None => unreachable!(), + } + }) + .cloned() + .collect(); + + let previous_epoch_head_attester_indices = + self.get_attestation_participants_union(&previous_epoch_head_attestations[..], spec)?; + let previous_epoch_head_attesting_balance = + self.get_effective_balances(&previous_epoch_head_attester_indices[..], spec); + + debug!( + "previous_epoch_head_attester_balance of {} wei.", + previous_epoch_head_attesting_balance + ); + + /* + * Eth1 Data + */ + if self.slot % spec.eth1_data_voting_period == 0 { + for eth1_data_vote in &self.eth1_data_votes { + if eth1_data_vote.vote_count * 2 > spec.eth1_data_voting_period { + self.latest_eth1_data = eth1_data_vote.eth1_data.clone(); + } + } + self.eth1_data_votes = vec![]; + } + + /* + * Justification + */ + self.previous_justified_slot = self.justified_slot; + let (new_bitfield, _) = self.justification_bitfield.overflowing_mul(2); + self.justification_bitfield = new_bitfield; + + // If >= 2/3 of validators voted for the previous epoch boundary + if (3 * previous_epoch_boundary_attesting_balance) >= (2 * total_balance) { + // TODO: check saturating_sub is correct. + self.justification_bitfield |= 2; + self.justified_slot = self.slot.saturating_sub(2 * spec.epoch_length); + debug!(">= 2/3 voted for previous epoch boundary"); + } + + // If >= 2/3 of validators voted for the current epoch boundary + if (3 * current_epoch_boundary_attesting_balance) >= (2 * total_balance) { + // TODO: check saturating_sub is correct. + self.justification_bitfield |= 1; + self.justified_slot = self.slot.saturating_sub(1 * spec.epoch_length); + debug!(">= 2/3 voted for current epoch boundary"); + } + + if (self.previous_justified_slot == self.slot.saturating_sub(2 * spec.epoch_length)) + && (self.justification_bitfield % 4 == 3) + { + self.finalized_slot = self.previous_justified_slot; + } + if (self.previous_justified_slot == self.slot.saturating_sub(3 * spec.epoch_length)) + && (self.justification_bitfield % 8 == 7) + { + self.finalized_slot = self.previous_justified_slot; + } + if (self.previous_justified_slot == self.slot.saturating_sub(4 * spec.epoch_length)) + && (self.justification_bitfield % 16 == 14) + { + self.finalized_slot = self.previous_justified_slot; + } + if (self.previous_justified_slot == self.slot.saturating_sub(4 * spec.epoch_length)) + && (self.justification_bitfield % 16 == 15) + { + self.finalized_slot = self.previous_justified_slot; + } + + debug!( + "Finalized slot {}, justified slot {}.", + self.finalized_slot, self.justified_slot + ); + + /* + * Crosslinks + */ + + // Cached for later lookups. + let mut winning_root_for_shards: HashMap> = + HashMap::new(); + + // for slot in self.slot.saturating_sub(2 * spec.epoch_length)..self.slot { + for slot in self.get_previous_epoch_boundaries(spec) { + let crosslink_committees_at_slot = self.get_crosslink_committees_at_slot(slot, spec)?; + + for (crosslink_committee, shard) in crosslink_committees_at_slot { + let shard = shard as u64; + + let winning_root = self.winning_root( + shard, + ¤t_epoch_attestations, + &previous_epoch_attestations, + spec, + ); + + if let Ok(winning_root) = &winning_root { + let total_committee_balance = + self.get_effective_balances(&crosslink_committee[..], spec); + + if (3 * winning_root.total_attesting_balance) >= (2 * total_committee_balance) { + self.latest_crosslinks[shard as usize] = Crosslink { + slot: self.slot, + shard_block_root: winning_root.shard_block_root, + } + } + } + winning_root_for_shards.insert(shard, winning_root); + } + } + + debug!( + "Found {} winning shard roots.", + winning_root_for_shards.len() + ); + + /* + * Rewards and Penalities + */ + let base_reward_quotient = total_balance.integer_sqrt(); + if base_reward_quotient == 0 { + return Err(EpochError::BaseRewardQuotientIsZero); + } + + /* + * Justification and finalization + */ + let epochs_since_finality = + self.slot.saturating_sub(self.finalized_slot) / spec.epoch_length; + + // TODO: fix this extra map + let previous_epoch_justified_attester_indices_hashset: HashSet = + HashSet::from_iter(previous_epoch_justified_attester_indices.iter().map(|i| *i)); + let previous_epoch_boundary_attester_indices_hashset: HashSet = + HashSet::from_iter(previous_epoch_boundary_attester_indices.iter().map(|i| *i)); + let previous_epoch_head_attester_indices_hashset: HashSet = + HashSet::from_iter(previous_epoch_head_attester_indices.iter().map(|i| *i)); + let previous_epoch_attester_indices_hashset: HashSet = + HashSet::from_iter(previous_epoch_attester_indices.iter().map(|i| *i)); + + debug!("previous epoch justified attesters: {}, previous epoch boundary attesters: {}, previous epoch head attesters: {}, previous epoch attesters: {}", previous_epoch_justified_attester_indices.len(), previous_epoch_boundary_attester_indices.len(), previous_epoch_head_attester_indices.len(), previous_epoch_attester_indices.len()); + + debug!("{} epochs since finality.", epochs_since_finality); + + if epochs_since_finality <= 4 { + for index in 0..self.validator_balances.len() { + let base_reward = self.base_reward(index, base_reward_quotient, spec); + + if previous_epoch_justified_attester_indices_hashset.contains(&index) { + safe_add_assign!( + self.validator_balances[index], + base_reward * previous_epoch_justified_attesting_balance / total_balance + ); + } else { + safe_sub_assign!(self.validator_balances[index], base_reward); + } + + if previous_epoch_boundary_attester_indices_hashset.contains(&index) { + safe_add_assign!( + self.validator_balances[index], + base_reward * previous_epoch_boundary_attesting_balance / total_balance + ); + } else { + safe_sub_assign!(self.validator_balances[index], base_reward); + } + + if previous_epoch_head_attester_indices_hashset.contains(&index) { + safe_add_assign!( + self.validator_balances[index], + base_reward * previous_epoch_head_attesting_balance / total_balance + ); + } else { + safe_sub_assign!(self.validator_balances[index], base_reward); + } + } + + for index in previous_epoch_attester_indices { + let base_reward = self.base_reward(index, base_reward_quotient, spec); + let inclusion_distance = + self.inclusion_distance(&previous_epoch_attestations, index, spec)?; + + safe_add_assign!( + self.validator_balances[index], + base_reward * spec.min_attestation_inclusion_delay / inclusion_distance + ) + } + } else { + for index in 0..self.validator_balances.len() { + let inactivity_penalty = self.inactivity_penalty( + index, + epochs_since_finality, + base_reward_quotient, + spec, + ); + + if !previous_epoch_justified_attester_indices_hashset.contains(&index) { + safe_sub_assign!(self.validator_balances[index], inactivity_penalty); + } + + if !previous_epoch_boundary_attester_indices_hashset.contains(&index) { + safe_sub_assign!(self.validator_balances[index], inactivity_penalty); + } + + if !previous_epoch_head_attester_indices_hashset.contains(&index) { + safe_sub_assign!(self.validator_balances[index], inactivity_penalty); + } + } + + for index in previous_epoch_attester_indices { + let base_reward = self.base_reward(index, base_reward_quotient, spec); + let inclusion_distance = + self.inclusion_distance(&previous_epoch_attestations, index, spec)?; + + safe_sub_assign!( + self.validator_balances[index], + base_reward + - base_reward * spec.min_attestation_inclusion_delay / inclusion_distance + ); + } + } + + debug!("Processed validator justification and finalization rewards/penalities."); + + /* + * Attestation inclusion + */ + for &index in &previous_epoch_attester_indices_hashset { + let inclusion_slot = + self.inclusion_slot(&previous_epoch_attestations[..], index, spec)?; + let proposer_index = self + .get_beacon_proposer_index(inclusion_slot, spec) + .map_err(|_| EpochError::UnableToDetermineProducer)?; + let base_reward = self.base_reward(proposer_index, base_reward_quotient, spec); + safe_add_assign!( + self.validator_balances[proposer_index], + base_reward / spec.includer_reward_quotient + ); + } + + debug!( + "Previous epoch attesters: {}.", + previous_epoch_attester_indices_hashset.len() + ); + + /* + * Crosslinks + */ + for slot in self.get_previous_epoch_boundaries(spec) { + let crosslink_committees_at_slot = self.get_crosslink_committees_at_slot(slot, spec)?; + + for (_crosslink_committee, shard) in crosslink_committees_at_slot { + let shard = shard as u64; + + if let Some(Ok(winning_root)) = winning_root_for_shards.get(&shard) { + // TODO: remove the map. + let attesting_validator_indices: HashSet = HashSet::from_iter( + winning_root.attesting_validator_indices.iter().map(|i| *i), + ); + + for index in 0..self.validator_balances.len() { + let base_reward = self.base_reward(index, base_reward_quotient, spec); + + if attesting_validator_indices.contains(&index) { + safe_add_assign!( + self.validator_balances[index], + base_reward * winning_root.total_attesting_balance + / winning_root.total_balance + ); + } else { + safe_sub_assign!(self.validator_balances[index], base_reward); + } + } + + for index in &winning_root.attesting_validator_indices { + let base_reward = self.base_reward(*index, base_reward_quotient, spec); + safe_add_assign!( + self.validator_balances[*index], + base_reward * winning_root.total_attesting_balance + / winning_root.total_balance + ); + } + } + } + } + + /* + * Ejections + */ + self.process_ejections(); + + /* + * Validator Registry + */ + self.previous_epoch_calculation_slot = self.current_epoch_calculation_slot; + self.previous_epoch_start_shard = self.current_epoch_start_shard; + self.previous_epoch_seed = self.current_epoch_seed; + + let should_update_validator_registy = if self.finalized_slot + > self.validator_registry_update_slot + { + (0..self.get_current_epoch_committee_count_per_slot(spec)).all(|i| { + let shard = (self.current_epoch_start_shard + i as u64) % spec.shard_count; + self.latest_crosslinks[shard as usize].slot > self.validator_registry_update_slot + }) + } else { + false + }; + + if should_update_validator_registy { + self.update_validator_registry(spec); + + self.current_epoch_calculation_slot = self.slot; + self.current_epoch_start_shard = (self.current_epoch_start_shard + + self.get_current_epoch_committee_count_per_slot(spec) as u64 * spec.epoch_length) + % spec.shard_count; + self.current_epoch_seed = self.get_randao_mix( + self.current_epoch_calculation_slot + .saturating_sub(spec.seed_lookahead), + spec, + ); + } else { + let epochs_since_last_registry_change = + (self.slot - self.validator_registry_update_slot) / spec.epoch_length; + if epochs_since_last_registry_change.is_power_of_two() { + self.current_epoch_calculation_slot = self.slot; + self.current_epoch_seed = self.get_randao_mix( + self.current_epoch_calculation_slot + .saturating_sub(spec.seed_lookahead), + spec, + ); + } + } + + self.process_penalties_and_exits(spec); + + let e = self.slot / spec.epoch_length; + self.latest_penalized_balances[((e + 1) % spec.latest_penalized_exit_length) as usize] = + self.latest_penalized_balances[(e % spec.latest_penalized_exit_length) as usize]; + + self.latest_attestations = self + .latest_attestations + .iter() + .filter(|a| a.data.slot / spec.epoch_length >= self.current_epoch(spec)) + .cloned() + .collect(); + + debug!("Epoch transition complete."); + + Ok(()) + } + + fn process_penalties_and_exits(&mut self, spec: &ChainSpec) { + let active_validator_indices = + get_active_validator_indices(&self.validator_registry, self.slot); + let total_balance = self.get_effective_balances(&active_validator_indices[..], spec); + + for index in 0..self.validator_balances.len() { + let validator = &self.validator_registry[index]; + + if (self.slot / spec.epoch_length) + == (validator.penalized_slot / spec.epoch_length) + + spec.latest_penalized_exit_length / 2 + { + let e = (self.slot / spec.epoch_length) % spec.latest_penalized_exit_length; + let total_at_start = self.latest_penalized_balances + [((e + 1) % spec.latest_penalized_exit_length) as usize]; + let total_at_end = self.latest_penalized_balances[e as usize]; + let total_penalities = total_at_end.saturating_sub(total_at_start); + let penalty = self.get_effective_balance(index, spec) + * std::cmp::min(total_penalities * 3, total_balance) + / total_balance; + safe_sub_assign!(self.validator_balances[index], penalty); + } + } + + let eligible = |index: usize| { + let validator = &self.validator_registry[index]; + + if validator.penalized_slot <= self.slot { + let penalized_withdrawal_time = + spec.latest_penalized_exit_length * spec.epoch_length / 2; + self.slot >= validator.penalized_slot + penalized_withdrawal_time + } else { + self.slot >= validator.exit_slot + spec.min_validator_withdrawal_time + } + }; + + let mut eligable_indices: Vec = (0..self.validator_registry.len()) + .filter(|i| eligible(*i)) + .collect(); + eligable_indices.sort_by_key(|i| self.validator_registry[*i].exit_count); + let mut withdrawn_so_far = 0; + for index in eligable_indices { + self.prepare_validator_for_withdrawal(index); + withdrawn_so_far += 1; + if withdrawn_so_far >= spec.max_withdrawals_per_epoch { + break; + } + } + } + + fn prepare_validator_for_withdrawal(&mut self, index: usize) { + //TODO: we're not ANDing here, we're setting. Potentially wrong. + self.validator_registry[index].status_flags = Some(StatusFlags::Withdrawable); + } + + fn get_randao_mix(&mut self, slot: u64, spec: &ChainSpec) -> Hash256 { + assert!(self.slot < slot + spec.latest_randao_mixes_length); + assert!(slot <= self.slot); + self.latest_randao_mixes[(slot & spec.latest_randao_mixes_length) as usize] + } + + fn update_validator_registry(&mut self, spec: &ChainSpec) { + let active_validator_indices = + get_active_validator_indices(&self.validator_registry, self.slot); + let total_balance = self.get_effective_balances(&active_validator_indices[..], spec); + + let max_balance_churn = std::cmp::max( + spec.max_deposit, + total_balance / (2 * spec.max_balance_churn_quotient), + ); + + let mut balance_churn = 0; + for index in 0..self.validator_registry.len() { + let validator = &self.validator_registry[index]; + + if (validator.activation_slot > self.slot + spec.entry_exit_delay) + && self.validator_balances[index] >= spec.max_deposit + { + balance_churn += self.get_effective_balance(index, spec); + if balance_churn > max_balance_churn { + break; + } + + self.activate_validator(index, false, spec); + } + } + + let mut balance_churn = 0; + for index in 0..self.validator_registry.len() { + let validator = &self.validator_registry[index]; + + if (validator.exit_slot > self.slot + spec.entry_exit_delay) + && validator.status_flags == Some(StatusFlags::InitiatedExit) + { + balance_churn += self.get_effective_balance(index, spec); + if balance_churn > max_balance_churn { + break; + } + + self.exit_validator(index, spec); + } + } + + self.validator_registry_update_slot = self.slot; + } + + fn exit_validator(&mut self, validator_index: usize, spec: &ChainSpec) { + if self.validator_registry[validator_index].exit_slot + <= self.entry_exit_effect_slot(self.slot, spec) + { + return; + } + + self.validator_registry[validator_index].exit_slot = + self.entry_exit_effect_slot(self.slot, spec); + + self.validator_registry_exit_count += 1; + self.validator_registry[validator_index].exit_count = self.validator_registry_exit_count; + } + + fn activate_validator(&mut self, validator_index: usize, is_genesis: bool, spec: &ChainSpec) { + self.validator_registry[validator_index].activation_slot = if is_genesis { + spec.genesis_slot + } else { + self.entry_exit_effect_slot(self.slot, spec) + } + } + + fn entry_exit_effect_slot(&self, slot: u64, spec: &ChainSpec) -> u64 { + (slot - slot % spec.epoch_length) + spec.epoch_length + spec.entry_exit_delay + } + + fn process_ejections(&self) { + //TODO: stubbed out. + } + + fn inactivity_penalty( + &self, + validator_index: usize, + epochs_since_finality: u64, + base_reward_quotient: u64, + spec: &ChainSpec, + ) -> u64 { + let effective_balance = self.get_effective_balance(validator_index, spec); + self.base_reward(validator_index, base_reward_quotient, spec) + + effective_balance * epochs_since_finality / spec.inactivity_penalty_quotient / 2 + } + + fn inclusion_distance( + &self, + attestations: &[&PendingAttestation], + validator_index: usize, + spec: &ChainSpec, + ) -> Result { + let attestation = + self.earliest_included_attestation(attestations, validator_index, spec)?; + Ok(attestation + .slot_included + .saturating_sub(attestation.data.slot)) + } + + fn inclusion_slot( + &self, + attestations: &[&PendingAttestation], + validator_index: usize, + spec: &ChainSpec, + ) -> Result { + let attestation = + self.earliest_included_attestation(attestations, validator_index, spec)?; + Ok(attestation.slot_included) + } + + fn earliest_included_attestation( + &self, + attestations: &[&PendingAttestation], + validator_index: usize, + spec: &ChainSpec, + ) -> Result { + let mut included_attestations = vec![]; + + for (i, a) in attestations.iter().enumerate() { + let participants = + self.get_attestation_participants(&a.data, &a.aggregation_bitfield, spec)?; + if participants + .iter() + .find(|i| **i == validator_index) + .is_some() + { + included_attestations.push(i); + } + } + + let earliest_attestation_index = included_attestations + .iter() + .min_by_key(|i| attestations[**i].slot_included) + .ok_or_else(|| InclusionError::NoIncludedAttestations)?; + Ok(attestations[*earliest_attestation_index].clone()) + } + + fn base_reward( + &self, + validator_index: usize, + base_reward_quotient: u64, + spec: &ChainSpec, + ) -> u64 { + self.get_effective_balance(validator_index, spec) / base_reward_quotient / 5 + } + + pub fn get_effective_balances(&self, validator_indices: &[usize], spec: &ChainSpec) -> u64 { + validator_indices + .iter() + .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)) + } + + pub fn get_effective_balance(&self, validator_index: usize, spec: &ChainSpec) -> u64 { + std::cmp::min(self.validator_balances[validator_index], spec.max_deposit) + } + + pub fn get_block_root(&self, slot: u64, spec: &ChainSpec) -> Option<&Hash256> { + if self.slot <= slot + spec.latest_block_roots_length && slot <= self.slot { + self.latest_block_roots + .get((slot % spec.latest_block_roots_length) as usize) + } else { + None + } + } + + pub(crate) fn winning_root( + &self, + shard: u64, + current_epoch_attestations: &[&PendingAttestation], + previous_epoch_attestations: &[&PendingAttestation], + spec: &ChainSpec, + ) -> Result { + let mut attestations = current_epoch_attestations.to_vec(); + attestations.append(&mut previous_epoch_attestations.to_vec()); + + let mut candidates: HashMap = HashMap::new(); + + let mut highest_seen_balance = 0; + + for a in &attestations { + if a.data.shard != shard { + continue; + } + + let shard_block_root = &a.data.shard_block_root; + + if candidates.contains_key(shard_block_root) { + continue; + } + + // TODO: `cargo fmt` makes this rather ugly; tidy up. + let attesting_validator_indices = attestations.iter().try_fold::<_, _, Result< + _, + AttestationParticipantsError, + >>( + vec![], + |mut acc, a| { + if (a.data.shard == shard) && (a.data.shard_block_root == *shard_block_root) { + acc.append(&mut self.get_attestation_participants( + &a.data, + &a.aggregation_bitfield, + spec, + )?); + } + Ok(acc) + }, + )?; + + let total_balance: u64 = attesting_validator_indices + .iter() + .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)); + + let total_attesting_balance: u64 = attesting_validator_indices + .iter() + .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)); + + if total_attesting_balance > highest_seen_balance { + highest_seen_balance = total_attesting_balance; + } + + let candidate_root = WinningRoot { + shard_block_root: shard_block_root.clone(), + attesting_validator_indices, + total_attesting_balance, + total_balance, + }; + + candidates.insert(*shard_block_root, candidate_root); + } + + Ok(candidates + .iter() + .filter_map(|(_hash, candidate)| { + if candidate.total_attesting_balance == highest_seen_balance { + Some(candidate) + } else { + None + } + }) + .min_by_key(|candidate| candidate.shard_block_root) + .ok_or_else(|| WinningRootError::NoWinningRoot)? + // TODO: avoid clone. + .clone()) + } + + pub fn get_attestation_participants_union( + &self, + attestations: &[&PendingAttestation], + spec: &ChainSpec, + ) -> Result, AttestationParticipantsError> { + let mut all_participants = attestations.iter().try_fold::<_, _, Result< + Vec, + AttestationParticipantsError, + >>(vec![], |mut acc, a| { + acc.append(&mut self.get_attestation_participants( + &a.data, + &a.aggregation_bitfield, + spec, + )?); + Ok(acc) + })?; + all_participants.sort_unstable(); + all_participants.dedup(); + Ok(all_participants) + } + + // TODO: analyse for efficiency improvments. This implementation is naive. + pub fn get_attestation_participants( + &self, + attestation_data: &AttestationData, + aggregation_bitfield: &Bitfield, + spec: &ChainSpec, + ) -> Result, AttestationParticipantsError> { + let crosslink_committees = + self.get_crosslink_committees_at_slot(attestation_data.slot, spec)?; + + let committee_index: usize = crosslink_committees + .iter() + .position(|(_committee, shard)| *shard == attestation_data.shard) + .ok_or_else(|| AttestationParticipantsError::NoCommitteeForShard)?; + let (crosslink_committee, _shard) = &crosslink_committees[committee_index]; + + /* + * TODO: that bitfield length is valid. + * + */ + + let mut participants = vec![]; + for (i, validator_index) in crosslink_committee.iter().enumerate() { + if aggregation_bitfield.get(i).unwrap() { + participants.push(*validator_index); + } + } + Ok(participants) + } + + pub fn validate_attestation( + &self, + attestation: &Attestation, + spec: &ChainSpec, + ) -> Result<(), AttestationValidationError> { + self.validate_attestation_signature_optional(attestation, spec, true) + } + + pub fn validate_attestation_without_signature( + &self, + attestation: &Attestation, + spec: &ChainSpec, + ) -> Result<(), AttestationValidationError> { + self.validate_attestation_signature_optional(attestation, spec, false) + } + + fn validate_attestation_signature_optional( + &self, + attestation: &Attestation, + spec: &ChainSpec, + verify_signature: bool, + ) -> Result<(), AttestationValidationError> { + ensure!( + attestation.data.slot + spec.min_attestation_inclusion_delay <= self.slot, + AttestationValidationError::IncludedTooEarly + ); + ensure!( + attestation.data.slot + spec.epoch_length >= self.slot, + AttestationValidationError::IncludedTooLate + ); + if attestation.data.slot >= self.current_epoch_start_slot(spec) { + ensure!( + attestation.data.justified_slot == self.justified_slot, + AttestationValidationError::WrongJustifiedSlot + ); + } else { + ensure!( + attestation.data.justified_slot == self.previous_justified_slot, + AttestationValidationError::WrongJustifiedSlot + ); + } + ensure!( + attestation.data.justified_block_root + == *self + .get_block_root(attestation.data.justified_slot, &spec) + .ok_or(AttestationValidationError::NoBlockRoot)?, + AttestationValidationError::WrongJustifiedRoot + ); + ensure!( + (attestation.data.latest_crosslink_root + == self.latest_crosslinks[attestation.data.shard as usize].shard_block_root) + || (attestation.data.shard_block_root + == self.latest_crosslinks[attestation.data.shard as usize].shard_block_root), + AttestationValidationError::BadLatestCrosslinkRoot + ); + if verify_signature { + let participants = self.get_attestation_participants( + &attestation.data, + &attestation.aggregation_bitfield, + spec, + )?; + let mut group_public_key = AggregatePublicKey::new(); + for participant in participants { + group_public_key.add( + self.validator_registry[participant as usize] + .pubkey + .as_raw(), + ) + } + ensure!( + bls_verify_aggregate( + &group_public_key, + &attestation.signable_message(PHASE_0_CUSTODY_BIT), + &attestation.aggregate_signature, + get_domain(&self.fork_data, attestation.data.slot, DOMAIN_ATTESTATION) + ), + AttestationValidationError::BadSignature + ); + } + ensure!( + attestation.data.shard_block_root == spec.zero_hash, + AttestationValidationError::ShardBlockRootNotZero + ); + Ok(()) + } +} + +fn merkle_root(_input: &[Hash256]) -> Hash256 { + Hash256::zero() +} + +fn initiate_validator_exit(_state: &BeaconState, _index: u32) { + // TODO: stubbed out. +} + +fn penalize_validator(_state: &BeaconState, _proposer_index: usize) { + // TODO: stubbed out. +} + +fn get_domain(_fork: &Fork, _slot: u64, _domain_type: u64) -> u64 { + // TODO: stubbed out. + 0 +} + +fn bls_verify(pubkey: &PublicKey, message: &[u8], signature: &Signature, _domain: u64) -> bool { + // TODO: add domain + signature.verify(message, pubkey) +} + +impl From for AttestationValidationError { + fn from(e: AttestationParticipantsError) -> AttestationValidationError { + AttestationValidationError::AttestationParticipantsError(e) + } +} + +impl From for WinningRootError { + fn from(e: AttestationParticipantsError) -> WinningRootError { + WinningRootError::AttestationParticipantsError(e) + } +} + +impl From for AttestationParticipantsError { + fn from(e: CommitteesError) -> AttestationParticipantsError { + AttestationParticipantsError::CommitteesError(e) + } +} + +impl From for BlockProcessingError { + fn from(e: AttestationValidationError) -> BlockProcessingError { + BlockProcessingError::InvalidAttestation(e) + } +} + +impl From for BlockProcessingError { + fn from(e: CommitteesError) -> BlockProcessingError { + BlockProcessingError::CommitteesError(e) + } +} + +impl From for BlockProcessingError { + fn from(e: SlotProcessingError) -> BlockProcessingError { + BlockProcessingError::SlotProcessingError(e) + } +} + +impl From for SlotProcessingError { + fn from(e: CommitteesError) -> SlotProcessingError { + SlotProcessingError::CommitteesError(e) + } +} + +impl From for SlotProcessingError { + fn from(e: EpochError) -> SlotProcessingError { + SlotProcessingError::EpochProcessingError(e) + } +} + +impl From for InclusionError { + fn from(e: AttestationParticipantsError) -> InclusionError { + InclusionError::AttestationParticipantsError(e) + } +} + +impl From for EpochError { + fn from(e: InclusionError) -> EpochError { + EpochError::InclusionError(e) + } +} + +impl From for EpochError { + fn from(e: CommitteesError) -> EpochError { + EpochError::CommitteesError(e) + } +} + +impl From for EpochError { + fn from(e: AttestationParticipantsError) -> EpochError { + EpochError::AttestationParticipantsError(e) + } +} + +impl From for Error { + fn from(e: CommitteesError) -> Error { + Error::CommitteesError(e) + } +} + +impl Encodable for BeaconState { + fn ssz_append(&self, s: &mut SszStream) { + s.append(&self.slot); + s.append(&self.genesis_time); + s.append(&self.fork_data); + s.append(&self.validator_registry); + s.append(&self.validator_balances); + s.append(&self.validator_registry_update_slot); + s.append(&self.validator_registry_exit_count); + s.append(&self.validator_registry_delta_chain_tip); + s.append(&self.latest_randao_mixes); + s.append(&self.latest_vdf_outputs); + s.append(&self.previous_epoch_start_shard); + s.append(&self.current_epoch_start_shard); + s.append(&self.previous_epoch_calculation_slot); + s.append(&self.current_epoch_calculation_slot); + s.append(&self.previous_epoch_seed); + s.append(&self.current_epoch_seed); + s.append(&self.custody_challenges); + s.append(&self.previous_justified_slot); + s.append(&self.justified_slot); + s.append(&self.justification_bitfield); + s.append(&self.finalized_slot); + s.append(&self.latest_crosslinks); + s.append(&self.latest_block_roots); + s.append(&self.latest_penalized_balances); + s.append(&self.latest_attestations); + s.append(&self.batched_block_roots); + s.append(&self.latest_eth1_data); + s.append(&self.eth1_data_votes); + } +} + +impl Decodable for BeaconState { + fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { + let (slot, i) = <_>::ssz_decode(bytes, i)?; + let (genesis_time, i) = <_>::ssz_decode(bytes, i)?; + let (fork_data, i) = <_>::ssz_decode(bytes, i)?; + let (validator_registry, i) = <_>::ssz_decode(bytes, i)?; + let (validator_balances, i) = <_>::ssz_decode(bytes, i)?; + let (validator_registry_update_slot, i) = <_>::ssz_decode(bytes, i)?; + let (validator_registry_exit_count, i) = <_>::ssz_decode(bytes, i)?; + let (validator_registry_delta_chain_tip, i) = <_>::ssz_decode(bytes, i)?; + let (latest_randao_mixes, i) = <_>::ssz_decode(bytes, i)?; + let (latest_vdf_outputs, i) = <_>::ssz_decode(bytes, i)?; + let (previous_epoch_start_shard, i) = <_>::ssz_decode(bytes, i)?; + let (current_epoch_start_shard, i) = <_>::ssz_decode(bytes, i)?; + let (previous_epoch_calculation_slot, i) = <_>::ssz_decode(bytes, i)?; + let (current_epoch_calculation_slot, i) = <_>::ssz_decode(bytes, i)?; + let (previous_epoch_seed, i) = <_>::ssz_decode(bytes, i)?; + let (current_epoch_seed, i) = <_>::ssz_decode(bytes, i)?; + let (custody_challenges, i) = <_>::ssz_decode(bytes, i)?; + let (previous_justified_slot, i) = <_>::ssz_decode(bytes, i)?; + let (justified_slot, i) = <_>::ssz_decode(bytes, i)?; + let (justification_bitfield, i) = <_>::ssz_decode(bytes, i)?; + let (finalized_slot, i) = <_>::ssz_decode(bytes, i)?; + let (latest_crosslinks, i) = <_>::ssz_decode(bytes, i)?; + let (latest_block_roots, i) = <_>::ssz_decode(bytes, i)?; + let (latest_penalized_balances, i) = <_>::ssz_decode(bytes, i)?; + let (latest_attestations, i) = <_>::ssz_decode(bytes, i)?; + let (batched_block_roots, i) = <_>::ssz_decode(bytes, i)?; + let (latest_eth1_data, i) = <_>::ssz_decode(bytes, i)?; + let (eth1_data_votes, i) = <_>::ssz_decode(bytes, i)?; + + Ok(( + Self { + slot, + genesis_time, + fork_data, + validator_registry, + validator_balances, + validator_registry_update_slot, + validator_registry_exit_count, + validator_registry_delta_chain_tip, + latest_randao_mixes, + latest_vdf_outputs, + previous_epoch_start_shard, + current_epoch_start_shard, + previous_epoch_calculation_slot, + current_epoch_calculation_slot, + previous_epoch_seed, + current_epoch_seed, + custody_challenges, + previous_justified_slot, + justified_slot, + justification_bitfield, + finalized_slot, + latest_crosslinks, + latest_block_roots, + latest_penalized_balances, + latest_attestations, + batched_block_roots, + latest_eth1_data, + eth1_data_votes, + }, + i, + )) + } +} + +impl TreeHash for BeaconState { + fn hash_tree_root(&self) -> Vec { + let mut result: Vec = vec![]; + result.append(&mut self.slot.hash_tree_root()); + result.append(&mut self.genesis_time.hash_tree_root()); + result.append(&mut self.fork_data.hash_tree_root()); + result.append(&mut self.validator_registry.hash_tree_root()); + result.append(&mut self.validator_balances.hash_tree_root()); + result.append(&mut self.validator_registry_update_slot.hash_tree_root()); + result.append(&mut self.validator_registry_exit_count.hash_tree_root()); + result.append(&mut self.validator_registry_delta_chain_tip.hash_tree_root()); + result.append(&mut self.latest_randao_mixes.hash_tree_root()); + result.append(&mut self.latest_vdf_outputs.hash_tree_root()); + result.append(&mut self.previous_epoch_start_shard.hash_tree_root()); + result.append(&mut self.current_epoch_start_shard.hash_tree_root()); + result.append(&mut self.previous_epoch_calculation_slot.hash_tree_root()); + result.append(&mut self.current_epoch_calculation_slot.hash_tree_root()); + result.append(&mut self.previous_epoch_seed.hash_tree_root()); + result.append(&mut self.current_epoch_seed.hash_tree_root()); + result.append(&mut self.custody_challenges.hash_tree_root()); + result.append(&mut self.previous_justified_slot.hash_tree_root()); + result.append(&mut self.justified_slot.hash_tree_root()); + result.append(&mut self.justification_bitfield.hash_tree_root()); + result.append(&mut self.finalized_slot.hash_tree_root()); + result.append(&mut self.latest_crosslinks.hash_tree_root()); + result.append(&mut self.latest_block_roots.hash_tree_root()); + result.append(&mut self.latest_penalized_balances.hash_tree_root()); + result.append(&mut self.latest_attestations.hash_tree_root()); + result.append(&mut self.batched_block_roots.hash_tree_root()); + result.append(&mut self.latest_eth1_data.hash_tree_root()); + result.append(&mut self.eth1_data_votes.hash_tree_root()); + hash(&result) + } +} + +impl TestRandom for BeaconState { + fn random_for_test(rng: &mut T) -> Self { + Self { + slot: <_>::random_for_test(rng), + genesis_time: <_>::random_for_test(rng), + fork_data: <_>::random_for_test(rng), + validator_registry: <_>::random_for_test(rng), + validator_balances: <_>::random_for_test(rng), + validator_registry_update_slot: <_>::random_for_test(rng), + validator_registry_exit_count: <_>::random_for_test(rng), + validator_registry_delta_chain_tip: <_>::random_for_test(rng), + latest_randao_mixes: <_>::random_for_test(rng), + latest_vdf_outputs: <_>::random_for_test(rng), + previous_epoch_start_shard: <_>::random_for_test(rng), + current_epoch_start_shard: <_>::random_for_test(rng), + previous_epoch_calculation_slot: <_>::random_for_test(rng), + current_epoch_calculation_slot: <_>::random_for_test(rng), + previous_epoch_seed: <_>::random_for_test(rng), + current_epoch_seed: <_>::random_for_test(rng), + custody_challenges: <_>::random_for_test(rng), + previous_justified_slot: <_>::random_for_test(rng), + justified_slot: <_>::random_for_test(rng), + justification_bitfield: <_>::random_for_test(rng), + finalized_slot: <_>::random_for_test(rng), + latest_crosslinks: <_>::random_for_test(rng), + latest_block_roots: <_>::random_for_test(rng), + latest_penalized_balances: <_>::random_for_test(rng), + latest_attestations: <_>::random_for_test(rng), + batched_block_roots: <_>::random_for_test(rng), + latest_eth1_data: <_>::random_for_test(rng), + eth1_data_votes: <_>::random_for_test(rng), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + use ssz::ssz_encode; + + #[test] + pub fn test_ssz_round_trip() { + let mut rng = XorShiftRng::from_seed([42; 16]); + let original = BeaconState::random_for_test(&mut rng); + + let bytes = ssz_encode(&original); + let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); + + assert_eq!(original, decoded); + } + + #[test] + pub fn test_hash_tree_root() { + let mut rng = XorShiftRng::from_seed([42; 16]); + let original = BeaconState::random_for_test(&mut rng); + + let result = original.hash_tree_root(); + + assert_eq!(result.len(), 32); + // TODO: Add further tests + // https://github.com/sigp/lighthouse/issues/170 + } +} diff --git a/eth2/types/src/beacon_state/attestation_participants.rs b/eth2/types/src/beacon_state/attestation_participants.rs deleted file mode 100644 index 28d3ed4a9..000000000 --- a/eth2/types/src/beacon_state/attestation_participants.rs +++ /dev/null @@ -1,71 +0,0 @@ -use crate::{ - beacon_state::CommitteesError, AttestationData, BeaconState, Bitfield, ChainSpec, - PendingAttestation, -}; -use log::debug; - -#[derive(Debug, PartialEq)] -pub enum Error { - NoCommitteeForShard, - NoCommittees, - BadBitfieldLength, - CommitteesError(CommitteesError), -} - -impl BeaconState { - pub fn get_attestation_participants_union( - &self, - attestations: &[&PendingAttestation], - spec: &ChainSpec, - ) -> Result, Error> { - let mut all_participants = attestations - .iter() - .try_fold::<_, _, Result, Error>>(vec![], |mut acc, a| { - acc.append(&mut self.get_attestation_participants( - &a.data, - &a.aggregation_bitfield, - spec, - )?); - Ok(acc) - })?; - all_participants.sort_unstable(); - all_participants.dedup(); - Ok(all_participants) - } - - // TODO: analyse for efficiency improvments. This implementation is naive. - pub fn get_attestation_participants( - &self, - attestation_data: &AttestationData, - aggregation_bitfield: &Bitfield, - spec: &ChainSpec, - ) -> Result, Error> { - let crosslink_committees = - self.get_crosslink_committees_at_slot(attestation_data.slot, spec)?; - - let committee_index: usize = crosslink_committees - .iter() - .position(|(_committee, shard)| *shard == attestation_data.shard) - .ok_or_else(|| Error::NoCommitteeForShard)?; - let (crosslink_committee, _shard) = &crosslink_committees[committee_index]; - - /* - * TODO: that bitfield length is valid. - * - */ - - let mut participants = vec![]; - for (i, validator_index) in crosslink_committee.iter().enumerate() { - if aggregation_bitfield.get(i).unwrap() { - participants.push(*validator_index); - } - } - Ok(participants) - } -} - -impl From for Error { - fn from(e: CommitteesError) -> Error { - Error::CommitteesError(e) - } -} diff --git a/eth2/types/src/beacon_state/attestation_validation.rs b/eth2/types/src/beacon_state/attestation_validation.rs deleted file mode 100644 index 6e6ade34f..000000000 --- a/eth2/types/src/beacon_state/attestation_validation.rs +++ /dev/null @@ -1,130 +0,0 @@ -use crate::{ - beacon_state::AttestationParticipantsError, AggregatePublicKey, Attestation, BeaconState, - ChainSpec, Fork, -}; -use bls::bls_verify_aggregate; - -#[derive(Debug, PartialEq)] -pub enum Error { - IncludedTooEarly, - IncludedTooLate, - WrongJustifiedSlot, - WrongJustifiedRoot, - BadLatestCrosslinkRoot, - BadSignature, - ShardBlockRootNotZero, - NoBlockRoot, - AttestationParticipantsError(AttestationParticipantsError), -} - -macro_rules! ensure { - ($condition: expr, $result: expr) => { - if !$condition { - return Err($result); - } - }; -} - -const PHASE_0_CUSTODY_BIT: bool = false; - -// TODO: define elsehwere. -const DOMAIN_ATTESTATION: u64 = 1; - -impl BeaconState { - pub fn validate_attestation( - &self, - attestation: &Attestation, - spec: &ChainSpec, - ) -> Result<(), Error> { - self.validate_attestation_signature_optional(attestation, spec, true) - } - - pub fn validate_attestation_without_signature( - &self, - attestation: &Attestation, - spec: &ChainSpec, - ) -> Result<(), Error> { - self.validate_attestation_signature_optional(attestation, spec, false) - } - - fn validate_attestation_signature_optional( - &self, - attestation: &Attestation, - spec: &ChainSpec, - verify_signature: bool, - ) -> Result<(), Error> { - ensure!( - attestation.data.slot + spec.min_attestation_inclusion_delay <= self.slot, - Error::IncludedTooEarly - ); - ensure!( - attestation.data.slot + spec.epoch_length >= self.slot, - Error::IncludedTooLate - ); - if attestation.data.slot >= self.current_epoch_start_slot(spec) { - ensure!( - attestation.data.justified_slot == self.justified_slot, - Error::WrongJustifiedSlot - ); - } else { - ensure!( - attestation.data.justified_slot == self.previous_justified_slot, - Error::WrongJustifiedSlot - ); - } - ensure!( - attestation.data.justified_block_root - == *self - .get_block_root(attestation.data.justified_slot, &spec) - .ok_or(Error::NoBlockRoot)?, - Error::WrongJustifiedRoot - ); - ensure!( - (attestation.data.latest_crosslink_root - == self.latest_crosslinks[attestation.data.shard as usize].shard_block_root) - || (attestation.data.shard_block_root - == self.latest_crosslinks[attestation.data.shard as usize].shard_block_root), - Error::BadLatestCrosslinkRoot - ); - if verify_signature { - let participants = self.get_attestation_participants( - &attestation.data, - &attestation.aggregation_bitfield, - spec, - )?; - let mut group_public_key = AggregatePublicKey::new(); - for participant in participants { - group_public_key.add( - self.validator_registry[participant as usize] - .pubkey - .as_raw(), - ) - } - ensure!( - bls_verify_aggregate( - &group_public_key, - &attestation.signable_message(PHASE_0_CUSTODY_BIT), - &attestation.aggregate_signature, - get_domain(&self.fork_data, attestation.data.slot, DOMAIN_ATTESTATION) - ), - Error::BadSignature - ); - } - ensure!( - attestation.data.shard_block_root == spec.zero_hash, - Error::ShardBlockRootNotZero - ); - Ok(()) - } -} - -pub fn get_domain(_fork: &Fork, _slot: u64, _domain_type: u64) -> u64 { - // TODO: stubbed out. - 0 -} - -impl From for Error { - fn from(e: AttestationParticipantsError) -> Error { - Error::AttestationParticipantsError(e) - } -} diff --git a/eth2/types/src/beacon_state/block_processing.rs b/eth2/types/src/beacon_state/block_processing.rs deleted file mode 100644 index df232d6d1..000000000 --- a/eth2/types/src/beacon_state/block_processing.rs +++ /dev/null @@ -1,318 +0,0 @@ -use crate::{ - beacon_state::{AttestationValidationError, CommitteesError, SlotProcessingError}, - readers::BeaconBlockReader, - BeaconBlock, BeaconState, ChainSpec, Exit, Fork, Hash256, PendingAttestation, -}; -use bls::{PublicKey, Signature}; -use hashing::hash; -use log::debug; -use ssz::{ssz_encode, TreeHash}; - -macro_rules! ensure { - ($condition: expr, $result: expr) => { - if !$condition { - return Err($result); - } - }; -} - -// TODO: define elsehwere. -const DOMAIN_PROPOSAL: u64 = 2; -const DOMAIN_EXIT: u64 = 3; -const DOMAIN_RANDAO: u64 = 4; - -#[derive(Debug, PartialEq)] -pub enum Error { - DBError(String), - StateAlreadyTransitioned, - NotImplemented, - PresentSlotIsNone, - UnableToDecodeBlock, - MissingParentState(Hash256), - InvalidParentState(Hash256), - MissingBeaconBlock(Hash256), - InvalidBeaconBlock(Hash256), - MissingParentBlock(Hash256), - NoBlockProducer, - StateSlotMismatch, - BadBlockSignature, - BadRandaoSignature, - MaxProposerSlashingsExceeded, - BadProposerSlashing, - MaxAttestationsExceeded, - InvalidAttestation(AttestationValidationError), - NoBlockRoot, - MaxDepositsExceeded, - MaxExitsExceeded, - BadExit, - BadCustodyReseeds, - BadCustodyChallenges, - BadCustodyResponses, - CommitteesError(CommitteesError), - SlotProcessingError(SlotProcessingError), -} - -impl BeaconState { - pub fn per_block_processing( - &mut self, - block: &BeaconBlock, - spec: &ChainSpec, - ) -> Result<(), Error> { - self.per_block_processing_signature_optional(block, true, spec) - } - - pub fn per_block_processing_without_verifying_block_signature( - &mut self, - block: &BeaconBlock, - spec: &ChainSpec, - ) -> Result<(), Error> { - self.per_block_processing_signature_optional(block, false, spec) - } - - fn per_block_processing_signature_optional( - &mut self, - block: &BeaconBlock, - verify_block_signature: bool, - spec: &ChainSpec, - ) -> Result<(), Error> { - ensure!(block.slot() == self.slot, Error::StateSlotMismatch); - - /* - * Proposer Signature - */ - let block_proposer_index = self - .get_beacon_proposer_index(block.slot, spec) - .map_err(|_| Error::NoBlockProducer)?; - let block_proposer = &self.validator_registry[block_proposer_index]; - - if verify_block_signature { - ensure!( - bls_verify( - &block_proposer.pubkey, - &block.proposal_root(spec)[..], - &block.signature, - get_domain(&self.fork_data, self.slot, DOMAIN_PROPOSAL) - ), - Error::BadBlockSignature - ); - } - - /* - * RANDAO - */ - ensure!( - bls_verify( - &block_proposer.pubkey, - &ssz_encode(&block_proposer.proposer_slots), - &block.randao_reveal, - get_domain(&self.fork_data, self.slot, DOMAIN_RANDAO) - ), - Error::BadRandaoSignature - ); - - // TODO: check this is correct. - let new_mix = { - let mut mix = self.latest_randao_mixes - [(self.slot % spec.latest_randao_mixes_length) as usize] - .to_vec(); - mix.append(&mut ssz_encode(&block.randao_reveal)); - Hash256::from(&hash(&mix)[..]) - }; - - self.latest_randao_mixes[(self.slot % spec.latest_randao_mixes_length) as usize] = new_mix; - - /* - * Eth1 data - */ - - // TODO: Eth1 data processing. - - /* - * Proposer slashings - */ - ensure!( - block.body.proposer_slashings.len() as u64 <= spec.max_proposer_slashings, - Error::MaxProposerSlashingsExceeded - ); - for proposer_slashing in &block.body.proposer_slashings { - let proposer = self - .validator_registry - .get(proposer_slashing.proposer_index as usize) - .ok_or(Error::BadProposerSlashing)?; - ensure!( - proposer_slashing.proposal_data_1.slot == proposer_slashing.proposal_data_2.slot, - Error::BadProposerSlashing - ); - ensure!( - proposer_slashing.proposal_data_1.shard == proposer_slashing.proposal_data_2.shard, - Error::BadProposerSlashing - ); - ensure!( - proposer_slashing.proposal_data_1.block_root - != proposer_slashing.proposal_data_2.block_root, - Error::BadProposerSlashing - ); - ensure!( - proposer.penalized_slot > self.slot, - Error::BadProposerSlashing - ); - ensure!( - bls_verify( - &proposer.pubkey, - &proposer_slashing.proposal_data_1.hash_tree_root(), - &proposer_slashing.proposal_signature_1, - get_domain( - &self.fork_data, - proposer_slashing.proposal_data_1.slot, - DOMAIN_PROPOSAL - ) - ), - Error::BadProposerSlashing - ); - ensure!( - bls_verify( - &proposer.pubkey, - &proposer_slashing.proposal_data_2.hash_tree_root(), - &proposer_slashing.proposal_signature_2, - get_domain( - &self.fork_data, - proposer_slashing.proposal_data_2.slot, - DOMAIN_PROPOSAL - ) - ), - Error::BadProposerSlashing - ); - penalize_validator(&self, proposer_slashing.proposer_index as usize); - } - - /* - * Attestations - */ - ensure!( - block.body.attestations.len() as u64 <= spec.max_attestations, - Error::MaxAttestationsExceeded - ); - - for attestation in &block.body.attestations { - self.validate_attestation(attestation, spec)?; - - let pending_attestation = PendingAttestation { - data: attestation.data.clone(), - aggregation_bitfield: attestation.aggregation_bitfield.clone(), - custody_bitfield: attestation.custody_bitfield.clone(), - slot_included: self.slot, - }; - self.latest_attestations.push(pending_attestation); - } - - debug!( - "{} attestations verified & processed.", - block.body.attestations.len() - ); - - /* - * Deposits - */ - ensure!( - block.body.deposits.len() as u64 <= spec.max_deposits, - Error::MaxDepositsExceeded - ); - - // TODO: process deposits. - - /* - * Exits - */ - - ensure!( - block.body.exits.len() as u64 <= spec.max_exits, - Error::MaxExitsExceeded - ); - - for exit in &block.body.exits { - let validator = self - .validator_registry - .get(exit.validator_index as usize) - .ok_or(Error::BadExit)?; - ensure!( - validator.exit_slot > self.slot + spec.entry_exit_delay, - Error::BadExit - ); - ensure!(self.slot >= exit.slot, Error::BadExit); - let exit_message = { - let exit_struct = Exit { - slot: exit.slot, - validator_index: exit.validator_index, - signature: spec.empty_signature.clone(), - }; - exit_struct.hash_tree_root() - }; - ensure!( - bls_verify( - &validator.pubkey, - &exit_message, - &exit.signature, - get_domain(&self.fork_data, exit.slot, DOMAIN_EXIT) - ), - Error::BadProposerSlashing - ); - initiate_validator_exit(&self, exit.validator_index); - } - - /* - * Custody - */ - ensure!( - block.body.custody_reseeds.is_empty(), - Error::BadCustodyReseeds - ); - ensure!( - block.body.custody_challenges.is_empty(), - Error::BadCustodyChallenges - ); - ensure!( - block.body.custody_responses.is_empty(), - Error::BadCustodyResponses - ); - - debug!("State transition complete."); - - Ok(()) - } -} - -fn initiate_validator_exit(_state: &BeaconState, _index: u32) { - // TODO: stubbed out. -} - -fn penalize_validator(_state: &BeaconState, _proposer_index: usize) { - // TODO: stubbed out. -} - -fn get_domain(_fork: &Fork, _slot: u64, _domain_type: u64) -> u64 { - // TODO: stubbed out. - 0 -} - -fn bls_verify(pubkey: &PublicKey, message: &[u8], signature: &Signature, _domain: u64) -> bool { - // TODO: add domain - signature.verify(message, pubkey) -} - -impl From for Error { - fn from(e: AttestationValidationError) -> Error { - Error::InvalidAttestation(e) - } -} - -impl From for Error { - fn from(e: CommitteesError) -> Error { - Error::CommitteesError(e) - } -} - -impl From for Error { - fn from(e: SlotProcessingError) -> Error { - Error::SlotProcessingError(e) - } -} diff --git a/eth2/types/src/beacon_state/committees.rs b/eth2/types/src/beacon_state/committees.rs deleted file mode 100644 index 1e1689e92..000000000 --- a/eth2/types/src/beacon_state/committees.rs +++ /dev/null @@ -1,148 +0,0 @@ -use crate::{validator_registry::get_active_validator_indices, BeaconState, ChainSpec}; -use log::debug; -use std::ops::Range; - -#[derive(Debug, PartialEq)] -pub enum Error { - InvalidEpoch(u64, Range), - InsufficientNumberOfValidators, -} - -macro_rules! ensure { - ($condition: expr, $result: expr) => { - if !$condition { - return Err($result); - } - }; -} - -type Result = std::result::Result; - -impl BeaconState { - pub fn current_epoch(&self, spec: &ChainSpec) -> u64 { - self.slot / spec.epoch_length - } - - pub fn previous_epoch(&self, spec: &ChainSpec) -> u64 { - self.current_epoch(spec).saturating_sub(1) - } - - pub fn current_epoch_start_slot(&self, spec: &ChainSpec) -> u64 { - self.current_epoch(spec) * spec.epoch_length - } - - pub fn previous_epoch_start_slot(&self, spec: &ChainSpec) -> u64 { - self.previous_epoch(spec) * spec.epoch_length - } - - /// Returns the number of committees per slot. - /// - /// Note: this is _not_ the committee size. - pub fn get_committee_count_per_slot( - &self, - active_validator_count: usize, - spec: &ChainSpec, - ) -> u64 { - std::cmp::max( - 1, - std::cmp::min( - spec.shard_count / spec.epoch_length, - active_validator_count as u64 / spec.epoch_length / spec.target_committee_size, - ), - ) - } - - /// Returns the start slot and end slot of the current epoch containing `self.slot`. - pub fn get_current_epoch_boundaries(&self, epoch_length: u64) -> Range { - let slot_in_epoch = self.slot % epoch_length; - let start = self.slot - slot_in_epoch; - let end = self.slot + (epoch_length - slot_in_epoch); - start..end - } - - /// Returns the start slot and end slot of the current epoch containing `self.slot`. - pub fn get_previous_epoch_boundaries(&self, spec: &ChainSpec) -> Range { - let current_epoch = self.slot / spec.epoch_length; - let previous_epoch = current_epoch.saturating_sub(1); - let start = previous_epoch * spec.epoch_length; - let end = start + spec.epoch_length; - start..end - } - - fn get_previous_epoch_committee_count_per_slot(&self, spec: &ChainSpec) -> u64 { - let previous_active_validators = get_active_validator_indices( - &self.validator_registry, - self.previous_epoch_calculation_slot, - ); - self.get_committee_count_per_slot(previous_active_validators.len(), spec) as u64 - } - - pub fn get_current_epoch_committee_count_per_slot(&self, spec: &ChainSpec) -> u64 { - let current_active_validators = get_active_validator_indices( - &self.validator_registry, - self.current_epoch_calculation_slot, - ); - self.get_committee_count_per_slot(current_active_validators.len(), spec) - } - - pub fn get_crosslink_committees_at_slot( - &self, - slot: u64, - spec: &ChainSpec, - ) -> Result, u64)>> { - let epoch = slot / spec.epoch_length; - let current_epoch = self.slot / spec.epoch_length; - let previous_epoch = if current_epoch == spec.genesis_slot { - current_epoch - } else { - current_epoch.saturating_sub(1) - }; - let next_epoch = current_epoch + 1; - - ensure!( - (previous_epoch <= epoch) & (epoch < next_epoch), - Error::InvalidEpoch(slot, previous_epoch..current_epoch) - ); - - let offset = slot % spec.epoch_length; - - let (committees_per_slot, shuffling, slot_start_shard) = if epoch < current_epoch { - let committees_per_slot = self.get_previous_epoch_committee_count_per_slot(spec); - let shuffling = self.get_shuffling( - self.previous_epoch_seed, - self.previous_epoch_calculation_slot, - spec, - ); - let slot_start_shard = - (self.previous_epoch_start_shard + committees_per_slot * offset) % spec.shard_count; - (committees_per_slot, shuffling, slot_start_shard) - } else { - let committees_per_slot = self.get_current_epoch_committee_count_per_slot(spec); - let shuffling = self.get_shuffling( - self.current_epoch_seed, - self.current_epoch_calculation_slot, - spec, - ); - let slot_start_shard = - (self.current_epoch_start_shard + committees_per_slot * offset) % spec.shard_count; - (committees_per_slot, shuffling, slot_start_shard) - }; - - let mut crosslinks_at_slot = vec![]; - for i in 0..committees_per_slot { - let tuple = ( - shuffling[(committees_per_slot * offset + i) as usize].clone(), - (slot_start_shard + i) % spec.shard_count, - ); - crosslinks_at_slot.push(tuple) - } - Ok(crosslinks_at_slot) - } -} - -/* -/// Utility function pending this functionality being stabilized on the `Range` type. -fn range_contains(range: &Range, target: T) -> bool { - range.start <= target && target < range.end -} -*/ diff --git a/eth2/types/src/beacon_state/epoch_processing.rs b/eth2/types/src/beacon_state/epoch_processing.rs deleted file mode 100644 index d93cd4fe5..000000000 --- a/eth2/types/src/beacon_state/epoch_processing.rs +++ /dev/null @@ -1,799 +0,0 @@ -use super::winning_root::{Error as WinningRootError, WinningRoot}; -use crate::{ - beacon_state::{AttestationParticipantsError, CommitteesError}, - validator::StatusFlags, - validator_registry::get_active_validator_indices, - BeaconState, ChainSpec, Crosslink, Hash256, PendingAttestation, -}; -use integer_sqrt::IntegerSquareRoot; -use log::debug; -use rayon::prelude::*; -use std::collections::{HashMap, HashSet}; -use std::iter::FromIterator; - -#[derive(Debug, PartialEq)] -pub enum Error { - UnableToDetermineProducer, - NoBlockRoots, - BaseRewardQuotientIsZero, - CommitteesError(CommitteesError), - AttestationParticipantsError(AttestationParticipantsError), - InclusionError(InclusionError), - WinningRootError(WinningRootError), -} - -#[derive(Debug, PartialEq)] -pub enum InclusionError { - NoIncludedAttestations, - AttestationParticipantsError(AttestationParticipantsError), -} - -macro_rules! safe_add_assign { - ($a: expr, $b: expr) => { - $a = $a.saturating_add($b); - }; -} -macro_rules! safe_sub_assign { - ($a: expr, $b: expr) => { - $a = $a.saturating_sub($b); - }; -} - -impl BeaconState { - pub fn per_epoch_processing(&mut self, spec: &ChainSpec) -> Result<(), Error> { - debug!( - "Starting per-epoch processing on epoch {}...", - self.current_epoch(spec) - ); - /* - * All Validators - */ - let active_validator_indices = - get_active_validator_indices(&self.validator_registry, self.slot); - let total_balance = self.get_effective_balances(&active_validator_indices[..], spec); - - debug!( - "{} validators with a total balance of {} wei.", - active_validator_indices.len(), - total_balance - ); - - let current_epoch_attestations: Vec<&PendingAttestation> = self - .latest_attestations - .par_iter() - .filter(|a| a.data.slot / spec.epoch_length == self.current_epoch(spec)) - .collect(); - - debug!( - "Current epoch attestations: {}", - current_epoch_attestations.len() - ); - - /* - * Validators attesting during the current epoch. - */ - if self.latest_block_roots.is_empty() { - return Err(Error::NoBlockRoots); - } - - let current_epoch_boundary_attestations: Vec<&PendingAttestation> = - current_epoch_attestations - .par_iter() - .filter(|a| { - match self.get_block_root(self.current_epoch_start_slot(spec), spec) { - Some(block_root) => { - (a.data.epoch_boundary_root == *block_root) - && (a.data.justified_slot == self.justified_slot) - } - // Protected by a check that latest_block_roots isn't empty. - // - // TODO: provide detailed reasoning. - None => unreachable!(), - } - }) - .cloned() - .collect(); - - let current_epoch_boundary_attester_indices = self - .get_attestation_participants_union(¤t_epoch_boundary_attestations[..], spec)?; - let current_epoch_boundary_attesting_balance = - self.get_effective_balances(¤t_epoch_boundary_attester_indices[..], spec); - - debug!( - "Current epoch boundary attesters: {}", - current_epoch_boundary_attester_indices.len() - ); - - /* - * Validators attesting during the previous epoch - */ - - /* - * Validators that made an attestation during the previous epoch - */ - let previous_epoch_attestations: Vec<&PendingAttestation> = self - .latest_attestations - .par_iter() - .filter(|a| { - //TODO: ensure these saturating subs are correct. - a.data.slot / spec.epoch_length == self.previous_epoch(spec) - }) - .collect(); - - debug!( - "previous epoch attestations: {}", - previous_epoch_attestations.len() - ); - - let previous_epoch_attester_indices = - self.get_attestation_participants_union(&previous_epoch_attestations[..], spec)?; - - /* - * Validators targetting the previous justified slot - */ - let previous_epoch_justified_attestations: Vec<&PendingAttestation> = { - let mut a: Vec<&PendingAttestation> = current_epoch_attestations - .iter() - .filter(|a| a.data.justified_slot == self.previous_justified_slot) - .cloned() - .collect(); - let mut b: Vec<&PendingAttestation> = previous_epoch_attestations - .iter() - .filter(|a| a.data.justified_slot == self.previous_justified_slot) - .cloned() - .collect(); - a.append(&mut b); - a - }; - - let previous_epoch_justified_attester_indices = self - .get_attestation_participants_union(&previous_epoch_justified_attestations[..], spec)?; - let previous_epoch_justified_attesting_balance = - self.get_effective_balances(&previous_epoch_justified_attester_indices[..], spec); - - /* - * Validators justifying the epoch boundary block at the start of the previous epoch - */ - let previous_epoch_boundary_attestations: Vec<&PendingAttestation> = - previous_epoch_justified_attestations - .iter() - .filter(|a| { - match self.get_block_root(self.previous_epoch_start_slot(spec), spec) { - Some(block_root) => a.data.epoch_boundary_root == *block_root, - // Protected by a check that latest_block_roots isn't empty. - // - // TODO: provide detailed reasoning. - None => unreachable!(), - } - }) - .cloned() - .collect(); - - let previous_epoch_boundary_attester_indices = self - .get_attestation_participants_union(&previous_epoch_boundary_attestations[..], spec)?; - let previous_epoch_boundary_attesting_balance = - self.get_effective_balances(&previous_epoch_boundary_attester_indices[..], spec); - - /* - * Validators attesting to the expected beacon chain head during the previous epoch. - */ - let previous_epoch_head_attestations: Vec<&PendingAttestation> = - previous_epoch_attestations - .iter() - .filter(|a| { - match self.get_block_root(a.data.slot, spec) { - Some(block_root) => a.data.beacon_block_root == *block_root, - // Protected by a check that latest_block_roots isn't empty. - // - // TODO: provide detailed reasoning. - None => unreachable!(), - } - }) - .cloned() - .collect(); - - let previous_epoch_head_attester_indices = - self.get_attestation_participants_union(&previous_epoch_head_attestations[..], spec)?; - let previous_epoch_head_attesting_balance = - self.get_effective_balances(&previous_epoch_head_attester_indices[..], spec); - - debug!( - "previous_epoch_head_attester_balance of {} wei.", - previous_epoch_head_attesting_balance - ); - - /* - * Eth1 Data - */ - if self.slot % spec.eth1_data_voting_period == 0 { - for eth1_data_vote in &self.eth1_data_votes { - if eth1_data_vote.vote_count * 2 > spec.eth1_data_voting_period { - self.latest_eth1_data = eth1_data_vote.eth1_data.clone(); - } - } - self.eth1_data_votes = vec![]; - } - - /* - * Justification - */ - self.previous_justified_slot = self.justified_slot; - let (new_bitfield, _) = self.justification_bitfield.overflowing_mul(2); - self.justification_bitfield = new_bitfield; - - // If >= 2/3 of validators voted for the previous epoch boundary - if (3 * previous_epoch_boundary_attesting_balance) >= (2 * total_balance) { - // TODO: check saturating_sub is correct. - self.justification_bitfield |= 2; - self.justified_slot = self.slot.saturating_sub(2 * spec.epoch_length); - debug!(">= 2/3 voted for previous epoch boundary"); - } - - // If >= 2/3 of validators voted for the current epoch boundary - if (3 * current_epoch_boundary_attesting_balance) >= (2 * total_balance) { - // TODO: check saturating_sub is correct. - self.justification_bitfield |= 1; - self.justified_slot = self.slot.saturating_sub(1 * spec.epoch_length); - debug!(">= 2/3 voted for current epoch boundary"); - } - - if (self.previous_justified_slot == self.slot.saturating_sub(2 * spec.epoch_length)) - && (self.justification_bitfield % 4 == 3) - { - self.finalized_slot = self.previous_justified_slot; - } - if (self.previous_justified_slot == self.slot.saturating_sub(3 * spec.epoch_length)) - && (self.justification_bitfield % 8 == 7) - { - self.finalized_slot = self.previous_justified_slot; - } - if (self.previous_justified_slot == self.slot.saturating_sub(4 * spec.epoch_length)) - && (self.justification_bitfield % 16 == 14) - { - self.finalized_slot = self.previous_justified_slot; - } - if (self.previous_justified_slot == self.slot.saturating_sub(4 * spec.epoch_length)) - && (self.justification_bitfield % 16 == 15) - { - self.finalized_slot = self.previous_justified_slot; - } - - debug!( - "Finalized slot {}, justified slot {}.", - self.finalized_slot, self.justified_slot - ); - - /* - * Crosslinks - */ - - // Cached for later lookups. - let mut winning_root_for_shards: HashMap> = - HashMap::new(); - - // for slot in self.slot.saturating_sub(2 * spec.epoch_length)..self.slot { - for slot in self.get_previous_epoch_boundaries(spec) { - let crosslink_committees_at_slot = self.get_crosslink_committees_at_slot(slot, spec)?; - - for (crosslink_committee, shard) in crosslink_committees_at_slot { - let shard = shard as u64; - - let winning_root = self.winning_root( - shard, - ¤t_epoch_attestations, - &previous_epoch_attestations, - spec, - ); - - if let Ok(winning_root) = &winning_root { - let total_committee_balance = - self.get_effective_balances(&crosslink_committee[..], spec); - - if (3 * winning_root.total_attesting_balance) >= (2 * total_committee_balance) { - self.latest_crosslinks[shard as usize] = Crosslink { - slot: self.slot, - shard_block_root: winning_root.shard_block_root, - } - } - } - winning_root_for_shards.insert(shard, winning_root); - } - } - - debug!( - "Found {} winning shard roots.", - winning_root_for_shards.len() - ); - - /* - * Rewards and Penalities - */ - let base_reward_quotient = total_balance.integer_sqrt(); - if base_reward_quotient == 0 { - return Err(Error::BaseRewardQuotientIsZero); - } - - /* - * Justification and finalization - */ - let epochs_since_finality = - self.slot.saturating_sub(self.finalized_slot) / spec.epoch_length; - - // TODO: fix this extra map - let previous_epoch_justified_attester_indices_hashset: HashSet = - HashSet::from_iter(previous_epoch_justified_attester_indices.iter().map(|i| *i)); - let previous_epoch_boundary_attester_indices_hashset: HashSet = - HashSet::from_iter(previous_epoch_boundary_attester_indices.iter().map(|i| *i)); - let previous_epoch_head_attester_indices_hashset: HashSet = - HashSet::from_iter(previous_epoch_head_attester_indices.iter().map(|i| *i)); - let previous_epoch_attester_indices_hashset: HashSet = - HashSet::from_iter(previous_epoch_attester_indices.iter().map(|i| *i)); - - debug!("previous epoch justified attesters: {}, previous epoch boundary attesters: {}, previous epoch head attesters: {}, previous epoch attesters: {}", previous_epoch_justified_attester_indices.len(), previous_epoch_boundary_attester_indices.len(), previous_epoch_head_attester_indices.len(), previous_epoch_attester_indices.len()); - - debug!("{} epochs since finality.", epochs_since_finality); - - if epochs_since_finality <= 4 { - for index in 0..self.validator_balances.len() { - let base_reward = self.base_reward(index, base_reward_quotient, spec); - - if previous_epoch_justified_attester_indices_hashset.contains(&index) { - safe_add_assign!( - self.validator_balances[index], - base_reward * previous_epoch_justified_attesting_balance / total_balance - ); - } else { - safe_sub_assign!(self.validator_balances[index], base_reward); - } - - if previous_epoch_boundary_attester_indices_hashset.contains(&index) { - safe_add_assign!( - self.validator_balances[index], - base_reward * previous_epoch_boundary_attesting_balance / total_balance - ); - } else { - safe_sub_assign!(self.validator_balances[index], base_reward); - } - - if previous_epoch_head_attester_indices_hashset.contains(&index) { - safe_add_assign!( - self.validator_balances[index], - base_reward * previous_epoch_head_attesting_balance / total_balance - ); - } else { - safe_sub_assign!(self.validator_balances[index], base_reward); - } - } - - for index in previous_epoch_attester_indices { - let base_reward = self.base_reward(index, base_reward_quotient, spec); - let inclusion_distance = - self.inclusion_distance(&previous_epoch_attestations, index, spec)?; - - safe_add_assign!( - self.validator_balances[index], - base_reward * spec.min_attestation_inclusion_delay / inclusion_distance - ) - } - } else { - for index in 0..self.validator_balances.len() { - let inactivity_penalty = self.inactivity_penalty( - index, - epochs_since_finality, - base_reward_quotient, - spec, - ); - - if !previous_epoch_justified_attester_indices_hashset.contains(&index) { - safe_sub_assign!(self.validator_balances[index], inactivity_penalty); - } - - if !previous_epoch_boundary_attester_indices_hashset.contains(&index) { - safe_sub_assign!(self.validator_balances[index], inactivity_penalty); - } - - if !previous_epoch_head_attester_indices_hashset.contains(&index) { - safe_sub_assign!(self.validator_balances[index], inactivity_penalty); - } - } - - for index in previous_epoch_attester_indices { - let base_reward = self.base_reward(index, base_reward_quotient, spec); - let inclusion_distance = - self.inclusion_distance(&previous_epoch_attestations, index, spec)?; - - safe_sub_assign!( - self.validator_balances[index], - base_reward - - base_reward * spec.min_attestation_inclusion_delay / inclusion_distance - ); - } - } - - debug!("Processed validator justification and finalization rewards/penalities."); - - /* - * Attestation inclusion - */ - for &index in &previous_epoch_attester_indices_hashset { - let inclusion_slot = - self.inclusion_slot(&previous_epoch_attestations[..], index, spec)?; - let proposer_index = self - .get_beacon_proposer_index(inclusion_slot, spec) - .map_err(|_| Error::UnableToDetermineProducer)?; - let base_reward = self.base_reward(proposer_index, base_reward_quotient, spec); - safe_add_assign!( - self.validator_balances[proposer_index], - base_reward / spec.includer_reward_quotient - ); - } - - debug!( - "Previous epoch attesters: {}.", - previous_epoch_attester_indices_hashset.len() - ); - - /* - * Crosslinks - */ - for slot in self.get_previous_epoch_boundaries(spec) { - let crosslink_committees_at_slot = self.get_crosslink_committees_at_slot(slot, spec)?; - - for (_crosslink_committee, shard) in crosslink_committees_at_slot { - let shard = shard as u64; - - if let Some(Ok(winning_root)) = winning_root_for_shards.get(&shard) { - // TODO: remove the map. - let attesting_validator_indices: HashSet = HashSet::from_iter( - winning_root.attesting_validator_indices.iter().map(|i| *i), - ); - - for index in 0..self.validator_balances.len() { - let base_reward = self.base_reward(index, base_reward_quotient, spec); - - if attesting_validator_indices.contains(&index) { - safe_add_assign!( - self.validator_balances[index], - base_reward * winning_root.total_attesting_balance - / winning_root.total_balance - ); - } else { - safe_sub_assign!(self.validator_balances[index], base_reward); - } - } - - for index in &winning_root.attesting_validator_indices { - let base_reward = self.base_reward(*index, base_reward_quotient, spec); - safe_add_assign!( - self.validator_balances[*index], - base_reward * winning_root.total_attesting_balance - / winning_root.total_balance - ); - } - } - } - } - - /* - * Ejections - */ - self.process_ejections(); - - /* - * Validator Registry - */ - self.previous_epoch_calculation_slot = self.current_epoch_calculation_slot; - self.previous_epoch_start_shard = self.current_epoch_start_shard; - self.previous_epoch_seed = self.current_epoch_seed; - - let should_update_validator_registy = if self.finalized_slot - > self.validator_registry_update_slot - { - (0..self.get_current_epoch_committee_count_per_slot(spec)).all(|i| { - let shard = (self.current_epoch_start_shard + i as u64) % spec.shard_count; - self.latest_crosslinks[shard as usize].slot > self.validator_registry_update_slot - }) - } else { - false - }; - - if should_update_validator_registy { - self.update_validator_registry(spec); - - self.current_epoch_calculation_slot = self.slot; - self.current_epoch_start_shard = (self.current_epoch_start_shard - + self.get_current_epoch_committee_count_per_slot(spec) as u64 * spec.epoch_length) - % spec.shard_count; - self.current_epoch_seed = self.get_randao_mix( - self.current_epoch_calculation_slot - .saturating_sub(spec.seed_lookahead), - spec, - ); - } else { - let epochs_since_last_registry_change = - (self.slot - self.validator_registry_update_slot) / spec.epoch_length; - if epochs_since_last_registry_change.is_power_of_two() { - self.current_epoch_calculation_slot = self.slot; - self.current_epoch_seed = self.get_randao_mix( - self.current_epoch_calculation_slot - .saturating_sub(spec.seed_lookahead), - spec, - ); - } - } - - self.process_penalties_and_exits(spec); - - let e = self.slot / spec.epoch_length; - self.latest_penalized_balances[((e + 1) % spec.latest_penalized_exit_length) as usize] = - self.latest_penalized_balances[(e % spec.latest_penalized_exit_length) as usize]; - - self.latest_attestations = self - .latest_attestations - .iter() - .filter(|a| a.data.slot / spec.epoch_length >= self.current_epoch(spec)) - .cloned() - .collect(); - - debug!("Epoch transition complete."); - - Ok(()) - } - - fn process_penalties_and_exits(&mut self, spec: &ChainSpec) { - let active_validator_indices = - get_active_validator_indices(&self.validator_registry, self.slot); - let total_balance = self.get_effective_balances(&active_validator_indices[..], spec); - - for index in 0..self.validator_balances.len() { - let validator = &self.validator_registry[index]; - - if (self.slot / spec.epoch_length) - == (validator.penalized_slot / spec.epoch_length) - + spec.latest_penalized_exit_length / 2 - { - let e = (self.slot / spec.epoch_length) % spec.latest_penalized_exit_length; - let total_at_start = self.latest_penalized_balances - [((e + 1) % spec.latest_penalized_exit_length) as usize]; - let total_at_end = self.latest_penalized_balances[e as usize]; - let total_penalities = total_at_end.saturating_sub(total_at_start); - let penalty = self.get_effective_balance(index, spec) - * std::cmp::min(total_penalities * 3, total_balance) - / total_balance; - safe_sub_assign!(self.validator_balances[index], penalty); - } - } - - let eligible = |index: usize| { - let validator = &self.validator_registry[index]; - - if validator.penalized_slot <= self.slot { - let penalized_withdrawal_time = - spec.latest_penalized_exit_length * spec.epoch_length / 2; - self.slot >= validator.penalized_slot + penalized_withdrawal_time - } else { - self.slot >= validator.exit_slot + spec.min_validator_withdrawal_time - } - }; - - let mut eligable_indices: Vec = (0..self.validator_registry.len()) - .filter(|i| eligible(*i)) - .collect(); - eligable_indices.sort_by_key(|i| self.validator_registry[*i].exit_count); - let mut withdrawn_so_far = 0; - for index in eligable_indices { - self.prepare_validator_for_withdrawal(index); - withdrawn_so_far += 1; - if withdrawn_so_far >= spec.max_withdrawals_per_epoch { - break; - } - } - } - - fn prepare_validator_for_withdrawal(&mut self, index: usize) { - //TODO: we're not ANDing here, we're setting. Potentially wrong. - self.validator_registry[index].status_flags = Some(StatusFlags::Withdrawable); - } - - fn get_randao_mix(&mut self, slot: u64, spec: &ChainSpec) -> Hash256 { - assert!(self.slot < slot + spec.latest_randao_mixes_length); - assert!(slot <= self.slot); - self.latest_randao_mixes[(slot & spec.latest_randao_mixes_length) as usize] - } - - fn update_validator_registry(&mut self, spec: &ChainSpec) { - let active_validator_indices = - get_active_validator_indices(&self.validator_registry, self.slot); - let total_balance = self.get_effective_balances(&active_validator_indices[..], spec); - - let max_balance_churn = std::cmp::max( - spec.max_deposit, - total_balance / (2 * spec.max_balance_churn_quotient), - ); - - let mut balance_churn = 0; - for index in 0..self.validator_registry.len() { - let validator = &self.validator_registry[index]; - - if (validator.activation_slot > self.slot + spec.entry_exit_delay) - && self.validator_balances[index] >= spec.max_deposit - { - balance_churn += self.get_effective_balance(index, spec); - if balance_churn > max_balance_churn { - break; - } - - self.activate_validator(index, false, spec); - } - } - - let mut balance_churn = 0; - for index in 0..self.validator_registry.len() { - let validator = &self.validator_registry[index]; - - if (validator.exit_slot > self.slot + spec.entry_exit_delay) - && validator.status_flags == Some(StatusFlags::InitiatedExit) - { - balance_churn += self.get_effective_balance(index, spec); - if balance_churn > max_balance_churn { - break; - } - - self.exit_validator(index, spec); - } - } - - self.validator_registry_update_slot = self.slot; - } - - fn exit_validator(&mut self, validator_index: usize, spec: &ChainSpec) { - if self.validator_registry[validator_index].exit_slot - <= self.entry_exit_effect_slot(self.slot, spec) - { - return; - } - - self.validator_registry[validator_index].exit_slot = - self.entry_exit_effect_slot(self.slot, spec); - - self.validator_registry_exit_count += 1; - self.validator_registry[validator_index].exit_count = self.validator_registry_exit_count; - } - - fn activate_validator(&mut self, validator_index: usize, is_genesis: bool, spec: &ChainSpec) { - self.validator_registry[validator_index].activation_slot = if is_genesis { - spec.genesis_slot - } else { - self.entry_exit_effect_slot(self.slot, spec) - } - } - - fn entry_exit_effect_slot(&self, slot: u64, spec: &ChainSpec) -> u64 { - (slot - slot % spec.epoch_length) + spec.epoch_length + spec.entry_exit_delay - } - - fn process_ejections(&self) { - //TODO: stubbed out. - } - - fn inactivity_penalty( - &self, - validator_index: usize, - epochs_since_finality: u64, - base_reward_quotient: u64, - spec: &ChainSpec, - ) -> u64 { - let effective_balance = self.get_effective_balance(validator_index, spec); - self.base_reward(validator_index, base_reward_quotient, spec) - + effective_balance * epochs_since_finality / spec.inactivity_penalty_quotient / 2 - } - - fn inclusion_distance( - &self, - attestations: &[&PendingAttestation], - validator_index: usize, - spec: &ChainSpec, - ) -> Result { - let attestation = - self.earliest_included_attestation(attestations, validator_index, spec)?; - Ok(attestation - .slot_included - .saturating_sub(attestation.data.slot)) - } - - fn inclusion_slot( - &self, - attestations: &[&PendingAttestation], - validator_index: usize, - spec: &ChainSpec, - ) -> Result { - let attestation = - self.earliest_included_attestation(attestations, validator_index, spec)?; - Ok(attestation.slot_included) - } - - fn earliest_included_attestation( - &self, - attestations: &[&PendingAttestation], - validator_index: usize, - spec: &ChainSpec, - ) -> Result { - let mut included_attestations = vec![]; - - for (i, a) in attestations.iter().enumerate() { - let participants = - self.get_attestation_participants(&a.data, &a.aggregation_bitfield, spec)?; - if participants - .iter() - .find(|i| **i == validator_index) - .is_some() - { - included_attestations.push(i); - } - } - - let earliest_attestation_index = included_attestations - .iter() - .min_by_key(|i| attestations[**i].slot_included) - .ok_or_else(|| InclusionError::NoIncludedAttestations)?; - Ok(attestations[*earliest_attestation_index].clone()) - } - - fn base_reward( - &self, - validator_index: usize, - base_reward_quotient: u64, - spec: &ChainSpec, - ) -> u64 { - self.get_effective_balance(validator_index, spec) / base_reward_quotient / 5 - } - - pub fn get_effective_balances(&self, validator_indices: &[usize], spec: &ChainSpec) -> u64 { - validator_indices - .iter() - .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)) - } - - pub fn get_effective_balance(&self, validator_index: usize, spec: &ChainSpec) -> u64 { - std::cmp::min(self.validator_balances[validator_index], spec.max_deposit) - } - - pub fn get_block_root(&self, slot: u64, spec: &ChainSpec) -> Option<&Hash256> { - if self.slot <= slot + spec.latest_block_roots_length && slot <= self.slot { - self.latest_block_roots - .get((slot % spec.latest_block_roots_length) as usize) - } else { - None - } - } -} - -impl From for Error { - fn from(e: CommitteesError) -> Error { - Error::CommitteesError(e) - } -} - -impl From for Error { - fn from(e: AttestationParticipantsError) -> Error { - Error::AttestationParticipantsError(e) - } -} - -impl From for Error { - fn from(e: InclusionError) -> Error { - Error::InclusionError(e) - } -} - -impl From for InclusionError { - fn from(e: AttestationParticipantsError) -> InclusionError { - InclusionError::AttestationParticipantsError(e) - } -} - -impl From for Error { - fn from(e: WinningRootError) -> Error { - Error::WinningRootError(e) - } -} diff --git a/eth2/types/src/beacon_state/mod.rs b/eth2/types/src/beacon_state/mod.rs deleted file mode 100644 index f0548914e..000000000 --- a/eth2/types/src/beacon_state/mod.rs +++ /dev/null @@ -1,281 +0,0 @@ -use super::crosslink::Crosslink; -use super::eth1_data::Eth1Data; -use super::eth1_data_vote::Eth1DataVote; -use super::fork::Fork; -use super::pending_attestation::PendingAttestation; -use super::validator::Validator; -use super::Hash256; -use crate::test_utils::TestRandom; -use rand::RngCore; -use serde_derive::Serialize; -use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; - -mod attestation_participants; -mod attestation_validation; -mod block_processing; -mod committees; -mod epoch_processing; -mod shuffling; -mod slot_processing; -mod winning_root; - -pub use self::attestation_participants::Error as AttestationParticipantsError; -pub use self::attestation_validation::Error as AttestationValidationError; -pub use self::block_processing::Error as BlockProcessingError; -pub use self::committees::Error as CommitteesError; -pub use self::epoch_processing::Error as EpochProcessingError; -pub use self::slot_processing::Error as SlotProcessingError; - -// Custody will not be added to the specs until Phase 1 (Sharding Phase) so dummy class used. -type CustodyChallenge = usize; - -#[derive(Debug, PartialEq, Clone, Default, Serialize)] -pub struct BeaconState { - // Misc - pub slot: u64, - pub genesis_time: u64, - pub fork_data: Fork, - - // Validator registry - pub validator_registry: Vec, - pub validator_balances: Vec, - pub validator_registry_update_slot: u64, - pub validator_registry_exit_count: u64, - pub validator_registry_delta_chain_tip: Hash256, - - // Randomness and committees - pub latest_randao_mixes: Vec, - pub latest_vdf_outputs: Vec, - pub previous_epoch_start_shard: u64, - pub current_epoch_start_shard: u64, - pub previous_epoch_calculation_slot: u64, - pub current_epoch_calculation_slot: u64, - pub previous_epoch_seed: Hash256, - pub current_epoch_seed: Hash256, - - // Custody challenges - pub custody_challenges: Vec, - - // Finality - pub previous_justified_slot: u64, - pub justified_slot: u64, - pub justification_bitfield: u64, - pub finalized_slot: u64, - - // Recent state - pub latest_crosslinks: Vec, - pub latest_block_roots: Vec, - pub latest_penalized_balances: Vec, - pub latest_attestations: Vec, - pub batched_block_roots: Vec, - - // Ethereum 1.0 chain data - pub latest_eth1_data: Eth1Data, - pub eth1_data_votes: Vec, -} - -impl BeaconState { - pub fn canonical_root(&self) -> Hash256 { - Hash256::from(&self.hash_tree_root()[..]) - } -} - -impl Encodable for BeaconState { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.slot); - s.append(&self.genesis_time); - s.append(&self.fork_data); - s.append(&self.validator_registry); - s.append(&self.validator_balances); - s.append(&self.validator_registry_update_slot); - s.append(&self.validator_registry_exit_count); - s.append(&self.validator_registry_delta_chain_tip); - s.append(&self.latest_randao_mixes); - s.append(&self.latest_vdf_outputs); - s.append(&self.previous_epoch_start_shard); - s.append(&self.current_epoch_start_shard); - s.append(&self.previous_epoch_calculation_slot); - s.append(&self.current_epoch_calculation_slot); - s.append(&self.previous_epoch_seed); - s.append(&self.current_epoch_seed); - s.append(&self.custody_challenges); - s.append(&self.previous_justified_slot); - s.append(&self.justified_slot); - s.append(&self.justification_bitfield); - s.append(&self.finalized_slot); - s.append(&self.latest_crosslinks); - s.append(&self.latest_block_roots); - s.append(&self.latest_penalized_balances); - s.append(&self.latest_attestations); - s.append(&self.batched_block_roots); - s.append(&self.latest_eth1_data); - s.append(&self.eth1_data_votes); - } -} - -impl Decodable for BeaconState { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (slot, i) = <_>::ssz_decode(bytes, i)?; - let (genesis_time, i) = <_>::ssz_decode(bytes, i)?; - let (fork_data, i) = <_>::ssz_decode(bytes, i)?; - let (validator_registry, i) = <_>::ssz_decode(bytes, i)?; - let (validator_balances, i) = <_>::ssz_decode(bytes, i)?; - let (validator_registry_update_slot, i) = <_>::ssz_decode(bytes, i)?; - let (validator_registry_exit_count, i) = <_>::ssz_decode(bytes, i)?; - let (validator_registry_delta_chain_tip, i) = <_>::ssz_decode(bytes, i)?; - let (latest_randao_mixes, i) = <_>::ssz_decode(bytes, i)?; - let (latest_vdf_outputs, i) = <_>::ssz_decode(bytes, i)?; - let (previous_epoch_start_shard, i) = <_>::ssz_decode(bytes, i)?; - let (current_epoch_start_shard, i) = <_>::ssz_decode(bytes, i)?; - let (previous_epoch_calculation_slot, i) = <_>::ssz_decode(bytes, i)?; - let (current_epoch_calculation_slot, i) = <_>::ssz_decode(bytes, i)?; - let (previous_epoch_seed, i) = <_>::ssz_decode(bytes, i)?; - let (current_epoch_seed, i) = <_>::ssz_decode(bytes, i)?; - let (custody_challenges, i) = <_>::ssz_decode(bytes, i)?; - let (previous_justified_slot, i) = <_>::ssz_decode(bytes, i)?; - let (justified_slot, i) = <_>::ssz_decode(bytes, i)?; - let (justification_bitfield, i) = <_>::ssz_decode(bytes, i)?; - let (finalized_slot, i) = <_>::ssz_decode(bytes, i)?; - let (latest_crosslinks, i) = <_>::ssz_decode(bytes, i)?; - let (latest_block_roots, i) = <_>::ssz_decode(bytes, i)?; - let (latest_penalized_balances, i) = <_>::ssz_decode(bytes, i)?; - let (latest_attestations, i) = <_>::ssz_decode(bytes, i)?; - let (batched_block_roots, i) = <_>::ssz_decode(bytes, i)?; - let (latest_eth1_data, i) = <_>::ssz_decode(bytes, i)?; - let (eth1_data_votes, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - Self { - slot, - genesis_time, - fork_data, - validator_registry, - validator_balances, - validator_registry_update_slot, - validator_registry_exit_count, - validator_registry_delta_chain_tip, - latest_randao_mixes, - latest_vdf_outputs, - previous_epoch_start_shard, - current_epoch_start_shard, - previous_epoch_calculation_slot, - current_epoch_calculation_slot, - previous_epoch_seed, - current_epoch_seed, - custody_challenges, - previous_justified_slot, - justified_slot, - justification_bitfield, - finalized_slot, - latest_crosslinks, - latest_block_roots, - latest_penalized_balances, - latest_attestations, - batched_block_roots, - latest_eth1_data, - eth1_data_votes, - }, - i, - )) - } -} - -impl TreeHash for BeaconState { - fn hash_tree_root(&self) -> Vec { - let mut result: Vec = vec![]; - result.append(&mut self.slot.hash_tree_root()); - result.append(&mut self.genesis_time.hash_tree_root()); - result.append(&mut self.fork_data.hash_tree_root()); - result.append(&mut self.validator_registry.hash_tree_root()); - result.append(&mut self.validator_balances.hash_tree_root()); - result.append(&mut self.validator_registry_update_slot.hash_tree_root()); - result.append(&mut self.validator_registry_exit_count.hash_tree_root()); - result.append(&mut self.validator_registry_delta_chain_tip.hash_tree_root()); - result.append(&mut self.latest_randao_mixes.hash_tree_root()); - result.append(&mut self.latest_vdf_outputs.hash_tree_root()); - result.append(&mut self.previous_epoch_start_shard.hash_tree_root()); - result.append(&mut self.current_epoch_start_shard.hash_tree_root()); - result.append(&mut self.previous_epoch_calculation_slot.hash_tree_root()); - result.append(&mut self.current_epoch_calculation_slot.hash_tree_root()); - result.append(&mut self.previous_epoch_seed.hash_tree_root()); - result.append(&mut self.current_epoch_seed.hash_tree_root()); - result.append(&mut self.custody_challenges.hash_tree_root()); - result.append(&mut self.previous_justified_slot.hash_tree_root()); - result.append(&mut self.justified_slot.hash_tree_root()); - result.append(&mut self.justification_bitfield.hash_tree_root()); - result.append(&mut self.finalized_slot.hash_tree_root()); - result.append(&mut self.latest_crosslinks.hash_tree_root()); - result.append(&mut self.latest_block_roots.hash_tree_root()); - result.append(&mut self.latest_penalized_balances.hash_tree_root()); - result.append(&mut self.latest_attestations.hash_tree_root()); - result.append(&mut self.batched_block_roots.hash_tree_root()); - result.append(&mut self.latest_eth1_data.hash_tree_root()); - result.append(&mut self.eth1_data_votes.hash_tree_root()); - hash(&result) - } -} - -impl TestRandom for BeaconState { - fn random_for_test(rng: &mut T) -> Self { - Self { - slot: <_>::random_for_test(rng), - genesis_time: <_>::random_for_test(rng), - fork_data: <_>::random_for_test(rng), - validator_registry: <_>::random_for_test(rng), - validator_balances: <_>::random_for_test(rng), - validator_registry_update_slot: <_>::random_for_test(rng), - validator_registry_exit_count: <_>::random_for_test(rng), - validator_registry_delta_chain_tip: <_>::random_for_test(rng), - latest_randao_mixes: <_>::random_for_test(rng), - latest_vdf_outputs: <_>::random_for_test(rng), - previous_epoch_start_shard: <_>::random_for_test(rng), - current_epoch_start_shard: <_>::random_for_test(rng), - previous_epoch_calculation_slot: <_>::random_for_test(rng), - current_epoch_calculation_slot: <_>::random_for_test(rng), - previous_epoch_seed: <_>::random_for_test(rng), - current_epoch_seed: <_>::random_for_test(rng), - custody_challenges: <_>::random_for_test(rng), - previous_justified_slot: <_>::random_for_test(rng), - justified_slot: <_>::random_for_test(rng), - justification_bitfield: <_>::random_for_test(rng), - finalized_slot: <_>::random_for_test(rng), - latest_crosslinks: <_>::random_for_test(rng), - latest_block_roots: <_>::random_for_test(rng), - latest_penalized_balances: <_>::random_for_test(rng), - latest_attestations: <_>::random_for_test(rng), - batched_block_roots: <_>::random_for_test(rng), - latest_eth1_data: <_>::random_for_test(rng), - eth1_data_votes: <_>::random_for_test(rng), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; - use ssz::ssz_encode; - - #[test] - pub fn test_ssz_round_trip() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = BeaconState::random_for_test(&mut rng); - - let bytes = ssz_encode(&original); - let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); - - assert_eq!(original, decoded); - } - - #[test] - pub fn test_hash_tree_root() { - let mut rng = XorShiftRng::from_seed([42; 16]); - let original = BeaconState::random_for_test(&mut rng); - - let result = original.hash_tree_root(); - - assert_eq!(result.len(), 32); - // TODO: Add further tests - // https://github.com/sigp/lighthouse/issues/170 - } -} diff --git a/eth2/types/src/beacon_state/shuffling.rs b/eth2/types/src/beacon_state/shuffling.rs deleted file mode 100644 index bef3abe71..000000000 --- a/eth2/types/src/beacon_state/shuffling.rs +++ /dev/null @@ -1,45 +0,0 @@ -use super::CommitteesError; -use crate::{validator_registry::get_active_validator_indices, BeaconState, ChainSpec, Hash256}; -use honey_badger_split::SplitExt; -use vec_shuffle::shuffle; - -impl BeaconState { - pub fn get_shuffling(&self, seed: Hash256, slot: u64, spec: &ChainSpec) -> Vec> { - let slot = slot - (slot % spec.epoch_length); - - let active_validator_indices = get_active_validator_indices(&self.validator_registry, slot); - - let committees_per_slot = - self.get_committee_count_per_slot(active_validator_indices.len(), spec); - - // TODO: check that Hash256 matches 'int_to_bytes32'. - let seed = seed ^ Hash256::from(slot); - let shuffled_active_validator_indices = - shuffle(&seed, active_validator_indices).expect("Max validator count exceed!"); - - shuffled_active_validator_indices - .honey_badger_split((committees_per_slot * spec.epoch_length) as usize) - .filter_map(|slice: &[usize]| Some(slice.to_vec())) - .collect() - } - - /// Returns the beacon proposer index for the `slot`. - /// If the state does not contain an index for a beacon proposer at the requested `slot`, then `None` is returned. - pub fn get_beacon_proposer_index( - &self, - slot: u64, - spec: &ChainSpec, - ) -> Result { - let committees = self.get_crosslink_committees_at_slot(slot, spec)?; - committees - .first() - .ok_or(CommitteesError::InsufficientNumberOfValidators) - .and_then(|(first_committee, _)| { - let index = (slot as usize) - .checked_rem(first_committee.len()) - .ok_or(CommitteesError::InsufficientNumberOfValidators)?; - // NOTE: next index will not panic as we have already returned if this is the case - Ok(first_committee[index]) - }) - } -} diff --git a/eth2/types/src/beacon_state/slot_processing.rs b/eth2/types/src/beacon_state/slot_processing.rs deleted file mode 100644 index 76135210f..000000000 --- a/eth2/types/src/beacon_state/slot_processing.rs +++ /dev/null @@ -1,73 +0,0 @@ -use crate::{ - beacon_state::{CommitteesError, EpochProcessingError}, - BeaconState, ChainSpec, Hash256, -}; - -#[derive(Debug, PartialEq)] -pub enum Error { - CommitteesError(CommitteesError), - EpochProcessingError(EpochProcessingError), -} - -impl BeaconState { - pub fn per_slot_processing( - &mut self, - previous_block_root: Hash256, - spec: &ChainSpec, - ) -> Result<(), Error> { - if (self.slot + 1) % spec.epoch_length == 0 { - self.per_epoch_processing(spec)?; - } - - self.slot += 1; - - let block_proposer = self.get_beacon_proposer_index(self.slot, spec)?; - - self.validator_registry[block_proposer].proposer_slots += 1; - self.latest_randao_mixes[(self.slot % spec.latest_randao_mixes_length) as usize] = - self.latest_randao_mixes[((self.slot - 1) % spec.latest_randao_mixes_length) as usize]; - - // Block roots. - self.latest_block_roots[((self.slot - 1) % spec.latest_block_roots_length) as usize] = - previous_block_root; - - if self.slot % spec.latest_block_roots_length == 0 { - let root = merkle_root(&self.latest_block_roots[..]); - self.batched_block_roots.push(root); - } - Ok(()) - } - - pub fn attestation_slot_and_shard_for_validator( - &self, - validator_index: usize, - spec: &ChainSpec, - ) -> Result, CommitteesError> { - let mut result = None; - for slot in self.get_current_epoch_boundaries(spec.epoch_length) { - for (committee, shard) in self.get_crosslink_committees_at_slot(slot, spec)? { - if let Some(committee_index) = committee.iter().position(|&i| i == validator_index) - { - result = Some((slot, shard, committee_index as u64)); - } - } - } - Ok(result) - } -} - -fn merkle_root(_input: &[Hash256]) -> Hash256 { - Hash256::zero() -} - -impl From for Error { - fn from(e: CommitteesError) -> Error { - Error::CommitteesError(e) - } -} - -impl From for Error { - fn from(e: EpochProcessingError) -> Error { - Error::EpochProcessingError(e) - } -} diff --git a/eth2/types/src/beacon_state/winning_root.rs b/eth2/types/src/beacon_state/winning_root.rs deleted file mode 100644 index 1b891af73..000000000 --- a/eth2/types/src/beacon_state/winning_root.rs +++ /dev/null @@ -1,106 +0,0 @@ -use crate::{ - beacon_state::AttestationParticipantsError, BeaconState, ChainSpec, Hash256, PendingAttestation, -}; -use std::collections::HashMap; - -#[derive(Debug, PartialEq)] -pub enum Error { - NoWinningRoot, - AttestationParticipantsError(AttestationParticipantsError), -} - -#[derive(Clone)] -pub struct WinningRoot { - pub shard_block_root: Hash256, - pub attesting_validator_indices: Vec, - pub total_balance: u64, - pub total_attesting_balance: u64, -} - -impl BeaconState { - pub(crate) fn winning_root( - &self, - shard: u64, - current_epoch_attestations: &[&PendingAttestation], - previous_epoch_attestations: &[&PendingAttestation], - spec: &ChainSpec, - ) -> Result { - let mut attestations = current_epoch_attestations.to_vec(); - attestations.append(&mut previous_epoch_attestations.to_vec()); - - let mut candidates: HashMap = HashMap::new(); - - let mut highest_seen_balance = 0; - - for a in &attestations { - if a.data.shard != shard { - continue; - } - - let shard_block_root = &a.data.shard_block_root; - - if candidates.contains_key(shard_block_root) { - continue; - } - - // TODO: `cargo fmt` makes this rather ugly; tidy up. - let attesting_validator_indices = attestations.iter().try_fold::<_, _, Result< - _, - AttestationParticipantsError, - >>( - vec![], - |mut acc, a| { - if (a.data.shard == shard) && (a.data.shard_block_root == *shard_block_root) { - acc.append(&mut self.get_attestation_participants( - &a.data, - &a.aggregation_bitfield, - spec, - )?); - } - Ok(acc) - }, - )?; - - let total_balance: u64 = attesting_validator_indices - .iter() - .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)); - - let total_attesting_balance: u64 = attesting_validator_indices - .iter() - .fold(0, |acc, i| acc + self.get_effective_balance(*i, spec)); - - if total_attesting_balance > highest_seen_balance { - highest_seen_balance = total_attesting_balance; - } - - let candidate_root = WinningRoot { - shard_block_root: shard_block_root.clone(), - attesting_validator_indices, - total_attesting_balance, - total_balance, - }; - - candidates.insert(*shard_block_root, candidate_root); - } - - Ok(candidates - .iter() - .filter_map(|(_hash, candidate)| { - if candidate.total_attesting_balance == highest_seen_balance { - Some(candidate) - } else { - None - } - }) - .min_by_key(|candidate| candidate.shard_block_root) - .ok_or_else(|| Error::NoWinningRoot)? - // TODO: avoid clone. - .clone()) - } -} - -impl From for Error { - fn from(e: AttestationParticipantsError) -> Error { - Error::AttestationParticipantsError(e) - } -} From 449312d95ceefc45b58baf21e0c63fe9e9cad178 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 5 Feb 2019 12:29:08 +1100 Subject: [PATCH 098/107] Set all test_harness tests to `ignore` They were taking too long, we'll need to build a different build cycle to cover these. --- beacon_node/beacon_chain/test_harness/tests/chain.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/beacon_node/beacon_chain/test_harness/tests/chain.rs b/beacon_node/beacon_chain/test_harness/tests/chain.rs index f84157533..c0b537695 100644 --- a/beacon_node/beacon_chain/test_harness/tests/chain.rs +++ b/beacon_node/beacon_chain/test_harness/tests/chain.rs @@ -4,6 +4,7 @@ use test_harness::BeaconChainHarness; use types::ChainSpec; #[test] +#[ignore] fn it_can_build_on_genesis_block() { let mut spec = ChainSpec::foundation(); spec.genesis_slot = spec.epoch_length * 8; From b9dc8437c9bd708ad906c1402dd87c382890e24f Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 5 Feb 2019 12:33:52 +1100 Subject: [PATCH 099/107] Move all beacon_block fns into the same file --- .../{beacon_block/mod.rs => beacon_block.rs} | 19 ++++++++++++++++--- eth2/types/src/beacon_block/signing.rs | 19 ------------------- 2 files changed, 16 insertions(+), 22 deletions(-) rename eth2/types/src/{beacon_block/mod.rs => beacon_block.rs} (84%) delete mode 100644 eth2/types/src/beacon_block/signing.rs diff --git a/eth2/types/src/beacon_block/mod.rs b/eth2/types/src/beacon_block.rs similarity index 84% rename from eth2/types/src/beacon_block/mod.rs rename to eth2/types/src/beacon_block.rs index 3e07572d1..3e2c51ede 100644 --- a/eth2/types/src/beacon_block/mod.rs +++ b/eth2/types/src/beacon_block.rs @@ -1,12 +1,10 @@ -use super::{BeaconBlockBody, Eth1Data, Hash256}; +use super::{BeaconBlockBody, ChainSpec, Eth1Data, Hash256, ProposalSignedData}; use crate::test_utils::TestRandom; use bls::Signature; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; -mod signing; - #[derive(Debug, PartialEq, Clone, Serialize)] pub struct BeaconBlock { pub slot: u64, @@ -22,6 +20,21 @@ impl BeaconBlock { pub fn canonical_root(&self) -> Hash256 { Hash256::from(&self.hash_tree_root()[..]) } + + pub fn proposal_root(&self, spec: &ChainSpec) -> Hash256 { + let block_without_signature_root = { + let mut block_without_signature = self.clone(); + block_without_signature.signature = spec.empty_signature.clone(); + block_without_signature.canonical_root() + }; + + let proposal = ProposalSignedData { + slot: self.slot, + shard: spec.beacon_chain_shard_number, + block_root: block_without_signature_root, + }; + Hash256::from_slice(&proposal.hash_tree_root()[..]) + } } impl Encodable for BeaconBlock { diff --git a/eth2/types/src/beacon_block/signing.rs b/eth2/types/src/beacon_block/signing.rs deleted file mode 100644 index e3317b920..000000000 --- a/eth2/types/src/beacon_block/signing.rs +++ /dev/null @@ -1,19 +0,0 @@ -use crate::{BeaconBlock, ChainSpec, Hash256, ProposalSignedData}; -use ssz::TreeHash; - -impl BeaconBlock { - pub fn proposal_root(&self, spec: &ChainSpec) -> Hash256 { - let block_without_signature_root = { - let mut block_without_signature = self.clone(); - block_without_signature.signature = spec.empty_signature.clone(); - block_without_signature.canonical_root() - }; - - let proposal = ProposalSignedData { - slot: self.slot, - shard: spec.beacon_chain_shard_number, - block_root: block_without_signature_root, - }; - Hash256::from_slice(&proposal.hash_tree_root()[..]) - } -} From 05b890860ac8208a46bd0cad2b540895398c0c4e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 5 Feb 2019 12:35:24 +1100 Subject: [PATCH 100/107] Move all attestaion fns into same file --- eth2/types/src/{attestation/mod.rs => attestation.rs} | 6 ++++-- eth2/types/src/attestation/signing.rs | 7 ------- 2 files changed, 4 insertions(+), 9 deletions(-) rename eth2/types/src/{attestation/mod.rs => attestation.rs} (96%) delete mode 100644 eth2/types/src/attestation/signing.rs diff --git a/eth2/types/src/attestation/mod.rs b/eth2/types/src/attestation.rs similarity index 96% rename from eth2/types/src/attestation/mod.rs rename to eth2/types/src/attestation.rs index 51c396a3b..73ea5eec1 100644 --- a/eth2/types/src/attestation/mod.rs +++ b/eth2/types/src/attestation.rs @@ -5,8 +5,6 @@ use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; -mod signing; - #[derive(Debug, Clone, PartialEq, Serialize)] pub struct Attestation { pub data: AttestationData, @@ -19,6 +17,10 @@ impl Attestation { pub fn canonical_root(&self) -> Hash256 { Hash256::from(&self.hash_tree_root()[..]) } + + pub fn signable_message(&self, custody_bit: bool) -> Vec { + self.data.signable_message(custody_bit) + } } impl Encodable for Attestation { diff --git a/eth2/types/src/attestation/signing.rs b/eth2/types/src/attestation/signing.rs deleted file mode 100644 index 1fefa068c..000000000 --- a/eth2/types/src/attestation/signing.rs +++ /dev/null @@ -1,7 +0,0 @@ -use crate::Attestation; - -impl Attestation { - pub fn signable_message(&self, custody_bit: bool) -> Vec { - self.data.signable_message(custody_bit) - } -} From cbb84c0d6963dac3d4b03a45b009f33c5570a614 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 5 Feb 2019 12:40:03 +1100 Subject: [PATCH 101/107] Move all attestation_data fns into same file --- .../mod.rs => attestation_data.rs} | 13 +++++++++---- eth2/types/src/attestation_data/signing.rs | 12 ------------ 2 files changed, 9 insertions(+), 16 deletions(-) rename eth2/types/src/{attestation_data/mod.rs => attestation_data.rs} (93%) delete mode 100644 eth2/types/src/attestation_data/signing.rs diff --git a/eth2/types/src/attestation_data/mod.rs b/eth2/types/src/attestation_data.rs similarity index 93% rename from eth2/types/src/attestation_data/mod.rs rename to eth2/types/src/attestation_data.rs index 28504127e..d509f6b3d 100644 --- a/eth2/types/src/attestation_data/mod.rs +++ b/eth2/types/src/attestation_data.rs @@ -1,11 +1,8 @@ -use super::Hash256; +use super::{AttestationDataAndCustodyBit, Hash256}; use crate::test_utils::TestRandom; use rand::RngCore; use serde_derive::Serialize; use ssz::{hash, Decodable, DecodeError, Encodable, SszStream, TreeHash}; -use std::hash::Hash; - -mod signing; pub const SSZ_ATTESTION_DATA_LENGTH: usize = { 8 + // slot @@ -51,6 +48,14 @@ impl AttestationData { pub fn canonical_root(&self) -> Hash256 { Hash256::zero() } + + pub fn signable_message(&self, custody_bit: bool) -> Vec { + let attestation_data_and_custody_bit = AttestationDataAndCustodyBit { + data: self.clone(), + custody_bit, + }; + attestation_data_and_custody_bit.hash_tree_root() + } } impl Encodable for AttestationData { diff --git a/eth2/types/src/attestation_data/signing.rs b/eth2/types/src/attestation_data/signing.rs deleted file mode 100644 index f14cc0ef4..000000000 --- a/eth2/types/src/attestation_data/signing.rs +++ /dev/null @@ -1,12 +0,0 @@ -use crate::{AttestationData, AttestationDataAndCustodyBit}; -use ssz::TreeHash; - -impl AttestationData { - pub fn signable_message(&self, custody_bit: bool) -> Vec { - let attestation_data_and_custody_bit = AttestationDataAndCustodyBit { - data: self.clone(), - custody_bit, - }; - attestation_data_and_custody_bit.hash_tree_root() - } -} From 87a326d8a26e82b7ed4fb29cfa50a6fbf8f3a6ff Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 5 Feb 2019 12:41:56 +1100 Subject: [PATCH 102/107] Complete canonical_root for attestation_data --- eth2/types/src/attestation_data.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/eth2/types/src/attestation_data.rs b/eth2/types/src/attestation_data.rs index d509f6b3d..e2140527b 100644 --- a/eth2/types/src/attestation_data.rs +++ b/eth2/types/src/attestation_data.rs @@ -43,10 +43,8 @@ impl AttestationData { } } - // TODO: Implement this as a merkle root, once tree_ssz is implemented. - // https://github.com/sigp/lighthouse/issues/92 pub fn canonical_root(&self) -> Hash256 { - Hash256::zero() + Hash256::from(&self.hash_tree_root()[..]) } pub fn signable_message(&self, custody_bit: bool) -> Vec { From da1498fc45867673ed382a94013db4e409da5675 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 5 Feb 2019 12:48:07 +1100 Subject: [PATCH 103/107] Remove validator_shuffling crate. It has been replaced by the `beacon_state.get_shuffling` function. --- Cargo.toml | 1 - eth2/genesis/Cargo.toml | 1 - eth2/validator_shuffling/Cargo.toml | 10 - eth2/validator_shuffling/src/lib.rs | 3 - eth2/validator_shuffling/src/shuffle.rs | 278 ------------------------ 5 files changed, 293 deletions(-) delete mode 100644 eth2/validator_shuffling/Cargo.toml delete mode 100644 eth2/validator_shuffling/src/lib.rs delete mode 100644 eth2/validator_shuffling/src/shuffle.rs diff --git a/Cargo.toml b/Cargo.toml index 59dfce953..b2efe55ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,6 @@ members = [ "eth2/utils/ssz", "eth2/utils/vec_shuffle", "eth2/validator_induction", - "eth2/validator_shuffling", "beacon_node", "beacon_node/db", "beacon_node/beacon_chain", diff --git a/eth2/genesis/Cargo.toml b/eth2/genesis/Cargo.toml index d56b3f929..09e5d1e88 100644 --- a/eth2/genesis/Cargo.toml +++ b/eth2/genesis/Cargo.toml @@ -9,4 +9,3 @@ bls = { path = "../utils/bls" } ssz = { path = "../utils/ssz" } types = { path = "../types" } validator_induction = { path = "../validator_induction" } -validator_shuffling = { path = "../validator_shuffling" } diff --git a/eth2/validator_shuffling/Cargo.toml b/eth2/validator_shuffling/Cargo.toml deleted file mode 100644 index ae2babf1a..000000000 --- a/eth2/validator_shuffling/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -name = "validator_shuffling" -version = "0.1.0" -authors = ["Paul Hauner "] -edition = "2018" - -[dependencies] -honey-badger-split = { path = "../utils/honey-badger-split" } -types = { path = "../types" } -vec_shuffle = { path = "../utils/vec_shuffle" } diff --git a/eth2/validator_shuffling/src/lib.rs b/eth2/validator_shuffling/src/lib.rs deleted file mode 100644 index 2307dd301..000000000 --- a/eth2/validator_shuffling/src/lib.rs +++ /dev/null @@ -1,3 +0,0 @@ -mod shuffle; - -pub use crate::shuffle::{shard_and_committees_for_cycle, ValidatorAssignmentError}; diff --git a/eth2/validator_shuffling/src/shuffle.rs b/eth2/validator_shuffling/src/shuffle.rs deleted file mode 100644 index 48d16fddc..000000000 --- a/eth2/validator_shuffling/src/shuffle.rs +++ /dev/null @@ -1,278 +0,0 @@ -use std::cmp::min; - -use honey_badger_split::SplitExt; -use types::validator_registry::get_active_validator_indices; -use types::{ChainSpec, ShardCommittee, Validator}; -use vec_shuffle::{shuffle, ShuffleErr}; - -type DelegatedCycle = Vec>; - -#[derive(Debug, PartialEq)] -pub enum ValidatorAssignmentError { - TooManyValidators, - TooFewShards, -} - -/// Delegates active validators into slots for a given cycle, given a random seed. -/// Returns a vector or ShardAndComitte vectors representing the shards and committiees for -/// each slot. -/// References get_new_shuffling (ethereum 2.1 specification) -pub fn shard_and_committees_for_cycle( - seed: &[u8], - validators: &[Validator], - crosslinking_shard_start: u16, - spec: &ChainSpec, -) -> Result { - let shuffled_validator_indices = { - let validator_indices = get_active_validator_indices(validators, 0); - shuffle(seed, validator_indices)? - }; - let shard_indices: Vec = (0_usize..spec.shard_count as usize).into_iter().collect(); - let crosslinking_shard_start = crosslinking_shard_start as usize; - let epoch_length = spec.epoch_length as usize; - let min_committee_size = spec.target_committee_size as usize; - generate_cycle( - &shuffled_validator_indices, - &shard_indices, - crosslinking_shard_start, - epoch_length, - min_committee_size, - ) -} - -/// Given the validator list, delegates the validators into slots and comittees for a given cycle. -fn generate_cycle( - validator_indices: &[usize], - shard_indices: &[usize], - crosslinking_shard_start: usize, - epoch_length: usize, - min_committee_size: usize, -) -> Result { - let validator_count = validator_indices.len(); - let shard_count = shard_indices.len(); - - if shard_count / epoch_length == 0 { - return Err(ValidatorAssignmentError::TooFewShards); - } - - let (committees_per_slot, slots_per_committee) = { - if validator_count >= epoch_length * min_committee_size { - let committees_per_slot = min( - validator_count / epoch_length / (min_committee_size * 2) + 1, - shard_count / epoch_length, - ); - let slots_per_committee = 1; - (committees_per_slot, slots_per_committee) - } else { - let committees_per_slot = 1; - let mut slots_per_committee = 1; - while (validator_count * slots_per_committee < epoch_length * min_committee_size) - & (slots_per_committee < epoch_length) - { - slots_per_committee *= 2; - } - (committees_per_slot, slots_per_committee) - } - }; - - let cycle = validator_indices - .honey_badger_split(epoch_length) - .enumerate() - .map(|(i, slot_indices)| { - let shard_start = - crosslinking_shard_start + i * committees_per_slot / slots_per_committee; - slot_indices - .honey_badger_split(committees_per_slot) - .enumerate() - .map(|(j, shard_indices)| ShardCommittee { - shard: ((shard_start + j) % shard_count) as u64, - committee: shard_indices.to_vec(), - }) - .collect() - }) - .collect(); - Ok(cycle) -} - -impl From for ValidatorAssignmentError { - fn from(e: ShuffleErr) -> ValidatorAssignmentError { - match e { - ShuffleErr::ExceedsListLength => ValidatorAssignmentError::TooManyValidators, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn generate_cycle_helper( - validator_count: &usize, - shard_count: &usize, - crosslinking_shard_start: usize, - epoch_length: usize, - min_committee_size: usize, - ) -> ( - Vec, - Vec, - Result, - ) { - let validator_indices: Vec = (0_usize..*validator_count).into_iter().collect(); - let shard_indices: Vec = (0_usize..*shard_count).into_iter().collect(); - let result = generate_cycle( - &validator_indices, - &shard_indices, - crosslinking_shard_start, - epoch_length, - min_committee_size, - ); - (validator_indices, shard_indices, result) - } - - #[allow(dead_code)] - fn print_cycle(cycle: &DelegatedCycle) { - cycle.iter().enumerate().for_each(|(i, slot)| { - println!("slot {:?}", &i); - slot.iter().enumerate().for_each(|(i, sac)| { - println!( - "#{:?}\tshard={}\tcommittee.len()={}", - &i, - &sac.shard, - &sac.committee.len() - ) - }) - }); - } - - fn flatten_validators(cycle: &DelegatedCycle) -> Vec { - let mut flattened = vec![]; - for slot in cycle.iter() { - for sac in slot.iter() { - for validator in sac.committee.iter() { - flattened.push(*validator); - } - } - } - flattened - } - - fn flatten_and_dedup_shards(cycle: &DelegatedCycle) -> Vec { - let mut flattened = vec![]; - for slot in cycle.iter() { - for sac in slot.iter() { - flattened.push(sac.shard as usize); - } - } - flattened.dedup(); - flattened - } - - fn flatten_shards_in_slots(cycle: &DelegatedCycle) -> Vec> { - let mut shards_in_slots: Vec> = vec![]; - for slot in cycle.iter() { - let mut shards: Vec = vec![]; - for sac in slot.iter() { - shards.push(sac.shard as usize); - } - shards_in_slots.push(shards); - } - shards_in_slots - } - - // TODO: Improve these tests to check committee lengths - #[test] - fn test_generate_cycle() { - let validator_count: usize = 100; - let shard_count: usize = 20; - let crosslinking_shard_start: usize = 0; - let epoch_length: usize = 20; - let min_committee_size: usize = 10; - let (validators, shards, result) = generate_cycle_helper( - &validator_count, - &shard_count, - crosslinking_shard_start, - epoch_length, - min_committee_size, - ); - let cycle = result.unwrap(); - - let assigned_validators = flatten_validators(&cycle); - let assigned_shards = flatten_and_dedup_shards(&cycle); - let shards_in_slots = flatten_shards_in_slots(&cycle); - let expected_shards = shards.get(0..10).unwrap(); - assert_eq!( - assigned_validators, validators, - "Validator assignment incorrect" - ); - assert_eq!( - assigned_shards, expected_shards, - "Shard assignment incorrect" - ); - - let expected_shards_in_slots: Vec> = vec![ - vec![0], - vec![0], // Each line is 2 slots.. - vec![1], - vec![1], - vec![2], - vec![2], - vec![3], - vec![3], - vec![4], - vec![4], - vec![5], - vec![5], - vec![6], - vec![6], - vec![7], - vec![7], - vec![8], - vec![8], - vec![9], - vec![9], - ]; - // assert!(compare_shards_in_slots(&cycle, &expected_shards_in_slots)); - assert_eq!( - expected_shards_in_slots, shards_in_slots, - "Shard assignment incorrect." - ) - } - - #[test] - // Check that the committees per slot is upper bounded by shard count - fn test_generate_cycle_committees_bounded() { - let validator_count: usize = 523; - let shard_count: usize = 31; - let crosslinking_shard_start: usize = 0; - let epoch_length: usize = 11; - let min_committee_size: usize = 5; - let (validators, shards, result) = generate_cycle_helper( - &validator_count, - &shard_count, - crosslinking_shard_start, - epoch_length, - min_committee_size, - ); - let cycle = result.unwrap(); - let assigned_validators = flatten_validators(&cycle); - let assigned_shards = flatten_and_dedup_shards(&cycle); - let shards_in_slots = flatten_shards_in_slots(&cycle); - let expected_shards = shards.get(0..22).unwrap(); - let expected_shards_in_slots: Vec> = (0_usize..11_usize) - .map(|x| vec![2 * x, 2 * x + 1]) - .collect(); - assert_eq!( - assigned_validators, validators, - "Validator assignment incorrect" - ); - assert_eq!( - assigned_shards, expected_shards, - "Shard assignment incorrect" - ); - // assert!(compare_shards_in_slots(&cycle, &expected_shards_in_slots)); - assert_eq!( - expected_shards_in_slots, shards_in_slots, - "Shard assignment incorrect." - ) - } -} From af50c28e0f593295bd6ef3b919c2fc6b0e704e74 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 5 Feb 2019 16:22:07 +1100 Subject: [PATCH 104/107] Improve naming consistency in `test_harness` With respect to filenames. Especially removing Test.. prefixes --- .../src/{harness.rs => beacon_chain_harness.rs} | 12 +++++++----- beacon_node/beacon_chain/test_harness/src/lib.rs | 8 ++++---- .../test_harness/src/validator/mod.rs | 6 ------ .../direct_beacon_node.rs | 0 .../direct_duties.rs | 0 .../local_signer.rs} | 8 ++++---- .../test_harness/src/validator_harness/mod.rs | 6 ++++++ .../validator_harness.rs} | 16 ++++++++-------- 8 files changed, 29 insertions(+), 27 deletions(-) rename beacon_node/beacon_chain/test_harness/src/{harness.rs => beacon_chain_harness.rs} (95%) delete mode 100644 beacon_node/beacon_chain/test_harness/src/validator/mod.rs rename beacon_node/beacon_chain/test_harness/src/{validator => validator_harness}/direct_beacon_node.rs (100%) rename beacon_node/beacon_chain/test_harness/src/{validator => validator_harness}/direct_duties.rs (100%) rename beacon_node/beacon_chain/test_harness/src/{validator/signer.rs => validator_harness/local_signer.rs} (90%) create mode 100644 beacon_node/beacon_chain/test_harness/src/validator_harness/mod.rs rename beacon_node/beacon_chain/test_harness/src/{validator/validator.rs => validator_harness/validator_harness.rs} (92%) diff --git a/beacon_node/beacon_chain/test_harness/src/harness.rs b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs similarity index 95% rename from beacon_node/beacon_chain/test_harness/src/harness.rs rename to beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs index 12ed40755..64d18b4be 100644 --- a/beacon_node/beacon_chain/test_harness/src/harness.rs +++ b/beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs @@ -1,4 +1,4 @@ -use super::TestValidator; +use super::ValidatorHarness; use beacon_chain::BeaconChain; pub use beacon_chain::{CheckPoint, Error as BeaconChainError}; use db::{ @@ -26,7 +26,7 @@ pub struct BeaconChainHarness { pub beacon_chain: Arc>, pub block_store: Arc>, pub state_store: Arc>, - pub validators: Vec, + pub validators: Vec, pub spec: Arc, } @@ -91,12 +91,14 @@ impl BeaconChainHarness { debug!("Creating validator producer and attester instances..."); // Spawn the test validator instances. - let validators: Vec = keypairs + let validators: Vec = keypairs .iter() - .map(|keypair| TestValidator::new(keypair.clone(), beacon_chain.clone(), spec.clone())) + .map(|keypair| { + ValidatorHarness::new(keypair.clone(), beacon_chain.clone(), spec.clone()) + }) .collect(); - debug!("Created {} TestValidators", validators.len()); + debug!("Created {} ValidatorHarnesss", validators.len()); Self { db, diff --git a/beacon_node/beacon_chain/test_harness/src/lib.rs b/beacon_node/beacon_chain/test_harness/src/lib.rs index 71f9a8855..b04fc6996 100644 --- a/beacon_node/beacon_chain/test_harness/src/lib.rs +++ b/beacon_node/beacon_chain/test_harness/src/lib.rs @@ -1,5 +1,5 @@ -mod harness; -mod validator; +mod beacon_chain_harness; +mod validator_harness; -pub use self::harness::BeaconChainHarness; -pub use self::validator::TestValidator; +pub use self::beacon_chain_harness::BeaconChainHarness; +pub use self::validator_harness::ValidatorHarness; diff --git a/beacon_node/beacon_chain/test_harness/src/validator/mod.rs b/beacon_node/beacon_chain/test_harness/src/validator/mod.rs deleted file mode 100644 index 3a0d53395..000000000 --- a/beacon_node/beacon_chain/test_harness/src/validator/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -mod direct_beacon_node; -mod direct_duties; -mod signer; -mod validator; - -pub use self::validator::TestValidator; diff --git a/beacon_node/beacon_chain/test_harness/src/validator/direct_beacon_node.rs b/beacon_node/beacon_chain/test_harness/src/validator_harness/direct_beacon_node.rs similarity index 100% rename from beacon_node/beacon_chain/test_harness/src/validator/direct_beacon_node.rs rename to beacon_node/beacon_chain/test_harness/src/validator_harness/direct_beacon_node.rs diff --git a/beacon_node/beacon_chain/test_harness/src/validator/direct_duties.rs b/beacon_node/beacon_chain/test_harness/src/validator_harness/direct_duties.rs similarity index 100% rename from beacon_node/beacon_chain/test_harness/src/validator/direct_duties.rs rename to beacon_node/beacon_chain/test_harness/src/validator_harness/direct_duties.rs diff --git a/beacon_node/beacon_chain/test_harness/src/validator/signer.rs b/beacon_node/beacon_chain/test_harness/src/validator_harness/local_signer.rs similarity index 90% rename from beacon_node/beacon_chain/test_harness/src/validator/signer.rs rename to beacon_node/beacon_chain/test_harness/src/validator_harness/local_signer.rs index 6e63c7f99..8e901b057 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/signer.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator_harness/local_signer.rs @@ -4,12 +4,12 @@ use std::sync::RwLock; use types::{Keypair, Signature}; /// A test-only struct used to perform signing for a proposer or attester. -pub struct TestSigner { +pub struct LocalSigner { keypair: Keypair, should_sign: RwLock, } -impl TestSigner { +impl LocalSigner { /// Produce a new TestSigner with signing enabled by default. pub fn new(keypair: Keypair) -> Self { Self { @@ -30,7 +30,7 @@ impl TestSigner { } } -impl BlockProposerSigner for TestSigner { +impl BlockProposerSigner for LocalSigner { fn sign_block_proposal(&self, message: &[u8]) -> Option { self.bls_sign(message) } @@ -40,7 +40,7 @@ impl BlockProposerSigner for TestSigner { } } -impl AttesterSigner for TestSigner { +impl AttesterSigner for LocalSigner { fn sign_attestation_message(&self, message: &[u8]) -> Option { self.bls_sign(message) } diff --git a/beacon_node/beacon_chain/test_harness/src/validator_harness/mod.rs b/beacon_node/beacon_chain/test_harness/src/validator_harness/mod.rs new file mode 100644 index 000000000..837334ade --- /dev/null +++ b/beacon_node/beacon_chain/test_harness/src/validator_harness/mod.rs @@ -0,0 +1,6 @@ +mod direct_beacon_node; +mod direct_duties; +mod local_signer; +mod validator_harness; + +pub use self::validator_harness::ValidatorHarness; diff --git a/beacon_node/beacon_chain/test_harness/src/validator/validator.rs b/beacon_node/beacon_chain/test_harness/src/validator_harness/validator_harness.rs similarity index 92% rename from beacon_node/beacon_chain/test_harness/src/validator/validator.rs rename to beacon_node/beacon_chain/test_harness/src/validator_harness/validator_harness.rs index d33a0412b..986d843bb 100644 --- a/beacon_node/beacon_chain/test_harness/src/validator/validator.rs +++ b/beacon_node/beacon_chain/test_harness/src/validator_harness/validator_harness.rs @@ -1,6 +1,6 @@ use super::direct_beacon_node::DirectBeaconNode; use super::direct_duties::DirectDuties; -use super::signer::TestSigner; +use super::local_signer::LocalSigner; use attester::PollOutcome as AttestationPollOutcome; use attester::{Attester, Error as AttestationPollError}; use beacon_chain::BeaconChain; @@ -28,29 +28,29 @@ pub enum AttestationProduceError { /// The test validator connects directly to a borrowed `BeaconChain` struct. It is useful for /// testing that the core proposer and attester logic is functioning. Also for supporting beacon /// chain tests. -pub struct TestValidator { +pub struct ValidatorHarness { pub block_producer: BlockProducer< TestingSlotClock, DirectBeaconNode, DirectDuties, - TestSigner, + LocalSigner, >, pub attester: Attester< TestingSlotClock, DirectBeaconNode, DirectDuties, - TestSigner, + LocalSigner, >, pub spec: Arc, pub epoch_map: Arc>, pub keypair: Keypair, pub beacon_node: Arc>, pub slot_clock: Arc, - pub signer: Arc, + pub signer: Arc, } -impl TestValidator { - /// Create a new TestValidator that signs with the given keypair, operates per the given spec and connects to the +impl ValidatorHarness { + /// Create a new ValidatorHarness that signs with the given keypair, operates per the given spec and connects to the /// supplied beacon node. /// /// A `BlockProducer` and `Attester` is created.. @@ -60,7 +60,7 @@ impl TestValidator { spec: Arc, ) -> Self { let slot_clock = Arc::new(TestingSlotClock::new(spec.genesis_slot)); - let signer = Arc::new(TestSigner::new(keypair.clone())); + let signer = Arc::new(LocalSigner::new(keypair.clone())); let beacon_node = Arc::new(DirectBeaconNode::new(beacon_chain.clone())); let epoch_map = Arc::new(DirectDuties::new(keypair.pk.clone(), beacon_chain.clone())); From 49dcb38c31365da0428637c83be9a7ee06d89eb5 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 5 Feb 2019 16:41:18 +1100 Subject: [PATCH 105/107] Fix naming consistency with `attester` crate. Adjusted naming of files to ensure they match the name of the struct. Also change the name of some structs so they don't look like tests. --- eth2/attester/src/lib.rs | 8 ++++---- eth2/attester/src/test_utils/epoch_map.rs | 6 +++--- .../src/test_utils/{signer.rs => local_signer.rs} | 8 ++++---- eth2/attester/src/test_utils/mod.rs | 10 +++++----- .../{beacon_node.rs => simulated_beacon_node.rs} | 6 +++--- 5 files changed, 19 insertions(+), 19 deletions(-) rename eth2/attester/src/test_utils/{signer.rs => local_signer.rs} (83%) rename eth2/attester/src/test_utils/{beacon_node.rs => simulated_beacon_node.rs} (93%) diff --git a/eth2/attester/src/lib.rs b/eth2/attester/src/lib.rs index 447bcb9ea..4dda24570 100644 --- a/eth2/attester/src/lib.rs +++ b/eth2/attester/src/lib.rs @@ -170,7 +170,7 @@ impl From for Error { #[cfg(test)] mod tests { - use super::test_utils::{TestBeaconNode, TestEpochMap, TestSigner}; + use super::test_utils::{EpochMap, LocalSigner, SimulatedBeaconNode}; use super::*; use slot_clock::TestingSlotClock; use types::{ @@ -189,10 +189,10 @@ mod tests { let spec = Arc::new(ChainSpec::foundation()); let slot_clock = Arc::new(TestingSlotClock::new(0)); - let beacon_node = Arc::new(TestBeaconNode::default()); - let signer = Arc::new(TestSigner::new(Keypair::random())); + let beacon_node = Arc::new(SimulatedBeaconNode::default()); + let signer = Arc::new(LocalSigner::new(Keypair::random())); - let mut duties = TestEpochMap::new(spec.epoch_length); + let mut duties = EpochMap::new(spec.epoch_length); let attest_slot = 100; let attest_epoch = attest_slot / spec.epoch_length; let attest_shard = 12; diff --git a/eth2/attester/src/test_utils/epoch_map.rs b/eth2/attester/src/test_utils/epoch_map.rs index 9f44ecf1a..88e36c93c 100644 --- a/eth2/attester/src/test_utils/epoch_map.rs +++ b/eth2/attester/src/test_utils/epoch_map.rs @@ -1,13 +1,13 @@ use crate::{DutiesReader, DutiesReaderError}; use std::collections::HashMap; -pub struct TestEpochMap { +pub struct EpochMap { epoch_length: u64, validator_index: Option, map: HashMap, } -impl TestEpochMap { +impl EpochMap { pub fn new(epoch_length: u64) -> Self { Self { epoch_length, @@ -27,7 +27,7 @@ impl TestEpochMap { } } -impl DutiesReader for TestEpochMap { +impl DutiesReader for EpochMap { fn attestation_shard(&self, slot: u64) -> Result, DutiesReaderError> { let epoch = slot / self.epoch_length; diff --git a/eth2/attester/src/test_utils/signer.rs b/eth2/attester/src/test_utils/local_signer.rs similarity index 83% rename from eth2/attester/src/test_utils/signer.rs rename to eth2/attester/src/test_utils/local_signer.rs index 185dec278..c256d1050 100644 --- a/eth2/attester/src/test_utils/signer.rs +++ b/eth2/attester/src/test_utils/local_signer.rs @@ -3,13 +3,13 @@ use std::sync::RwLock; use types::{Keypair, Signature}; /// A test-only struct used to simulate a Beacon Node. -pub struct TestSigner { +pub struct LocalSigner { keypair: Keypair, should_sign: RwLock, } -impl TestSigner { - /// Produce a new TestSigner with signing enabled by default. +impl LocalSigner { + /// Produce a new LocalSigner with signing enabled by default. pub fn new(keypair: Keypair) -> Self { Self { keypair, @@ -24,7 +24,7 @@ impl TestSigner { } } -impl Signer for TestSigner { +impl Signer for LocalSigner { fn sign_attestation_message(&self, message: &[u8]) -> Option { Some(Signature::new(message, &self.keypair.sk)) } diff --git a/eth2/attester/src/test_utils/mod.rs b/eth2/attester/src/test_utils/mod.rs index 0dd384b12..481247dd0 100644 --- a/eth2/attester/src/test_utils/mod.rs +++ b/eth2/attester/src/test_utils/mod.rs @@ -1,7 +1,7 @@ -mod beacon_node; mod epoch_map; -mod signer; +mod local_signer; +mod simulated_beacon_node; -pub use self::beacon_node::TestBeaconNode; -pub use self::epoch_map::TestEpochMap; -pub use self::signer::TestSigner; +pub use self::epoch_map::EpochMap; +pub use self::local_signer::LocalSigner; +pub use self::simulated_beacon_node::SimulatedBeaconNode; diff --git a/eth2/attester/src/test_utils/beacon_node.rs b/eth2/attester/src/test_utils/simulated_beacon_node.rs similarity index 93% rename from eth2/attester/src/test_utils/beacon_node.rs rename to eth2/attester/src/test_utils/simulated_beacon_node.rs index 5b4061c60..2f14c9fd3 100644 --- a/eth2/attester/src/test_utils/beacon_node.rs +++ b/eth2/attester/src/test_utils/simulated_beacon_node.rs @@ -7,7 +7,7 @@ type PublishResult = Result; /// A test-only struct used to simulate a Beacon Node. #[derive(Default)] -pub struct TestBeaconNode { +pub struct SimulatedBeaconNode { pub produce_input: RwLock>, pub produce_result: RwLock>, @@ -15,7 +15,7 @@ pub struct TestBeaconNode { pub publish_result: RwLock>, } -impl TestBeaconNode { +impl SimulatedBeaconNode { pub fn set_next_produce_result(&self, result: ProduceResult) { *self.produce_result.write().unwrap() = Some(result); } @@ -25,7 +25,7 @@ impl TestBeaconNode { } } -impl BeaconNode for TestBeaconNode { +impl BeaconNode for SimulatedBeaconNode { fn produce_attestation_data(&self, slot: u64, shard: u64) -> ProduceResult { *self.produce_input.write().unwrap() = Some((slot, shard)); match *self.produce_result.read().unwrap() { From 6c6ee4320da1e06b94ecf4e30aed52cce5a75773 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 5 Feb 2019 16:47:59 +1100 Subject: [PATCH 106/107] Fix `block_producer` naming to match `attester` --- eth2/block_producer/src/lib.rs | 8 ++++---- eth2/block_producer/src/test_utils/epoch_map.rs | 6 +++--- .../src/test_utils/{signer.rs => local_signer.rs} | 8 ++++---- eth2/block_producer/src/test_utils/mod.rs | 10 +++++----- .../{beacon_node.rs => simulated_beacon_node.rs} | 12 ++++++------ validator_client/src/main.rs | 4 ++-- 6 files changed, 24 insertions(+), 24 deletions(-) rename eth2/block_producer/src/test_utils/{signer.rs => local_signer.rs} (85%) rename eth2/block_producer/src/test_utils/{beacon_node.rs => simulated_beacon_node.rs} (87%) diff --git a/eth2/block_producer/src/lib.rs b/eth2/block_producer/src/lib.rs index ed65622d4..8ed10ce1a 100644 --- a/eth2/block_producer/src/lib.rs +++ b/eth2/block_producer/src/lib.rs @@ -213,7 +213,7 @@ impl From for Error { #[cfg(test)] mod tests { - use super::test_utils::{TestBeaconNode, TestEpochMap, TestSigner}; + use super::test_utils::{EpochMap, LocalSigner, SimulatedBeaconNode}; use super::*; use slot_clock::TestingSlotClock; use types::{ @@ -232,10 +232,10 @@ mod tests { let spec = Arc::new(ChainSpec::foundation()); let slot_clock = Arc::new(TestingSlotClock::new(0)); - let beacon_node = Arc::new(TestBeaconNode::default()); - let signer = Arc::new(TestSigner::new(Keypair::random())); + let beacon_node = Arc::new(SimulatedBeaconNode::default()); + let signer = Arc::new(LocalSigner::new(Keypair::random())); - let mut epoch_map = TestEpochMap::new(spec.epoch_length); + let mut epoch_map = EpochMap::new(spec.epoch_length); let produce_slot = 100; let produce_epoch = produce_slot / spec.epoch_length; epoch_map.map.insert(produce_epoch, produce_slot); diff --git a/eth2/block_producer/src/test_utils/epoch_map.rs b/eth2/block_producer/src/test_utils/epoch_map.rs index 848ae252e..c7d1e6b4c 100644 --- a/eth2/block_producer/src/test_utils/epoch_map.rs +++ b/eth2/block_producer/src/test_utils/epoch_map.rs @@ -1,12 +1,12 @@ use crate::{DutiesReader, DutiesReaderError}; use std::collections::HashMap; -pub struct TestEpochMap { +pub struct EpochMap { epoch_length: u64, pub map: HashMap, } -impl TestEpochMap { +impl EpochMap { pub fn new(epoch_length: u64) -> Self { Self { epoch_length, @@ -15,7 +15,7 @@ impl TestEpochMap { } } -impl DutiesReader for TestEpochMap { +impl DutiesReader for EpochMap { fn is_block_production_slot(&self, slot: u64) -> Result { let epoch = slot / self.epoch_length; match self.map.get(&epoch) { diff --git a/eth2/block_producer/src/test_utils/signer.rs b/eth2/block_producer/src/test_utils/local_signer.rs similarity index 85% rename from eth2/block_producer/src/test_utils/signer.rs rename to eth2/block_producer/src/test_utils/local_signer.rs index 62163f5b2..0ebefa29d 100644 --- a/eth2/block_producer/src/test_utils/signer.rs +++ b/eth2/block_producer/src/test_utils/local_signer.rs @@ -3,13 +3,13 @@ use std::sync::RwLock; use types::{Keypair, Signature}; /// A test-only struct used to simulate a Beacon Node. -pub struct TestSigner { +pub struct LocalSigner { keypair: Keypair, should_sign: RwLock, } -impl TestSigner { - /// Produce a new TestSigner with signing enabled by default. +impl LocalSigner { + /// Produce a new LocalSigner with signing enabled by default. pub fn new(keypair: Keypair) -> Self { Self { keypair, @@ -24,7 +24,7 @@ impl TestSigner { } } -impl Signer for TestSigner { +impl Signer for LocalSigner { fn sign_block_proposal(&self, message: &[u8]) -> Option { Some(Signature::new(message, &self.keypair.sk)) } diff --git a/eth2/block_producer/src/test_utils/mod.rs b/eth2/block_producer/src/test_utils/mod.rs index 0dd384b12..481247dd0 100644 --- a/eth2/block_producer/src/test_utils/mod.rs +++ b/eth2/block_producer/src/test_utils/mod.rs @@ -1,7 +1,7 @@ -mod beacon_node; mod epoch_map; -mod signer; +mod local_signer; +mod simulated_beacon_node; -pub use self::beacon_node::TestBeaconNode; -pub use self::epoch_map::TestEpochMap; -pub use self::signer::TestSigner; +pub use self::epoch_map::EpochMap; +pub use self::local_signer::LocalSigner; +pub use self::simulated_beacon_node::SimulatedBeaconNode; diff --git a/eth2/block_producer/src/test_utils/beacon_node.rs b/eth2/block_producer/src/test_utils/simulated_beacon_node.rs similarity index 87% rename from eth2/block_producer/src/test_utils/beacon_node.rs rename to eth2/block_producer/src/test_utils/simulated_beacon_node.rs index f132ce6ec..772670a12 100644 --- a/eth2/block_producer/src/test_utils/beacon_node.rs +++ b/eth2/block_producer/src/test_utils/simulated_beacon_node.rs @@ -8,7 +8,7 @@ type PublishResult = Result; /// A test-only struct used to simulate a Beacon Node. #[derive(Default)] -pub struct TestBeaconNode { +pub struct SimulatedBeaconNode { pub nonce_input: RwLock>, pub nonce_result: RwLock>, @@ -19,7 +19,7 @@ pub struct TestBeaconNode { pub publish_result: RwLock>, } -impl TestBeaconNode { +impl SimulatedBeaconNode { /// Set the result to be returned when `produce_beacon_block` is called. pub fn set_next_nonce_result(&self, result: NonceResult) { *self.nonce_result.write().unwrap() = Some(result); @@ -36,12 +36,12 @@ impl TestBeaconNode { } } -impl BeaconNode for TestBeaconNode { +impl BeaconNode for SimulatedBeaconNode { fn proposer_nonce(&self, pubkey: &PublicKey) -> NonceResult { *self.nonce_input.write().unwrap() = Some(pubkey.clone()); match *self.nonce_result.read().unwrap() { Some(ref r) => r.clone(), - None => panic!("TestBeaconNode: nonce_result == None"), + None => panic!("SimulatedBeaconNode: nonce_result == None"), } } @@ -50,7 +50,7 @@ impl BeaconNode for TestBeaconNode { *self.produce_input.write().unwrap() = Some((slot, randao_reveal.clone())); match *self.produce_result.read().unwrap() { Some(ref r) => r.clone(), - None => panic!("TestBeaconNode: produce_result == None"), + None => panic!("SimulatedBeaconNode: produce_result == None"), } } @@ -59,7 +59,7 @@ impl BeaconNode for TestBeaconNode { *self.publish_input.write().unwrap() = Some(block); match *self.publish_result.read().unwrap() { Some(ref r) => r.clone(), - None => panic!("TestBeaconNode: publish_result == None"), + None => panic!("SimulatedBeaconNode: publish_result == None"), } } } diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index 3b516870a..0e1fd4a6b 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -1,7 +1,7 @@ use self::block_producer_service::{BeaconBlockGrpcClient, BlockProducerService}; use self::duties::{DutiesManager, DutiesManagerService, EpochDutiesMap}; use crate::config::ClientConfig; -use block_producer::{test_utils::TestSigner, BlockProducer}; +use block_producer::{test_utils::LocalSigner, BlockProducer}; use bls::Keypair; use clap::{App, Arg}; use grpcio::{ChannelBuilder, EnvBuilder}; @@ -140,7 +140,7 @@ fn main() { let producer_thread = { let spec = spec.clone(); let pubkey = keypair.pk.clone(); - let signer = Arc::new(TestSigner::new(keypair.clone())); + let signer = Arc::new(LocalSigner::new(keypair.clone())); let duties_map = duties_map.clone(); let slot_clock = slot_clock.clone(); let log = log.clone(); From 36ee4aa81f493cb0b5e0ae173fa9c3bb87ce157e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 5 Feb 2019 16:53:44 +1100 Subject: [PATCH 107/107] Fix file naming inconsistencies. File names should be the same as the struct that they contain. --- .../{grpc.rs => beacon_block_grpc_client.rs} | 0 .../{service.rs => block_producer_service.rs} | 0 validator_client/src/block_producer_service/mod.rs | 8 ++++---- .../src/duties/{duties_map.rs => epoch_duties.rs} | 0 validator_client/src/duties/mod.rs | 6 +++--- 5 files changed, 7 insertions(+), 7 deletions(-) rename validator_client/src/block_producer_service/{grpc.rs => beacon_block_grpc_client.rs} (100%) rename validator_client/src/block_producer_service/{service.rs => block_producer_service.rs} (100%) rename validator_client/src/duties/{duties_map.rs => epoch_duties.rs} (100%) diff --git a/validator_client/src/block_producer_service/grpc.rs b/validator_client/src/block_producer_service/beacon_block_grpc_client.rs similarity index 100% rename from validator_client/src/block_producer_service/grpc.rs rename to validator_client/src/block_producer_service/beacon_block_grpc_client.rs diff --git a/validator_client/src/block_producer_service/service.rs b/validator_client/src/block_producer_service/block_producer_service.rs similarity index 100% rename from validator_client/src/block_producer_service/service.rs rename to validator_client/src/block_producer_service/block_producer_service.rs diff --git a/validator_client/src/block_producer_service/mod.rs b/validator_client/src/block_producer_service/mod.rs index a4ddf760a..52aac688b 100644 --- a/validator_client/src/block_producer_service/mod.rs +++ b/validator_client/src/block_producer_service/mod.rs @@ -1,5 +1,5 @@ -mod grpc; -mod service; +mod beacon_block_grpc_client; +mod block_producer_service; -pub use self::grpc::BeaconBlockGrpcClient; -pub use self::service::BlockProducerService; +pub use self::beacon_block_grpc_client::BeaconBlockGrpcClient; +pub use self::block_producer_service::BlockProducerService; diff --git a/validator_client/src/duties/duties_map.rs b/validator_client/src/duties/epoch_duties.rs similarity index 100% rename from validator_client/src/duties/duties_map.rs rename to validator_client/src/duties/epoch_duties.rs diff --git a/validator_client/src/duties/mod.rs b/validator_client/src/duties/mod.rs index c9ebda249..711a1e29c 100644 --- a/validator_client/src/duties/mod.rs +++ b/validator_client/src/duties/mod.rs @@ -1,12 +1,12 @@ -mod duties_map; +mod epoch_duties; mod grpc; mod service; #[cfg(test)] mod test_node; mod traits; -pub use self::duties_map::EpochDutiesMap; -use self::duties_map::{EpochDuties, EpochDutiesMapError}; +pub use self::epoch_duties::EpochDutiesMap; +use self::epoch_duties::{EpochDuties, EpochDutiesMapError}; pub use self::service::DutiesManagerService; use self::traits::{BeaconNode, BeaconNodeError}; use bls::PublicKey;