diff --git a/beacon_node/beacon_chain/test_harness/src/bin.rs b/beacon_node/beacon_chain/test_harness/src/bin.rs index b6a0529c7..5411efc4a 100644 --- a/beacon_node/beacon_chain/test_harness/src/bin.rs +++ b/beacon_node/beacon_chain/test_harness/src/bin.rs @@ -37,6 +37,18 @@ fn main() { }; for doc in &docs { + // For each `test_cases` YAML in the document, build a `Manifest`, execute it and + // assert that the execution result matches the manifest description. + // + // In effect, for each `test_case` a new `BeaconChainHarness` is created from genesis + // and a new `BeaconChain` is built as per the manifest. + // + // After the `BeaconChain` has been built out as per the manifest, a dump of all blocks + // and states in the chain is obtained and checked against the `results` specified in + // the `test_case`. + // + // If any of the expectations in the results are not met, the process + // panics with a message. for test_case in doc["test_cases"].as_vec().unwrap() { let manifest = Manifest::from_yaml(test_case); manifest.assert_result_valid(manifest.execute()) diff --git a/beacon_node/beacon_chain/test_harness/src/lib.rs b/beacon_node/beacon_chain/test_harness/src/lib.rs index b04fc6996..a7ada2433 100644 --- a/beacon_node/beacon_chain/test_harness/src/lib.rs +++ b/beacon_node/beacon_chain/test_harness/src/lib.rs @@ -1,4 +1,32 @@ +//! Provides a testing environment for the `BeaconChain`, `Attester` and `BlockProposer` objects. +//! +//! This environment bypasses networking client runtimes and connects the `Attester` and `Proposer` +//! directly to the `BeaconChain` via an `Arc`. +//! +//! The `BeaconChainHarness` contains a single `BeaconChain` instance and many `ValidatorHarness` +//! instances. All of the `ValidatorHarness` instances work to advance the `BeaconChain` by +//! producing blocks and attestations. +//! +//! Example: +//! ``` +//! use test_harness::BeaconChainHarness; +//! use types::ChainSpec; +//! +//! let validator_count = 8; +//! let spec = ChainSpec::few_validators(); +//! +//! let mut harness = BeaconChainHarness::new(spec, validator_count); +//! +//! harness.advance_chain_with_block(); +//! +//! let chain = harness.chain_dump().unwrap(); +//! +//! // One block should have been built on top of the genesis block. +//! assert_eq!(chain.len(), 2); +//! ``` + mod beacon_chain_harness; +pub mod manifest; mod validator_harness; pub use self::beacon_chain_harness::BeaconChainHarness; diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/config.rs b/beacon_node/beacon_chain/test_harness/src/manifest/config.rs index be01d48d8..8c88ee5d1 100644 --- a/beacon_node/beacon_chain/test_harness/src/manifest/config.rs +++ b/beacon_node/beacon_chain/test_harness/src/manifest/config.rs @@ -7,18 +7,29 @@ pub type DepositTuple = (u64, Deposit, Keypair); pub type ProposerSlashingTuple = (u64, u64); pub type AttesterSlashingTuple = (u64, Vec); +/// Defines the execution of a `BeaconStateHarness` across a series of slots. #[derive(Debug)] pub struct Config { + /// Initial validators. pub deposits_for_chain_start: usize, + /// Number of slots in an epoch. pub epoch_length: Option, + /// Number of slots to build before ending execution. pub num_slots: u64, + /// Number of slots that should be skipped due to inactive validator. pub skip_slots: Option>, + /// Deposits to be included during execution. pub deposits: Option>, + /// Proposer slashings to be included during execution. pub proposer_slashings: Option>, + /// Attester slashings to be including during execution. pub attester_slashings: Option>, } impl Config { + /// Load from a YAML document. + /// + /// Expects to receive the `config` section of the document. pub fn from_yaml(yaml: &Yaml) -> Self { Self { deposits_for_chain_start: as_usize(&yaml, "deposits_for_chain_start") @@ -33,6 +44,7 @@ impl Config { } } +/// Parse the `attester_slashings` section of the YAML document. fn parse_attester_slashings(yaml: &Yaml) -> Option> { let mut slashings = vec![]; @@ -47,6 +59,7 @@ fn parse_attester_slashings(yaml: &Yaml) -> Option> { Some(slashings) } +/// Parse the `proposer_slashings` section of the YAML document. fn parse_proposer_slashings(yaml: &Yaml) -> Option> { let mut slashings = vec![]; @@ -61,6 +74,7 @@ fn parse_proposer_slashings(yaml: &Yaml) -> Option> { Some(slashings) } +/// Parse the `deposits` section of the YAML document. fn parse_deposits(yaml: &Yaml) -> Option> { let mut deposits = vec![]; diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/mod.rs b/beacon_node/beacon_chain/test_harness/src/manifest/mod.rs index 4b8385035..7b9c9cb02 100644 --- a/beacon_node/beacon_chain/test_harness/src/manifest/mod.rs +++ b/beacon_node/beacon_chain/test_harness/src/manifest/mod.rs @@ -1,5 +1,6 @@ -use self::config::Config; -use self::results::Results; +//! Defines execution and testing specs for a `BeaconChainHarness` instance. Supports loading from +//! a YAML file. + use crate::beacon_chain_harness::BeaconChainHarness; use beacon_chain::CheckPoint; use log::{info, warn}; @@ -14,18 +15,36 @@ mod results; mod state_check; mod yaml_helpers; +pub use config::Config; +pub use results::Results; +pub use state_check::StateCheck; + +/// Defines the execution and testing of a `BeaconChainHarness` instantiation. +/// +/// Typical workflow is: +/// +/// 1. Instantiate the `Manifest` from YAML: `let manifest = Manifest::from_yaml(&my_yaml);` +/// 2. Execute the manifest: `let result = manifest.execute();` +/// 3. Test the results against the manifest: `manifest.assert_result_valid(result);` #[derive(Debug)] pub struct Manifest { - pub results: Results, + /// Defines the execution. pub config: Config, + /// Defines tests to run against the execution result. + pub results: Results, } +/// The result of executing a `Manifest`. +/// pub struct ExecutionResult { + /// The canonical beacon chain generated from the execution. pub chain: Vec, + /// The spec used for execution. pub spec: ChainSpec, } impl Manifest { + /// Load the manifest from a YAML document. pub fn from_yaml(test_case: &Yaml) -> Self { Self { results: Results::from_yaml(&test_case["results"]), @@ -33,6 +52,9 @@ impl Manifest { } } + /// Return a `ChainSpec::foundation()`. + /// + /// If specified in `config`, returns it with a modified `epoch_length`. fn spec(&self) -> ChainSpec { let mut spec = ChainSpec::foundation(); @@ -43,6 +65,7 @@ impl Manifest { spec } + /// Executes the manifest, returning an `ExecutionResult`. pub fn execute(&self) -> ExecutionResult { let spec = self.spec(); let validator_count = self.config.deposits_for_chain_start; @@ -123,6 +146,11 @@ impl Manifest { } } + /// Checks that the `ExecutionResult` is consistent with the specifications in `self.results`. + /// + /// # Panics + /// + /// Panics with a message if any result does not match exepectations. pub fn assert_result_valid(&self, execution_result: ExecutionResult) { info!("Verifying test results..."); let spec = &execution_result.spec; @@ -157,6 +185,9 @@ impl Manifest { } } +/// Builds an `AttesterSlashing` for some `validator_indices`. +/// +/// Signs the message using a `BeaconChainHarness`. fn build_double_vote_attester_slashing( harness: &BeaconChainHarness, validator_indices: &[u64], @@ -170,6 +201,9 @@ fn build_double_vote_attester_slashing( AttesterSlashingBuilder::double_vote(validator_indices, signer, &harness.spec) } +/// Builds an `ProposerSlashing` for some `validator_index`. +/// +/// Signs the message using a `BeaconChainHarness`. fn build_proposer_slashing(harness: &BeaconChainHarness, validator_index: u64) -> ProposerSlashing { let signer = |validator_index: u64, message: &[u8], epoch: Epoch, domain: u64| { harness diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/results.rs b/beacon_node/beacon_chain/test_harness/src/manifest/results.rs index 844695910..d16c91874 100644 --- a/beacon_node/beacon_chain/test_harness/src/manifest/results.rs +++ b/beacon_node/beacon_chain/test_harness/src/manifest/results.rs @@ -2,6 +2,8 @@ use super::state_check::StateCheck; use super::yaml_helpers::as_usize; use yaml_rust::Yaml; +/// A series of tests to be carried out upon an `ExecutionResult`, returned from executing a +/// `Manifest`. #[derive(Debug)] pub struct Results { pub num_skipped_slots: Option, @@ -9,6 +11,9 @@ pub struct Results { } impl Results { + /// Load from a YAML document. + /// + /// Expects the `results` section of the YAML document. pub fn from_yaml(yaml: &Yaml) -> Self { Self { num_skipped_slots: as_usize(yaml, "num_skipped_slots"), @@ -17,6 +22,7 @@ impl Results { } } +/// Parse the `state_checks` section of the YAML document. fn parse_state_checks(yaml: &Yaml) -> Option> { let mut states = vec![]; diff --git a/beacon_node/beacon_chain/test_harness/src/manifest/state_check.rs b/beacon_node/beacon_chain/test_harness/src/manifest/state_check.rs index 0415d4896..a729811d5 100644 --- a/beacon_node/beacon_chain/test_harness/src/manifest/state_check.rs +++ b/beacon_node/beacon_chain/test_harness/src/manifest/state_check.rs @@ -3,15 +3,24 @@ use log::info; use types::*; use yaml_rust::Yaml; +/// Tests to be conducted upon a `BeaconState` object generated during the execution of a +/// `Manifest`. #[derive(Debug)] pub struct StateCheck { + /// Checked against `beacon_state.slot`. pub slot: Slot, + /// Checked against `beacon_state.validator_registry.len()`. pub num_validators: Option, + /// A list of validator indices which have been penalized. Must be in ascending order. pub slashed_validators: Option>, + /// A list of validator indices which have been exited. Must be in ascending order. pub exited_validators: Option>, } impl StateCheck { + /// Load from a YAML document. + /// + /// Expects the `state_check` section of the YAML document. pub fn from_yaml(yaml: &Yaml) -> Self { Self { slot: Slot::from(as_u64(&yaml, "slot").expect("State must specify slot")), @@ -21,6 +30,11 @@ impl StateCheck { } } + /// Performs all checks against a `BeaconState` + /// + /// # Panics + /// + /// Panics with an error message if any test fails. pub fn assert_valid(&self, state: &BeaconState, spec: &ChainSpec) { let state_epoch = state.slot.epoch(spec.epoch_length);