merged master

.gitignore

@@ -4,3 +4,5 @@ Cargo.lock
 *.pk
 *.sk
 *.raw_keypairs
+flamegraph.svg
+perf.data*

.gitlab-ci.yml

@@ -0,0 +1,36 @@
+#Adapted from https://users.rust-lang.org/t/my-gitlab-config-docs-tests/16396
+
+image: 'sigp/lighthouse:latest'
+
+stages:
+  - test
+  - document
+
+variables:
+    CARGO_HOME: /cache/cargocache
+
+check-fmt:
+  stage: test
+  script:
+    - cargo build --manifest-path protos/Cargo.toml
+    - cargo fmt --all -- --check
+
+test-dev:
+  stage: test
+  script:
+    - cargo test --verbose --all
+
+test-release:
+  stage: test
+  script:
+    - cargo test --verbose --all --release
+
+documentation:
+  stage: document
+  script:
+    - cargo doc --no-deps
+    - aws s3 sync target/doc/ s3://lighthouse-docs.sigmaprime.io/ --exclude '.lock' --delete
+  # Configure the below when we want to have a default page (and update S3 bucket index).
+  #    - echo '<meta http-equiv="refresh" content="0; url={{ LIBRARY NAME }}">' > public/index.html
+  only:
+    - master

.travis.yml

@@ -1,4 +1,7 @@
 language: rust
+cache:
+  directories:
+    - /home/travis/.cargo
 before_install:
   - curl -OL https://github.com/google/protobuf/releases/download/v3.4.0/protoc-3.4.0-linux-x86_64.zip
   - unzip protoc-3.4.0-linux-x86_64.zip -d protoc3
@@ -7,15 +10,8 @@ before_install:
   - sudo chown $USER /usr/local/bin/protoc
   - sudo chown -R $USER /usr/local/include/google
 script:
-  - cargo build --verbose --all
+  - cargo build --verbose --all --release
-  - cargo build --verbose --release --all
-  - cargo test --verbose --all
-  - cargo test --verbose --release --all
-  - cargo fmt --all -- --check
-  # No clippy until later...
-  #- cargo clippy
 rust:
-  - stable
   - beta
   - nightly
 matrix:
@@ -24,4 +20,3 @@ matrix:
   fast_finish: true
 install:
   - rustup component add rustfmt
-  - rustup component add clippy

Cargo.toml

@@ -1,14 +1,13 @@
 [workspace]
 members = [
-	"eth2/attester",
-	"eth2/block_proposer",
 	"eth2/fork_choice",
 	"eth2/operation_pool",
 	"eth2/state_processing",
-	"eth2/state_processing/yaml_utils",
 	"eth2/types",
 	"eth2/utils/bls",
 	"eth2/utils/boolean-bitfield",
+	"eth2/utils/cached_tree_hash",
+	"eth2/utils/fixed_len_vec",
 	"eth2/utils/hashing",
 	"eth2/utils/honey-badger-split",
 	"eth2/utils/merkle_proof",
@@ -18,6 +17,8 @@ members = [
 	"eth2/utils/ssz",
 	"eth2/utils/ssz_derive",
 	"eth2/utils/swap_or_not_shuffle",
+	"eth2/utils/tree_hash",
+	"eth2/utils/tree_hash_derive",
 	"eth2/utils/fisher_yates_shuffle",
     "eth2/utils/test_random_derive",
 	"beacon_node",
@@ -28,7 +29,6 @@ members = [
     "beacon_node/rpc",
 	"beacon_node/version",
 	"beacon_node/beacon_chain",
-	"beacon_node/beacon_chain/test_harness",
 	"protos",
 	"validator_client",
 	"account_manager",

Dockerfile

@@ -1,6 +1,6 @@
 FROM rust:latest
 
-RUN apt-get update && apt-get install -y clang libclang-dev cmake build-essential git unzip autoconf libtool
+RUN apt-get update && apt-get install -y clang libclang-dev cmake build-essential git unzip autoconf libtool awscli
 
 RUN git clone https://github.com/google/protobuf.git && \
     cd protobuf && \
@@ -14,8 +14,8 @@ RUN git clone https://github.com/google/protobuf.git && \
     rm -r protobuf
 
 
-RUN mkdir /cargocache && chmod -R ugo+rwX /cargocache
+RUN mkdir -p /cache/cargocache && chmod -R ugo+rwX /cache/cargocache
 
-ENV CARGO_HOME /cargocache
+ENV CARGO_HOME /cache/cargocache
 
 RUN rustup component add rustfmt clippy

Jenkinsfile

@@ -1,29 +0,0 @@
-pipeline {
-	agent {
-		dockerfile {
-			filename 'Dockerfile'
-			args '-v cargo-cache:/cargocache:rw -e "CARGO_HOME=/cargocache"'
-		}
-	}
-	stages {
-		stage('Build') {
-			steps {
-				sh 'cargo build --verbose --all'
-				sh 'cargo build --verbose --all --release'
-			}
-		}
-        stage('Check') {
-            steps {
-                sh 'cargo fmt --all -- --check'
-                // No clippy until later...
-                //sh 'cargo clippy'
-            }
-        }
-		stage('Test') {
-			steps {
-				sh 'cargo test --verbose --all'
-				sh 'cargo test --verbose --all --release'
-			}
-		}
-	}
-}

README.md

@@ -1,6 +1,13 @@
 # Lighthouse: an Ethereum Serenity client
 
-[![Build Status](https://travis-ci.org/sigp/lighthouse.svg?branch=master)](https://travis-ci.org/sigp/lighthouse) [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/sigp/lighthouse?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
+[![Build Status]][Build Link] [![Doc Status]][Doc Link] [![Gitter Badge]][Gitter Link]
+
+[Build Status]: https://gitlab.sigmaprime.io/sigp/lighthouse/badges/master/build.svg
+[Build Link]: https://gitlab.sigmaprime.io/sigp/lighthouse/pipelines
+[Gitter Badge]: https://badges.gitter.im/Join%20Chat.svg
+[Gitter Link]: https://gitter.im/sigp/lighthouse
+[Doc Status]: https://img.shields.io/badge/docs-master-blue.svg
+[Doc Link]: http://lighthouse-docs.sigmaprime.io/
 
 A work-in-progress, open-source implementation of the Serenity Beacon
 Chain, maintained by Sigma Prime.
@@ -24,6 +31,7 @@ present-Ethereum functionality.
 - [About Lighthouse](docs/lighthouse.md): Goals, Ideology and Ethos surrounding
 this implementation.
 - [What is Ethereum Serenity](docs/serenity.md): an introduction to Ethereum Serenity.
+- [Lighthouse Technical Documentation](http://lighthouse-docs.sigmaprime.io/): The Rust generated documentation, updated regularly.
 
 If you'd like some background on Sigma Prime, please see the [Lighthouse Update
 \#00](https://lighthouse.sigmaprime.io/update-00.html) blog post or the

account_manager/Cargo.toml

@@ -11,3 +11,4 @@ slog = "^2.2.3"
 slog-term = "^2.4.0"
 slog-async = "^2.3.0"
 validator_client = { path = "../validator_client" }
+types = { path = "../eth2/types" }

account_manager/README.md

@@ -1,6 +1,6 @@
-# Lighthouse Accounts Manager
+# Lighthouse Account Manager
 
-The accounts manager (AM) is a stand-alone binary which allows
+The account manager (AM) is a stand-alone binary which allows
 users to generate and manage the cryptographic keys necessary to
 interact with Ethereum Serenity.
 
@@ -21,4 +21,14 @@ staking on Ethereum 1.x (TPD)
 The AM is not a service, and does not run continuously, nor does it
 interact with any running services.
 It is intended to be executed separately from other Lighthouse binaries
-and produce files which can be consumed by them.
+and produce files which can be consumed by them.&
+
+## Usage
+
+Simply run `./account_manager generate` to generate a new random private key,
+which will be automatically saved to the correct directory.
+
+If you prefer to use our "deterministic" keys for testing purposes, simply
+run `./accounts_manager generate_deterministic -i <index>`, where `index` is
+the validator index for the key. This will reliably produce the same key each time
+and save it to the directory.

account_manager/src/main.rs

@@ -2,6 +2,7 @@ use bls::Keypair;
 use clap::{App, Arg, SubCommand};
 use slog::{debug, info, o, Drain};
 use std::path::PathBuf;
+use types::test_utils::generate_deterministic_keypair;
 use validator_client::Config as ValidatorClientConfig;
 
 fn main() {
@@ -29,6 +30,21 @@ fn main() {
                 .version("0.0.1")
                 .author("Sigma Prime <contact@sigmaprime.io>"),
         )
+        .subcommand(
+            SubCommand::with_name("generate_deterministic")
+                .about("Generates a deterministic validator private key FOR TESTING")
+                .version("0.0.1")
+                .author("Sigma Prime <contact@sigmaprime.io>")
+                .arg(
+                    Arg::with_name("validator index")
+                        .long("index")
+                        .short("i")
+                        .value_name("index")
+                        .help("The index of the validator, for which the test key is generated")
+                        .takes_value(true)
+                        .required(true),
+                ),
+        )
         .get_matches();
 
     let config = ValidatorClientConfig::parse_args(&matches, &log)
@@ -51,6 +67,23 @@ fn main() {
                 key_path.to_string_lossy()
             );
         }
+        ("generate_deterministic", Some(gen_d_matches)) => {
+            let validator_index = gen_d_matches
+                .value_of("validator index")
+                .expect("Validator index required.")
+                .parse::<u64>()
+                .expect("Invalid validator index.") as usize;
+            let keypair = generate_deterministic_keypair(validator_index);
+            let key_path: PathBuf = config
+                .save_key(&keypair)
+                .expect("Unable to save newly generated deterministic private key.");
+            debug!(
+                log,
+                "Deterministic Keypair generated {:?}, saved to: {:?}",
+                keypair.identifier(),
+                key_path.to_string_lossy()
+            );
+        }
         _ => panic!(
             "The account manager must be run with a subcommand. See help for more information."
         ),

beacon_node/beacon_chain/Cargo.toml

@@ -5,7 +5,6 @@ authors = ["Paul Hauner <paul@paulhauner.com>", "Age Manning <Age@AgeManning.com
 edition = "2018"
 
 [dependencies]
-block_proposer = { path = "../../eth2/block_proposer" }
 bls = { path = "../../eth2/utils/bls" }
 boolean-bitfield = { path = "../../eth2/utils/boolean-bitfield" }
 db = { path = "../db" }
@@ -23,4 +22,5 @@ serde_json = "1.0"
 slot_clock = { path = "../../eth2/utils/slot_clock" }
 ssz = { path = "../../eth2/utils/ssz" }
 state_processing = { path = "../../eth2/state_processing" }
+tree_hash = { path = "../../eth2/utils/tree_hash" }
 types = { path = "../../eth2/types" }

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -83,30 +83,31 @@ impl BlockProcessingOutcome {
     }
 }
 
-pub struct BeaconChain<T: ClientDB + Sized, U: SlotClock, F: ForkChoice> {
+pub struct BeaconChain<T: ClientDB + Sized, U: SlotClock, F: ForkChoice, E: EthSpec> {
     pub block_store: Arc<BeaconBlockStore<T>>,
     pub state_store: Arc<BeaconStateStore<T>>,
     pub slot_clock: U,
-    pub op_pool: OperationPool,
+    pub op_pool: OperationPool<E>,
-    canonical_head: RwLock<CheckPoint>,
+    canonical_head: RwLock<CheckPoint<E>>,
-    finalized_head: RwLock<CheckPoint>,
+    finalized_head: RwLock<CheckPoint<E>>,
-    pub state: RwLock<BeaconState>,
+    pub state: RwLock<BeaconState<E>>,
     pub spec: ChainSpec,
     pub fork_choice: RwLock<F>,
 }
 
-impl<T, U, F> BeaconChain<T, U, F>
+impl<T, U, F, E> BeaconChain<T, U, F, E>
 where
     T: ClientDB,
     U: SlotClock,
     F: ForkChoice,
+    E: EthSpec,
 {
     /// Instantiate a new Beacon Chain, from genesis.
     pub fn from_genesis(
         state_store: Arc<BeaconStateStore<T>>,
         block_store: Arc<BeaconBlockStore<T>>,
         slot_clock: U,
-        mut genesis_state: BeaconState,
+        mut genesis_state: BeaconState<E>,
         genesis_block: BeaconBlock,
         spec: ChainSpec,
         fork_choice: F,
@@ -190,7 +191,6 @@ where
         count: usize,
         skip: usize,
     ) -> Result<Vec<Hash256>, Error> {
-        let spec = &self.spec;
         let step_by = Slot::from(skip + 1);
 
         let mut roots: Vec<Hash256> = vec![];
@@ -218,7 +218,7 @@ where
             //
             // If we get `SlotOutOfBounds` error, load the oldest available historic
             // state from the DB.
-            match state.get_block_root(slot, spec) {
+            match state.get_block_root(slot) {
                 Ok(root) => {
                     if slot < earliest_slot {
                         break;
@@ -230,9 +230,9 @@ where
                 Err(BeaconStateError::SlotOutOfBounds) => {
                     // Read the earliest historic state in the current slot.
                     let earliest_historic_slot =
-                        state.slot - Slot::from(spec.slots_per_historical_root);
+                        state.slot - Slot::from(E::SlotsPerHistoricalRoot::to_usize());
                     // Load the earlier state from disk.
-                    let new_state_root = state.get_state_root(earliest_historic_slot, spec)?;
+                    let new_state_root = state.get_state_root(earliest_historic_slot)?;
 
                     // Break if the DB is unable to load the state.
                     state = match self.state_store.get_deserialized(&new_state_root) {
@@ -270,7 +270,7 @@ where
         &self,
         new_beacon_block: BeaconBlock,
         new_beacon_block_root: Hash256,
-        new_beacon_state: BeaconState,
+        new_beacon_state: BeaconState<E>,
         new_beacon_state_root: Hash256,
     ) {
         debug!(
@@ -292,7 +292,7 @@ where
     /// It is important to note that the `beacon_state` returned may not match the present slot. It
     /// is the state as it was when the head block was received, which could be some slots prior to
     /// now.
-    pub fn head(&self) -> RwLockReadGuard<CheckPoint> {
+    pub fn head(&self) -> RwLockReadGuard<CheckPoint<E>> {
         self.canonical_head.read()
     }
 
@@ -302,9 +302,7 @@ where
     /// state and calling `catchup_state` as it will not result in an old state being installed and
     /// then having it iteratively updated -- in such a case it's possible for another thread to
     /// find the state at an old slot.
-    pub fn update_state(&self, mut state: BeaconState) -> Result<(), Error> {
+    pub fn update_state(&self, mut state: BeaconState<E>) -> Result<(), Error> {
-        let latest_block_header = self.head().beacon_block.block_header();
-
         let present_slot = match self.slot_clock.present_slot() {
             Ok(Some(slot)) => slot,
             _ => return Err(Error::UnableToReadSlot),
@@ -312,7 +310,7 @@ where
 
         // If required, transition the new state to the present slot.
         for _ in state.slot.as_u64()..present_slot.as_u64() {
-            per_slot_processing(&mut state, &latest_block_header, &self.spec)?;
+            per_slot_processing(&mut state, &self.spec)?;
         }
 
         state.build_all_caches(&self.spec)?;
@@ -324,8 +322,6 @@ where
 
     /// Ensures the current canonical `BeaconState` has been transitioned to match the `slot_clock`.
     pub fn catchup_state(&self) -> Result<(), Error> {
-        let latest_block_header = self.head().beacon_block.block_header();
-
         let present_slot = match self.slot_clock.present_slot() {
             Ok(Some(slot)) => slot,
             _ => return Err(Error::UnableToReadSlot),
@@ -339,7 +335,7 @@ where
             state.build_epoch_cache(RelativeEpoch::NextWithoutRegistryChange, &self.spec)?;
             state.build_epoch_cache(RelativeEpoch::NextWithRegistryChange, &self.spec)?;
 
-            per_slot_processing(&mut *state, &latest_block_header, &self.spec)?;
+            per_slot_processing(&mut *state, &self.spec)?;
         }
 
         state.build_all_caches(&self.spec)?;
@@ -361,7 +357,7 @@ where
         &self,
         new_beacon_block: BeaconBlock,
         new_beacon_block_root: Hash256,
-        new_beacon_state: BeaconState,
+        new_beacon_state: BeaconState<E>,
         new_beacon_state_root: Hash256,
     ) {
         let mut finalized_head = self.finalized_head.write();
@@ -375,7 +371,7 @@ where
 
     /// Returns a read-lock guarded `CheckPoint` struct for reading the justified head (as chosen,
     /// indirectly,  by the fork-choice rule).
-    pub fn finalized_head(&self) -> RwLockReadGuard<CheckPoint> {
+    pub fn finalized_head(&self) -> RwLockReadGuard<CheckPoint<E>> {
         self.finalized_head.read()
     }
 
@@ -497,17 +493,14 @@ where
             } else {
                 // If the current head block is not from this slot, use the slot from the previous
                 // epoch.
-                *self.state.read().get_block_root(
+                *self
-                    current_epoch_start_slot - self.spec.slots_per_epoch,
+                    .state
-                    &self.spec,
+                    .read()
-                )?
+                    .get_block_root(current_epoch_start_slot - self.spec.slots_per_epoch)?
             }
         } else {
             // If we're not on the first slot of the epoch.
-            *self
+            *self.state.read().get_block_root(current_epoch_start_slot)?
-                .state
-                .read()
-                .get_block_root(current_epoch_start_slot, &self.spec)?
         };
 
         Ok(AttestationData {
@@ -617,9 +610,8 @@ where
 
         // Transition the parent state to the block slot.
         let mut state = parent_state;
-        let previous_block_header = parent_block.block_header();
         for _ in state.slot.as_u64()..block.slot.as_u64() {
-            if let Err(e) = per_slot_processing(&mut state, &previous_block_header, &self.spec) {
+            if let Err(e) = per_slot_processing(&mut state, &self.spec) {
                 return Ok(BlockProcessingOutcome::InvalidBlock(
                     InvalidBlock::SlotProcessingError(e),
                 ));
@@ -672,7 +664,7 @@ where
     pub fn produce_block(
         &self,
         randao_reveal: Signature,
-    ) -> Result<(BeaconBlock, BeaconState), BlockProductionError> {
+    ) -> Result<(BeaconBlock, BeaconState<E>), BlockProductionError> {
         debug!("Producing block at slot {}...", self.state.read().slot);
 
         let mut state = self.state.read().clone();
@@ -682,7 +674,7 @@ where
         trace!("Finding attestations for new block...");
 
         let previous_block_root = *state
-            .get_block_root(state.slot - 1, &self.spec)
+            .get_block_root(state.slot - 1)
             .map_err(|_| BlockProductionError::UnableToGetBlockRootFromState)?;
 
         let (proposer_slashings, attester_slashings) =
@@ -767,7 +759,7 @@ where
     ///
     /// This could be a very expensive operation and should only be done in testing/analysis
     /// activities.
-    pub fn chain_dump(&self) -> Result<Vec<CheckPoint>, Error> {
+    pub fn chain_dump(&self) -> Result<Vec<CheckPoint<E>>, Error> {
         let mut dump = vec![];
 
         let mut last_slot = CheckPoint {

beacon_node/beacon_chain/src/checkpoint.rs

@@ -1,22 +1,22 @@
 use serde_derive::Serialize;
-use types::{BeaconBlock, BeaconState, Hash256};
+use types::{BeaconBlock, BeaconState, EthSpec, Hash256};
 
 /// Represents some block and it's associated state. Generally, this will be used for tracking the
 /// head, justified head and finalized head.
 #[derive(Clone, Serialize, PartialEq, Debug)]
-pub struct CheckPoint {
+pub struct CheckPoint<E: EthSpec> {
     pub beacon_block: BeaconBlock,
     pub beacon_block_root: Hash256,
-    pub beacon_state: BeaconState,
+    pub beacon_state: BeaconState<E>,
     pub beacon_state_root: Hash256,
 }
 
-impl CheckPoint {
+impl<E: EthSpec> CheckPoint<E> {
     /// Create a new checkpoint.
     pub fn new(
         beacon_block: BeaconBlock,
         beacon_block_root: Hash256,
-        beacon_state: BeaconState,
+        beacon_state: BeaconState<E>,
         beacon_state_root: Hash256,
     ) -> Self {
         Self {
@@ -32,7 +32,7 @@ impl CheckPoint {
         &mut self,
         beacon_block: BeaconBlock,
         beacon_block_root: Hash256,
-        beacon_state: BeaconState,
+        beacon_state: BeaconState<E>,
         beacon_state_root: Hash256,
     ) {
         self.beacon_block = beacon_block;

beacon_node/beacon_chain/src/initialise.rs

@@ -7,18 +7,25 @@ use db::stores::{BeaconBlockStore, BeaconStateStore};
 use db::{DiskDB, MemoryDB};
 use fork_choice::BitwiseLMDGhost;
 use slot_clock::SystemTimeSlotClock;
-use ssz::TreeHash;
 use std::path::PathBuf;
 use std::sync::Arc;
+use tree_hash::TreeHash;
 use types::test_utils::TestingBeaconStateBuilder;
-use types::{BeaconBlock, ChainSpec, Hash256};
+use types::{BeaconBlock, ChainSpec, FewValidatorsEthSpec, FoundationEthSpec, Hash256};
 
 //TODO: Correct this for prod
 //TODO: Account for historical db
 pub fn initialise_beacon_chain(
     spec: &ChainSpec,
     db_name: Option<&PathBuf>,
-) -> Arc<BeaconChain<DiskDB, SystemTimeSlotClock, BitwiseLMDGhost<DiskDB>>> {
+) -> Arc<
+    BeaconChain<
+        DiskDB,
+        SystemTimeSlotClock,
+        BitwiseLMDGhost<DiskDB, FoundationEthSpec>,
+        FoundationEthSpec,
+    >,
+> {
     // set up the db
     let db = Arc::new(DiskDB::open(
         db_name.expect("Database directory must be included"),
@@ -32,7 +39,7 @@ pub fn initialise_beacon_chain(
     let (genesis_state, _keypairs) = state_builder.build();
 
     let mut genesis_block = BeaconBlock::empty(&spec);
-    genesis_block.state_root = Hash256::from_slice(&genesis_state.hash_tree_root());
+    genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root());
 
     // Slot clock
     let slot_clock = SystemTimeSlotClock::new(
@@ -64,7 +71,14 @@ pub fn initialise_beacon_chain(
 pub fn initialise_test_beacon_chain(
     spec: &ChainSpec,
     _db_name: Option<&PathBuf>,
-) -> Arc<BeaconChain<MemoryDB, SystemTimeSlotClock, BitwiseLMDGhost<MemoryDB>>> {
+) -> Arc<
+    BeaconChain<
+        MemoryDB,
+        SystemTimeSlotClock,
+        BitwiseLMDGhost<MemoryDB, FewValidatorsEthSpec>,
+        FewValidatorsEthSpec,
+    >,
+> {
     let db = Arc::new(MemoryDB::open());
     let block_store = Arc::new(BeaconBlockStore::new(db.clone()));
     let state_store = Arc::new(BeaconStateStore::new(db.clone()));
@@ -73,7 +87,7 @@ pub fn initialise_test_beacon_chain(
     let (genesis_state, _keypairs) = state_builder.build();
 
     let mut genesis_block = BeaconBlock::empty(spec);
-    genesis_block.state_root = Hash256::from_slice(&genesis_state.hash_tree_root());
+    genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root());
 
     // Slot clock
     let slot_clock = SystemTimeSlotClock::new(

beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs

@@ -5,19 +5,20 @@ use db::{
 };
 use fork_choice::BitwiseLMDGhost;
 use slot_clock::TestingSlotClock;
-use ssz::TreeHash;
 use std::sync::Arc;
-use types::test_utils::TestingBeaconStateBuilder;
+use tree_hash::TreeHash;
 use types::*;
+use types::{test_utils::TestingBeaconStateBuilder, EthSpec, FewValidatorsEthSpec};
 
-type TestingBeaconChain = BeaconChain<MemoryDB, TestingSlotClock, BitwiseLMDGhost<MemoryDB>>;
+type TestingBeaconChain<E> =
+    BeaconChain<MemoryDB, TestingSlotClock, BitwiseLMDGhost<MemoryDB, FewValidatorsEthSpec>, E>;
 
-pub struct TestingBeaconChainBuilder {
+pub struct TestingBeaconChainBuilder<E: EthSpec> {
-    state_builder: TestingBeaconStateBuilder,
+    state_builder: TestingBeaconStateBuilder<E>,
 }
 
-impl TestingBeaconChainBuilder {
+impl<E: EthSpec> TestingBeaconChainBuilder<E> {
-    pub fn build(self, spec: &ChainSpec) -> TestingBeaconChain {
+    pub fn build(self, spec: &ChainSpec) -> TestingBeaconChain<E> {
         let db = Arc::new(MemoryDB::open());
         let block_store = Arc::new(BeaconBlockStore::new(db.clone()));
         let state_store = Arc::new(BeaconStateStore::new(db.clone()));
@@ -27,7 +28,7 @@ impl TestingBeaconChainBuilder {
         let (genesis_state, _keypairs) = self.state_builder.build();
 
         let mut genesis_block = BeaconBlock::empty(&spec);
-        genesis_block.state_root = Hash256::from_slice(&genesis_state.hash_tree_root());
+        genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root());
 
         // Create the Beacon Chain
         BeaconChain::from_genesis(
@@ -43,8 +44,8 @@ impl TestingBeaconChainBuilder {
     }
 }
 
-impl From<TestingBeaconStateBuilder> for TestingBeaconChainBuilder {
+impl<E: EthSpec> From<TestingBeaconStateBuilder<E>> for TestingBeaconChainBuilder<E> {
-    fn from(state_builder: TestingBeaconStateBuilder) -> TestingBeaconChainBuilder {
+    fn from(state_builder: TestingBeaconStateBuilder<E>) -> TestingBeaconChainBuilder<E> {
         TestingBeaconChainBuilder { state_builder }
     }
 }

beacon_node/beacon_chain/test_harness/Cargo.toml

@@ -1,42 +0,0 @@
-[package]
-name = "test_harness"
-version = "0.1.0"
-authors = ["Paul Hauner <paul@paulhauner.com>"]
-edition = "2018"
-
-[[bin]]
-name = "test_harness"
-path = "src/bin.rs"
-
-[lib]
-name = "test_harness"
-path = "src/lib.rs"
-
-[dev-dependencies]
-state_processing = { path = "../../../eth2/state_processing" }
-
-[dependencies]
-attester = { path = "../../../eth2/attester" }
-beacon_chain = { path = "../../beacon_chain" }
-block_proposer = { path = "../../../eth2/block_proposer" }
-bls = { path = "../../../eth2/utils/bls" }
-boolean-bitfield = { path = "../../../eth2/utils/boolean-bitfield" }
-clap = "2.32.0"
-db = { path = "../../db" }
-parking_lot = "0.7"
-failure = "0.1"
-failure_derive = "0.1"
-fork_choice = { path = "../../../eth2/fork_choice" }
-hashing = { path = "../../../eth2/utils/hashing" }
-int_to_bytes = { path = "../../../eth2/utils/int_to_bytes" }
-log = "0.4"
-env_logger = "0.6.0"
-rayon = "1.0"
-serde = "1.0"
-serde_derive = "1.0"
-serde_json = "1.0"
-serde_yaml = "0.8"
-slot_clock = { path = "../../../eth2/utils/slot_clock" }
-ssz = { path = "../../../eth2/utils/ssz" }
-types = { path = "../../../eth2/types" }
-yaml-rust = "0.4.2"

beacon_node/beacon_chain/test_harness/README.md

@@ -1,150 +0,0 @@
-# Test Harness
-
-Provides a testing environment for the `BeaconChain`, `Attester` and `BlockProposer` objects.
-
-This environment bypasses networking and client run-times and connects the `Attester` and `Proposer`
-directly to the `BeaconChain` via an `Arc`.
-
-The `BeaconChainHarness` contains a single `BeaconChain` instance and many `ValidatorHarness`
-instances. All of the `ValidatorHarness` instances work to advance the `BeaconChain` by
-producing blocks and attestations.
-
-The crate consists of a library and binary, examples for using both are
-described below.
-
-## YAML
-
-Both the library and the binary are capable of parsing tests from a YAML file,
-in fact this is the sole purpose of the binary.
-
-You can find YAML test cases [here](specs/). An example is included below:
-
-```yaml
-title: Validator Registry Tests
-summary: Tests deposit and slashing effects on validator registry.
-test_suite: validator_registry
-fork: tchaikovsky
-version: 1.0
-test_cases:
-  - config:
-      slots_per_epoch: 64
-      deposits_for_chain_start: 1000
-      num_slots: 64
-      skip_slots: [2, 3]
-      deposits:
-          # At slot 1, create a new validator deposit of 32 ETH.
-        - slot: 1
-          amount: 32
-          # Trigger more deposits...
-        - slot: 3
-          amount: 32
-        - slot: 5
-          amount: 32
-      proposer_slashings:
-          # At slot 2, trigger a proposer slashing for validator #42.
-        - slot: 2
-          validator_index: 42
-          # Trigger another slashing...
-        - slot: 8
-          validator_index: 13
-      attester_slashings:
-          # At slot 2, trigger an attester slashing for validators #11 and #12.
-        - slot: 2
-          validator_indices: [11, 12]
-          # Trigger another slashing...
-        - slot: 5
-          validator_indices: [14]
-    results:
-      num_skipped_slots: 2
-      states:
-        - slot: 63
-          num_validators: 1003
-          slashed_validators: [11, 12, 13, 14, 42]
-          exited_validators: []
-
-```
-
-Thanks to [prsym](http://github.com/prysmaticlabs/prysm) for coming up with the
-base YAML format.
-
-### Notes
-
-Wherever `slot` is used, it is actually the "slot height", or slots since
-genesis. This allows the tests to disregard the `GENESIS_EPOCH`.
-
-### Differences from Prysmatic's format
-
-1. The detail for `deposits`, `proposer_slashings` and `attester_slashings` is
-   ommitted from the test specification. It assumed they should be valid
-   objects.
-2. There is a `states` list in `results` that runs checks against any state
-   specified by a `slot` number. This is in contrast to the variables in
-   `results` that assume the last (highest) state should be inspected.
-
-#### Reasoning
-
-Respective reasonings for above changes:
-
-1. This removes the concerns of the actual object structure from the tests.
-   This allows for more variation in the deposits/slashings objects without
-   needing to update the tests. Also, it makes it makes it easier to create
-   tests.
-2. This gives more fine-grained control over the tests. It allows for checking
-   that certain events happened at certain times whilst making the tests only
-   slightly more verbose.
-
-_Notes: it may be useful to add an extra field to each slashing type to
-indicate if it should be valid or not. It also may be useful to add an option
-for double-vote/surround-vote attester slashings. The `amount` field was left
-on `deposits` as it changes the behaviour of state significantly._
-
-## Binary Usage Example
-
-Follow these steps to run as a binary:
-
-1. Navigate to the root of this crate (where this readme is located)
-2. Run `$ cargo run --release -- --yaml examples/validator_registry.yaml`
-
-_Note: the `--release` flag builds the binary without all the debugging
-instrumentation. The test is much faster built using `--release`. As is
-customary in cargo, the flags before `--` are passed to cargo and the flags
-after are passed to the binary._
-
-### CLI Options
-
-```
-Lighthouse Test Harness Runner 0.0.1
-Sigma Prime <contact@sigmaprime.io>
-Runs `test_harness` using a YAML test_case.
-
-USAGE:
-    test_harness --log-level <LOG_LEVEL> --yaml <FILE>
-
-FLAGS:
-    -h, --help       Prints help information
-    -V, --version    Prints version information
-
-OPTIONS:
-        --log-level <LOG_LEVEL>    Logging level. [default: debug]  [possible values: error, warn, info, debug, trace]
-        --yaml <FILE>              YAML file test_case.
-```
-
-
-## Library Usage Example
-
-```rust
-use test_harness::BeaconChainHarness;
-use types::ChainSpec;
-
-let validator_count = 8;
-let spec = ChainSpec::few_validators();
-
-let mut harness = BeaconChainHarness::new(spec, validator_count);
-
-harness.advance_chain_with_block();
-
-let chain = harness.chain_dump().unwrap();
-
-// One block should have been built on top of the genesis block.
-assert_eq!(chain.len(), 2);
-```

beacon_node/beacon_chain/test_harness/specs/validator_registry.yaml

@@ -1,63 +0,0 @@
-title: Validator Registry Tests
-summary: Tests deposit and slashing effects on validator registry.
-test_suite: validator_registry
-fork: tchaikovsky
-version: 1.0
-test_cases:
-  - config:
-      slots_per_epoch: 64
-      deposits_for_chain_start: 1000
-      num_slots: 64
-      skip_slots: [2, 3]
-      persistent_committee_period: 0
-      deposits:
-          # At slot 1, create a new validator deposit of 5 ETH.
-        - slot: 1
-          amount: 5000000000
-          # Trigger more deposits...
-        - slot: 3
-          amount: 5000000000
-        - slot: 5
-          amount: 32000000000
-      exits:
-          # At slot 10, submit an exit for validator #50.
-        - slot: 10
-          validator_index: 50
-      transfers:
-        - slot: 6
-          from: 1000
-          to: 1001
-          amount: 5000000000
-      proposer_slashings:
-          # At slot 2, trigger a proposer slashing for validator #42.
-        - slot: 2
-          validator_index: 42
-          # Trigger another slashing...
-        - slot: 8
-          validator_index: 13
-      attester_slashings:
-          # At slot 2, trigger an attester slashing for validators #11 and #12.
-        - slot: 2
-          validator_indices: [11, 12]
-          # Trigger another slashing...
-        - slot: 5
-          validator_indices: [14]
-    results:
-      num_skipped_slots: 2
-      states:
-        - slot: 63
-          num_validators: 1003
-          num_previous_epoch_attestations: 0
-          # slots_per_epoch - attestation_inclusion_delay - skip_slots
-          num_current_epoch_attestations: 57
-          slashed_validators: [11, 12, 13, 14, 42]
-          exited_validators: []
-          exit_initiated_validators: [50]
-          balances:
-            - validator_index: 1000
-              comparison: "eq"
-              balance: 0
-            - validator_index: 1001
-              comparison: "eq"
-              balance: 10000000000
-

beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs

@@ -1,350 +0,0 @@
-use super::ValidatorHarness;
-use beacon_chain::{BeaconChain, BlockProcessingOutcome};
-pub use beacon_chain::{BeaconChainError, CheckPoint};
-use db::{
-    stores::{BeaconBlockStore, BeaconStateStore},
-    MemoryDB,
-};
-use fork_choice::BitwiseLMDGhost;
-use log::debug;
-use rayon::prelude::*;
-use slot_clock::TestingSlotClock;
-use ssz::TreeHash;
-use std::sync::Arc;
-use types::{test_utils::TestingBeaconStateBuilder, *};
-
-type TestingBeaconChain = BeaconChain<MemoryDB, TestingSlotClock, BitwiseLMDGhost<MemoryDB>>;
-
-/// The beacon chain harness simulates a single beacon node with `validator_count` validators connected
-/// to it. Each validator is provided a borrow to the beacon chain, where it may read
-/// information and submit blocks/attestations for processing.
-///
-/// This test harness is useful for testing validator and internal state transition logic. It
-/// is not useful for testing that multiple beacon nodes can reach consensus.
-pub struct BeaconChainHarness {
-    pub db: Arc<MemoryDB>,
-    pub beacon_chain: Arc<TestingBeaconChain>,
-    pub block_store: Arc<BeaconBlockStore<MemoryDB>>,
-    pub state_store: Arc<BeaconStateStore<MemoryDB>>,
-    pub validators: Vec<ValidatorHarness>,
-    pub spec: Arc<ChainSpec>,
-}
-
-impl BeaconChainHarness {
-    /// Create a new harness with:
-    ///
-    /// - A keypair, `BlockProducer` and `Attester` for each validator.
-    /// - A new BeaconChain struct where the given validators are in the genesis.
-    pub fn new(spec: ChainSpec, validator_count: usize) -> Self {
-        let state_builder =
-            TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(validator_count, &spec);
-        Self::from_beacon_state_builder(state_builder, spec)
-    }
-
-    pub fn from_beacon_state_builder(
-        state_builder: TestingBeaconStateBuilder,
-        spec: ChainSpec,
-    ) -> Self {
-        let db = Arc::new(MemoryDB::open());
-        let block_store = Arc::new(BeaconBlockStore::new(db.clone()));
-        let state_store = Arc::new(BeaconStateStore::new(db.clone()));
-        let slot_clock = TestingSlotClock::new(spec.genesis_slot.as_u64());
-        let fork_choice = BitwiseLMDGhost::new(block_store.clone(), state_store.clone());
-
-        let (mut genesis_state, keypairs) = state_builder.build();
-
-        let mut genesis_block = BeaconBlock::empty(&spec);
-        genesis_block.state_root = Hash256::from_slice(&genesis_state.hash_tree_root());
-
-        genesis_state
-            .build_epoch_cache(RelativeEpoch::Previous, &spec)
-            .unwrap();
-        genesis_state
-            .build_epoch_cache(RelativeEpoch::Current, &spec)
-            .unwrap();
-        genesis_state
-            .build_epoch_cache(RelativeEpoch::NextWithoutRegistryChange, &spec)
-            .unwrap();
-        genesis_state
-            .build_epoch_cache(RelativeEpoch::NextWithRegistryChange, &spec)
-            .unwrap();
-
-        // Create the Beacon Chain
-        let beacon_chain = Arc::new(
-            BeaconChain::from_genesis(
-                state_store.clone(),
-                block_store.clone(),
-                slot_clock,
-                genesis_state,
-                genesis_block,
-                spec.clone(),
-                fork_choice,
-            )
-            .unwrap(),
-        );
-
-        let spec = Arc::new(spec);
-
-        debug!("Creating validator producer and attester instances...");
-
-        // Spawn the test validator instances.
-        let validators: Vec<ValidatorHarness> = keypairs
-            .iter()
-            .map(|keypair| {
-                ValidatorHarness::new(keypair.clone(), beacon_chain.clone(), spec.clone())
-            })
-            .collect();
-
-        debug!("Created {} ValidatorHarnesss", validators.len());
-
-        Self {
-            db,
-            beacon_chain,
-            block_store,
-            state_store,
-            validators,
-            spec,
-        }
-    }
-
-    /// Move the `slot_clock` for the `BeaconChain` forward one slot.
-    ///
-    /// This is the equivalent of advancing a system clock forward one `SLOT_DURATION`.
-    ///
-    /// Returns the new slot.
-    pub fn increment_beacon_chain_slot(&mut self) -> Slot {
-        let slot = self.beacon_chain.present_slot() + 1;
-
-        let nth_slot = slot
-            - slot
-                .epoch(self.spec.slots_per_epoch)
-                .start_slot(self.spec.slots_per_epoch);
-        let nth_epoch = slot.epoch(self.spec.slots_per_epoch) - self.spec.genesis_epoch;
-        debug!(
-            "Advancing BeaconChain to slot {}, epoch {} (epoch height: {}, slot {} in epoch.).",
-            slot,
-            slot.epoch(self.spec.slots_per_epoch),
-            nth_epoch,
-            nth_slot
-        );
-
-        self.beacon_chain.slot_clock.set_slot(slot.as_u64());
-        self.beacon_chain
-            .catchup_state()
-            .expect("Failed to catch state");
-        slot
-    }
-
-    pub fn gather_attesations(&mut self) -> Vec<Attestation> {
-        let present_slot = self.beacon_chain.present_slot();
-        let state = self.beacon_chain.state.read();
-
-        let mut attestations = vec![];
-
-        for committee in state
-            .get_crosslink_committees_at_slot(present_slot, &self.spec)
-            .unwrap()
-        {
-            for &validator in &committee.committee {
-                let duties = state
-                    .get_attestation_duties(validator, &self.spec)
-                    .unwrap()
-                    .expect("Attesting validators by definition have duties");
-
-                // Obtain `AttestationData` from the beacon chain.
-                let data = self
-                    .beacon_chain
-                    .produce_attestation_data(duties.shard)
-                    .unwrap();
-
-                // Produce an aggregate signature with a single signature.
-                let aggregate_signature = {
-                    let message = AttestationDataAndCustodyBit {
-                        data: data.clone(),
-                        custody_bit: false,
-                    }
-                    .hash_tree_root();
-                    let domain = self.spec.get_domain(
-                        state.slot.epoch(self.spec.slots_per_epoch),
-                        Domain::Attestation,
-                        &state.fork,
-                    );
-                    let sig =
-                        Signature::new(&message, domain, &self.validators[validator].keypair.sk);
-
-                    let mut agg_sig = AggregateSignature::new();
-                    agg_sig.add(&sig);
-
-                    agg_sig
-                };
-
-                let mut aggregation_bitfield = Bitfield::with_capacity(duties.committee_len);
-                let custody_bitfield = Bitfield::with_capacity(duties.committee_len);
-
-                aggregation_bitfield.set(duties.committee_index, true);
-
-                attestations.push(Attestation {
-                    aggregation_bitfield,
-                    data,
-                    custody_bitfield,
-                    aggregate_signature,
-                })
-            }
-        }
-
-        attestations
-    }
-
-    /// Get the block from the proposer for the slot.
-    ///
-    /// Note: the validator will only produce it _once per slot_. So, if you call this twice you'll
-    /// only get a block once.
-    pub fn produce_block(&mut self) -> BeaconBlock {
-        let present_slot = self.beacon_chain.present_slot();
-
-        let proposer = self.beacon_chain.block_proposer(present_slot).unwrap();
-
-        debug!(
-            "Producing block from validator #{} for slot {}.",
-            proposer, present_slot
-        );
-
-        // Ensure the validators slot clock is accurate.
-        self.validators[proposer].set_slot(present_slot);
-
-        self.validators[proposer].produce_block().unwrap()
-    }
-
-    /// Advances the chain with a BeaconBlock and attestations from all validators.
-    ///
-    /// This is the ideal scenario for the Beacon Chain, 100% honest participation from
-    /// validators.
-    pub fn advance_chain_with_block(&mut self) -> BeaconBlock {
-        self.increment_beacon_chain_slot();
-
-        // Produce a new block.
-        let block = self.produce_block();
-        debug!("Submitting block for processing...");
-        match self.beacon_chain.process_block(block.clone()) {
-            Ok(BlockProcessingOutcome::ValidBlock(_)) => {}
-            other => panic!("block processing failed with {:?}", other),
-        };
-        debug!("...block processed by BeaconChain.");
-
-        debug!("Producing attestations...");
-
-        // Produce new attestations.
-        let attestations = self.gather_attesations();
-
-        debug!("Processing {} attestations...", attestations.len());
-
-        attestations
-            .par_iter()
-            .enumerate()
-            .for_each(|(i, attestation)| {
-                self.beacon_chain
-                    .process_attestation(attestation.clone())
-                    .unwrap_or_else(|_| panic!("Attestation {} invalid: {:?}", i, attestation));
-            });
-
-        debug!("Attestations processed.");
-
-        block
-    }
-
-    /// Signs a message using some validators secret key with the `Fork` info from the latest state
-    /// of the `BeaconChain`.
-    ///
-    /// Useful for producing slashable messages and other objects that `BeaconChainHarness` does
-    /// not produce naturally.
-    pub fn validator_sign(
-        &self,
-        validator_index: usize,
-        message: &[u8],
-        epoch: Epoch,
-        domain_type: Domain,
-    ) -> Option<Signature> {
-        let validator = self.validators.get(validator_index)?;
-
-        let domain = self
-            .spec
-            .get_domain(epoch, domain_type, &self.beacon_chain.state.read().fork);
-
-        Some(Signature::new(message, domain, &validator.keypair.sk))
-    }
-
-    /// Returns the current `Fork` of the `beacon_chain`.
-    pub fn fork(&self) -> Fork {
-        self.beacon_chain.state.read().fork.clone()
-    }
-
-    /// Returns the current `epoch` of the `beacon_chain`.
-    pub fn epoch(&self) -> Epoch {
-        self.beacon_chain
-            .state
-            .read()
-            .slot
-            .epoch(self.spec.slots_per_epoch)
-    }
-
-    /// Returns the keypair for some validator index.
-    pub fn validator_keypair(&self, validator_index: usize) -> Option<&Keypair> {
-        self.validators
-            .get(validator_index)
-            .and_then(|v| Some(&v.keypair))
-    }
-
-    /// Submit a deposit to the `BeaconChain` and, if given a keypair, create a new
-    /// `ValidatorHarness` instance for this validator.
-    ///
-    /// If a new `ValidatorHarness` was created, the validator should become fully operational as
-    /// if the validator were created during `BeaconChainHarness` instantiation.
-    pub fn add_deposit(&mut self, deposit: Deposit, keypair: Option<Keypair>) {
-        self.beacon_chain.process_deposit(deposit).unwrap();
-
-        // If a keypair is present, add a new `ValidatorHarness` to the rig.
-        if let Some(keypair) = keypair {
-            let validator =
-                ValidatorHarness::new(keypair, self.beacon_chain.clone(), self.spec.clone());
-            self.validators.push(validator);
-        }
-    }
-
-    /// Submit an exit to the `BeaconChain` for inclusion in some block.
-    ///
-    /// Note: the `ValidatorHarness` for this validator continues to exist. Once it is exited it
-    /// will stop receiving duties from the beacon chain and just do nothing when prompted to
-    /// produce/attest.
-    pub fn add_exit(&mut self, exit: VoluntaryExit) {
-        self.beacon_chain.process_voluntary_exit(exit).unwrap();
-    }
-
-    /// Submit an transfer to the `BeaconChain` for inclusion in some block.
-    pub fn add_transfer(&mut self, transfer: Transfer) {
-        self.beacon_chain.process_transfer(transfer).unwrap();
-    }
-
-    /// Submit a proposer slashing to the `BeaconChain` for inclusion in some block.
-    pub fn add_proposer_slashing(&mut self, proposer_slashing: ProposerSlashing) {
-        self.beacon_chain
-            .process_proposer_slashing(proposer_slashing)
-            .unwrap();
-    }
-
-    /// Submit an attester slashing to the `BeaconChain` for inclusion in some block.
-    pub fn add_attester_slashing(&mut self, attester_slashing: AttesterSlashing) {
-        self.beacon_chain
-            .process_attester_slashing(attester_slashing)
-            .unwrap();
-    }
-
-    /// Executes the fork choice rule on the `BeaconChain`, selecting a new canonical head.
-    pub fn run_fork_choice(&mut self) {
-        self.beacon_chain.fork_choice().unwrap()
-    }
-
-    /// Dump all blocks and states from the canonical beacon chain.
-    pub fn chain_dump(&self) -> Result<Vec<CheckPoint>, BeaconChainError> {
-        self.beacon_chain.chain_dump()
-    }
-}

beacon_node/beacon_chain/test_harness/src/bin.rs

@@ -1,102 +0,0 @@
-use clap::{App, Arg, SubCommand};
-use env_logger::{Builder, Env};
-use gen_keys::gen_keys;
-use run_test::run_test;
-use std::fs;
-use types::test_utils::keypairs_path;
-use types::ChainSpec;
-
-mod beacon_chain_harness;
-mod gen_keys;
-mod run_test;
-mod test_case;
-mod validator_harness;
-
-use validator_harness::ValidatorHarness;
-
-fn main() {
-    let validator_file_path = keypairs_path();
-
-    let _ = fs::create_dir(validator_file_path.parent().unwrap());
-
-    let matches = App::new("Lighthouse Test Harness Runner")
-        .version("0.0.1")
-        .author("Sigma Prime <contact@sigmaprime.io>")
-        .about("Runs `test_harness` using a YAML test_case.")
-        .arg(
-            Arg::with_name("log")
-                .long("log-level")
-                .short("l")
-                .value_name("LOG_LEVEL")
-                .help("Logging level.")
-                .possible_values(&["error", "warn", "info", "debug", "trace"])
-                .default_value("debug")
-                .required(true),
-        )
-        .arg(
-            Arg::with_name("spec")
-                .long("spec")
-                .short("s")
-                .value_name("SPECIFICATION")
-                .help("ChainSpec instantiation.")
-                .possible_values(&["foundation", "few_validators"])
-                .default_value("foundation"),
-        )
-        .subcommand(
-            SubCommand::with_name("run_test")
-                .about("Executes a YAML test specification")
-                .arg(
-                    Arg::with_name("yaml")
-                        .long("yaml")
-                        .value_name("FILE")
-                        .help("YAML file test_case.")
-                        .required(true),
-                )
-                .arg(
-                    Arg::with_name("validators_dir")
-                        .long("validators-dir")
-                        .short("v")
-                        .value_name("VALIDATORS_DIR")
-                        .help("A directory with validator deposits and keypair YAML."),
-                ),
-        )
-        .subcommand(
-            SubCommand::with_name("gen_keys")
-                .about("Builds a file of BLS keypairs for faster tests.")
-                .arg(
-                    Arg::with_name("validator_count")
-                        .long("validator_count")
-                        .short("n")
-                        .value_name("VALIDATOR_COUNT")
-                        .help("Number of validators to generate.")
-                        .required(true),
-                )
-                .arg(
-                    Arg::with_name("output_file")
-                        .long("output_file")
-                        .short("d")
-                        .value_name("GENESIS_TIME")
-                        .help("Output directory for generated YAML.")
-                        .default_value(validator_file_path.to_str().unwrap()),
-                ),
-        )
-        .get_matches();
-
-    if let Some(log_level) = matches.value_of("log") {
-        Builder::from_env(Env::default().default_filter_or(log_level)).init();
-    }
-
-    let _spec = match matches.value_of("spec") {
-        Some("foundation") => ChainSpec::foundation(),
-        Some("few_validators") => ChainSpec::few_validators(),
-        _ => unreachable!(), // Has a default value, should always exist.
-    };
-
-    if let Some(matches) = matches.subcommand_matches("run_test") {
-        run_test(matches);
-    }
-
-    if let Some(matches) = matches.subcommand_matches("gen_keys") {
-        gen_keys(matches);
-    }
-}

beacon_node/beacon_chain/test_harness/src/gen_keys.rs

@@ -1,21 +0,0 @@
-use clap::{value_t, ArgMatches};
-use log::debug;
-use std::path::Path;
-use types::test_utils::{generate_deterministic_keypairs, KeypairsFile};
-
-/// Creates a file containing BLS keypairs.
-pub fn gen_keys(matches: &ArgMatches) {
-    let validator_count = value_t!(matches.value_of("validator_count"), usize)
-        .expect("Validator count is required argument");
-    let output_file = matches
-        .value_of("output_file")
-        .expect("Output file has a default value.");
-
-    let keypairs = generate_deterministic_keypairs(validator_count);
-
-    debug!("Writing keypairs to file...");
-
-    let keypairs_path = Path::new(output_file);
-
-    keypairs.to_raw_file(&keypairs_path, &keypairs).unwrap();
-}

beacon_node/beacon_chain/test_harness/src/lib.rs

@@ -1,33 +0,0 @@
-//! Provides a testing environment for the `BeaconChain`, `Attester` and `BlockProposer` objects.
-//!
-//! This environment bypasses networking and client run-times and connects the `Attester` and `Proposer`
-//! directly to the `BeaconChain` via an `Arc`.
-//!
-//! The `BeaconChainHarness` contains a single `BeaconChain` instance and many `ValidatorHarness`
-//! instances. All of the `ValidatorHarness` instances work to advance the `BeaconChain` by
-//! producing blocks and attestations.
-//!
-//! Example:
-//! ```
-//! use test_harness::BeaconChainHarness;
-//! use types::ChainSpec;
-//!
-//! let validator_count = 8;
-//! let spec = ChainSpec::few_validators();
-//!
-//! let mut harness = BeaconChainHarness::new(spec, validator_count);
-//!
-//! harness.advance_chain_with_block();
-//!
-//! let chain = harness.chain_dump().unwrap();
-//!
-//! // One block should have been built on top of the genesis block.
-//! assert_eq!(chain.len(), 2);
-//! ```
-
-mod beacon_chain_harness;
-pub mod test_case;
-mod validator_harness;
-
-pub use self::beacon_chain_harness::BeaconChainHarness;
-pub use self::validator_harness::ValidatorHarness;

beacon_node/beacon_chain/test_harness/src/run_test.rs

@@ -1,37 +0,0 @@
-use crate::test_case::TestCase;
-use clap::ArgMatches;
-use std::{fs::File, io::prelude::*};
-use yaml_rust::YamlLoader;
-
-/// Runs a YAML-specified test case.
-pub fn run_test(matches: &ArgMatches) {
-    if let Some(yaml_file) = matches.value_of("yaml") {
-        let docs = {
-            let mut file = File::open(yaml_file).unwrap();
-
-            let mut yaml_str = String::new();
-            file.read_to_string(&mut yaml_str).unwrap();
-
-            YamlLoader::load_from_str(&yaml_str).unwrap()
-        };
-
-        for doc in &docs {
-            // For each `test_cases` YAML in the document, build a `TestCase`, execute it and
-            // assert that the execution result matches the test_case description.
-            //
-            // In effect, for each `test_case` a new `BeaconChainHarness` is created from genesis
-            // and a new `BeaconChain` is built as per the test_case.
-            //
-            // After the `BeaconChain` has been built out as per the test_case, a dump of all blocks
-            // and states in the chain is obtained and checked against the `results` specified in
-            // the `test_case`.
-            //
-            // If any of the expectations in the results are not met, the process
-            // panics with a message.
-            for test_case in doc["test_cases"].as_vec().unwrap() {
-                let test_case = TestCase::from_yaml(test_case);
-                test_case.assert_result_valid(test_case.execute())
-            }
-        }
-    }
-}

beacon_node/beacon_chain/test_harness/src/test_case.rs

@@ -1,312 +0,0 @@
-//! Defines execution and testing specs for a `BeaconChainHarness` instance. Supports loading from
-//! a YAML file.
-
-use crate::beacon_chain_harness::BeaconChainHarness;
-use beacon_chain::CheckPoint;
-use log::{info, warn};
-use ssz::SignedRoot;
-use types::*;
-
-use types::test_utils::*;
-use yaml_rust::Yaml;
-
-mod config;
-mod results;
-mod state_check;
-mod yaml_helpers;
-
-pub use config::Config;
-pub use results::Results;
-pub use state_check::StateCheck;
-
-/// Defines the execution and testing of a `BeaconChainHarness` instantiation.
-///
-/// Typical workflow is:
-///
-/// 1. Instantiate the `TestCase` from YAML: `let test_case = TestCase::from_yaml(&my_yaml);`
-/// 2. Execute the test_case: `let result = test_case.execute();`
-/// 3. Test the results against the test_case: `test_case.assert_result_valid(result);`
-#[derive(Debug)]
-pub struct TestCase {
-    /// Defines the execution.
-    pub config: Config,
-    /// Defines tests to run against the execution result.
-    pub results: Results,
-}
-
-/// The result of executing a `TestCase`.
-///
-pub struct ExecutionResult {
-    /// The canonical beacon chain generated from the execution.
-    pub chain: Vec<CheckPoint>,
-    /// The spec used for execution.
-    pub spec: ChainSpec,
-}
-
-impl TestCase {
-    /// Load the test case from a YAML document.
-    pub fn from_yaml(test_case: &Yaml) -> Self {
-        Self {
-            results: Results::from_yaml(&test_case["results"]),
-            config: Config::from_yaml(&test_case["config"]),
-        }
-    }
-
-    /// Return a `ChainSpec::foundation()`.
-    ///
-    /// If specified in `config`, returns it with a modified `slots_per_epoch`.
-    fn spec(&self) -> ChainSpec {
-        let mut spec = ChainSpec::foundation();
-
-        if let Some(n) = self.config.slots_per_epoch {
-            spec.slots_per_epoch = n;
-        }
-
-        if let Some(n) = self.config.persistent_committee_period {
-            spec.persistent_committee_period = n;
-        }
-
-        spec
-    }
-
-    /// Executes the test case, returning an `ExecutionResult`.
-    #[allow(clippy::cyclomatic_complexity)]
-    pub fn execute(&self) -> ExecutionResult {
-        let spec = self.spec();
-        let validator_count = self.config.deposits_for_chain_start;
-        let slots = self.config.num_slots;
-
-        info!(
-            "Building BeaconChainHarness with {} validators...",
-            validator_count
-        );
-
-        let mut harness = BeaconChainHarness::new(spec, validator_count);
-
-        info!("Starting simulation across {} slots...", slots);
-
-        // Start at 1 because genesis counts as a slot.
-        for slot_height in 1..slots {
-            // Used to ensure that deposits in the same slot have incremental deposit indices.
-            let mut deposit_index_offset = 0;
-
-            // Feed deposits to the BeaconChain.
-            if let Some(ref deposits) = self.config.deposits {
-                for (slot, amount) in deposits {
-                    if *slot == slot_height {
-                        info!("Including deposit at slot height {}.", slot_height);
-                        let (deposit, keypair) =
-                            build_deposit(&harness, *amount, deposit_index_offset);
-                        harness.add_deposit(deposit, Some(keypair.clone()));
-                        deposit_index_offset += 1;
-                    }
-                }
-            }
-
-            // Feed proposer slashings to the BeaconChain.
-            if let Some(ref slashings) = self.config.proposer_slashings {
-                for (slot, validator_index) in slashings {
-                    if *slot == slot_height {
-                        info!(
-                            "Including proposer slashing at slot height {} for validator #{}.",
-                            slot_height, validator_index
-                        );
-                        let slashing = build_proposer_slashing(&harness, *validator_index);
-                        harness.add_proposer_slashing(slashing);
-                    }
-                }
-            }
-
-            // Feed attester slashings to the BeaconChain.
-            if let Some(ref slashings) = self.config.attester_slashings {
-                for (slot, validator_indices) in slashings {
-                    if *slot == slot_height {
-                        info!(
-                            "Including attester slashing at slot height {} for validators {:?}.",
-                            slot_height, validator_indices
-                        );
-                        let slashing =
-                            build_double_vote_attester_slashing(&harness, &validator_indices[..]);
-                        harness.add_attester_slashing(slashing);
-                    }
-                }
-            }
-
-            // Feed exits to the BeaconChain.
-            if let Some(ref exits) = self.config.exits {
-                for (slot, validator_index) in exits {
-                    if *slot == slot_height {
-                        info!(
-                            "Including exit at slot height {} for validator {}.",
-                            slot_height, validator_index
-                        );
-                        let exit = build_exit(&harness, *validator_index);
-                        harness.add_exit(exit);
-                    }
-                }
-            }
-
-            // Feed transfers to the BeaconChain.
-            if let Some(ref transfers) = self.config.transfers {
-                for (slot, from, to, amount) in transfers {
-                    if *slot == slot_height {
-                        info!(
-                            "Including transfer at slot height {} from validator {}.",
-                            slot_height, from
-                        );
-                        let transfer = build_transfer(&harness, *from, *to, *amount);
-                        harness.add_transfer(transfer);
-                    }
-                }
-            }
-
-            // Build a block or skip a slot.
-            match self.config.skip_slots {
-                Some(ref skip_slots) if skip_slots.contains(&slot_height) => {
-                    warn!("Skipping slot at height {}.", slot_height);
-                    harness.increment_beacon_chain_slot();
-                }
-                _ => {
-                    info!("Producing block at slot height {}.", slot_height);
-                    harness.advance_chain_with_block();
-                }
-            }
-        }
-
-        harness.run_fork_choice();
-
-        info!("Test execution complete!");
-
-        info!("Building chain dump for analysis...");
-
-        ExecutionResult {
-            chain: harness.chain_dump().expect("Chain dump failed."),
-            spec: (*harness.spec).clone(),
-        }
-    }
-
-    /// Checks that the `ExecutionResult` is consistent with the specifications in `self.results`.
-    ///
-    /// # Panics
-    ///
-    /// Panics with a message if any result does not match exepectations.
-    pub fn assert_result_valid(&self, execution_result: ExecutionResult) {
-        info!("Verifying test results...");
-        let spec = &execution_result.spec;
-
-        if let Some(num_skipped_slots) = self.results.num_skipped_slots {
-            assert_eq!(
-                execution_result.chain.len(),
-                self.config.num_slots as usize - num_skipped_slots,
-                "actual skipped slots != expected."
-            );
-            info!(
-                "OK: Chain length is {} ({} skipped slots).",
-                execution_result.chain.len(),
-                num_skipped_slots
-            );
-        }
-
-        if let Some(ref state_checks) = self.results.state_checks {
-            for checkpoint in &execution_result.chain {
-                let state = &checkpoint.beacon_state;
-
-                for state_check in state_checks {
-                    let adjusted_state_slot =
-                        state.slot - spec.genesis_epoch.start_slot(spec.slots_per_epoch);
-
-                    if state_check.slot == adjusted_state_slot {
-                        state_check.assert_valid(state, spec);
-                    }
-                }
-            }
-        }
-    }
-}
-
-/// Builds a `Deposit` this is valid for the given `BeaconChainHarness` at its next slot.
-fn build_transfer(
-    harness: &BeaconChainHarness,
-    sender: u64,
-    recipient: u64,
-    amount: u64,
-) -> Transfer {
-    let slot = harness.beacon_chain.state.read().slot + 1;
-
-    let mut builder = TestingTransferBuilder::new(sender, recipient, amount, slot);
-
-    let keypair = harness.validator_keypair(sender as usize).unwrap();
-    builder.sign(keypair.clone(), &harness.fork(), &harness.spec);
-
-    builder.build()
-}
-
-/// Builds a `Deposit` this is valid for the given `BeaconChainHarness`.
-///
-/// `index_offset` is used to ensure that `deposit.index == state.index` when adding multiple
-/// deposits.
-fn build_deposit(
-    harness: &BeaconChainHarness,
-    amount: u64,
-    index_offset: u64,
-) -> (Deposit, Keypair) {
-    let keypair = Keypair::random();
-
-    let mut builder = TestingDepositBuilder::new(keypair.pk.clone(), amount);
-    builder.set_index(harness.beacon_chain.state.read().deposit_index + index_offset);
-    builder.sign(&keypair, harness.epoch(), &harness.fork(), &harness.spec);
-
-    (builder.build(), keypair)
-}
-
-/// Builds a `VoluntaryExit` this is valid for the given `BeaconChainHarness`.
-fn build_exit(harness: &BeaconChainHarness, validator_index: u64) -> VoluntaryExit {
-    let epoch = harness
-        .beacon_chain
-        .state
-        .read()
-        .current_epoch(&harness.spec);
-
-    let mut exit = VoluntaryExit {
-        epoch,
-        validator_index,
-        signature: Signature::empty_signature(),
-    };
-
-    let message = exit.signed_root();
-
-    exit.signature = harness
-        .validator_sign(validator_index as usize, &message[..], epoch, Domain::Exit)
-        .expect("Unable to sign VoluntaryExit");
-
-    exit
-}
-
-/// Builds an `AttesterSlashing` for some `validator_indices`.
-///
-/// Signs the message using a `BeaconChainHarness`.
-fn build_double_vote_attester_slashing(
-    harness: &BeaconChainHarness,
-    validator_indices: &[u64],
-) -> AttesterSlashing {
-    let signer = |validator_index: u64, message: &[u8], epoch: Epoch, domain: Domain| {
-        harness
-            .validator_sign(validator_index as usize, message, epoch, domain)
-            .expect("Unable to sign AttesterSlashing")
-    };
-
-    TestingAttesterSlashingBuilder::double_vote(validator_indices, signer)
-}
-
-/// Builds an `ProposerSlashing` for some `validator_index`.
-///
-/// Signs the message using a `BeaconChainHarness`.
-fn build_proposer_slashing(harness: &BeaconChainHarness, validator_index: u64) -> ProposerSlashing {
-    let signer = |validator_index: u64, message: &[u8], epoch: Epoch, domain: Domain| {
-        harness
-            .validator_sign(validator_index as usize, message, epoch, domain)
-            .expect("Unable to sign AttesterSlashing")
-    };
-
-    TestingProposerSlashingBuilder::double_vote(validator_index, signer, &harness.spec)
-}

beacon_node/beacon_chain/test_harness/src/test_case/config.rs

@@ -1,135 +0,0 @@
-use super::yaml_helpers::{as_u64, as_usize, as_vec_u64};
-use types::*;
-use yaml_rust::Yaml;
-
-pub type ValidatorIndex = u64;
-pub type ValidatorIndices = Vec<u64>;
-pub type GweiAmount = u64;
-
-pub type DepositTuple = (SlotHeight, GweiAmount);
-pub type ExitTuple = (SlotHeight, ValidatorIndex);
-pub type ProposerSlashingTuple = (SlotHeight, ValidatorIndex);
-pub type AttesterSlashingTuple = (SlotHeight, ValidatorIndices);
-/// (slot_height, from, to, amount)
-pub type TransferTuple = (SlotHeight, ValidatorIndex, ValidatorIndex, GweiAmount);
-
-/// Defines the execution of a `BeaconStateHarness` across a series of slots.
-#[derive(Debug)]
-pub struct Config {
-    /// Initial validators.
-    pub deposits_for_chain_start: usize,
-    /// Number of slots in an epoch.
-    pub slots_per_epoch: Option<u64>,
-    /// Affects the number of epochs a validator must be active before they can withdraw.
-    pub persistent_committee_period: Option<u64>,
-    /// Number of slots to build before ending execution.
-    pub num_slots: u64,
-    /// Number of slots that should be skipped due to inactive validator.
-    pub skip_slots: Option<Vec<u64>>,
-    /// Deposits to be included during execution.
-    pub deposits: Option<Vec<DepositTuple>>,
-    /// Proposer slashings to be included during execution.
-    pub proposer_slashings: Option<Vec<ProposerSlashingTuple>>,
-    /// Attester slashings to be including during execution.
-    pub attester_slashings: Option<Vec<AttesterSlashingTuple>>,
-    /// Exits to be including during execution.
-    pub exits: Option<Vec<ExitTuple>>,
-    /// Transfers to be including during execution.
-    pub transfers: Option<Vec<TransferTuple>>,
-}
-
-impl Config {
-    /// Load from a YAML document.
-    ///
-    /// Expects to receive the `config` section of the document.
-    pub fn from_yaml(yaml: &Yaml) -> Self {
-        Self {
-            deposits_for_chain_start: as_usize(&yaml, "deposits_for_chain_start")
-                .expect("Must specify validator count"),
-            slots_per_epoch: as_u64(&yaml, "slots_per_epoch"),
-            persistent_committee_period: as_u64(&yaml, "persistent_committee_period"),
-            num_slots: as_u64(&yaml, "num_slots").expect("Must specify `config.num_slots`"),
-            skip_slots: as_vec_u64(yaml, "skip_slots"),
-            deposits: parse_deposits(&yaml),
-            proposer_slashings: parse_proposer_slashings(&yaml),
-            attester_slashings: parse_attester_slashings(&yaml),
-            exits: parse_exits(&yaml),
-            transfers: parse_transfers(&yaml),
-        }
-    }
-}
-
-/// Parse the `transfers` section of the YAML document.
-fn parse_transfers(yaml: &Yaml) -> Option<Vec<TransferTuple>> {
-    let mut tuples = vec![];
-
-    for exit in yaml["transfers"].as_vec()? {
-        let slot = as_u64(exit, "slot").expect("Incomplete transfer (slot)");
-        let from = as_u64(exit, "from").expect("Incomplete transfer (from)");
-        let to = as_u64(exit, "to").expect("Incomplete transfer (to)");
-        let amount = as_u64(exit, "amount").expect("Incomplete transfer (amount)");
-
-        tuples.push((SlotHeight::from(slot), from, to, amount));
-    }
-
-    Some(tuples)
-}
-
-/// Parse the `attester_slashings` section of the YAML document.
-fn parse_exits(yaml: &Yaml) -> Option<Vec<ExitTuple>> {
-    let mut tuples = vec![];
-
-    for exit in yaml["exits"].as_vec()? {
-        let slot = as_u64(exit, "slot").expect("Incomplete exit (slot)");
-        let validator_index =
-            as_u64(exit, "validator_index").expect("Incomplete exit (validator_index)");
-
-        tuples.push((SlotHeight::from(slot), validator_index));
-    }
-
-    Some(tuples)
-}
-
-/// Parse the `attester_slashings` section of the YAML document.
-fn parse_attester_slashings(yaml: &Yaml) -> Option<Vec<AttesterSlashingTuple>> {
-    let mut slashings = vec![];
-
-    for slashing in yaml["attester_slashings"].as_vec()? {
-        let slot = as_u64(slashing, "slot").expect("Incomplete attester_slashing (slot)");
-        let validator_indices = as_vec_u64(slashing, "validator_indices")
-            .expect("Incomplete attester_slashing (validator_indices)");
-
-        slashings.push((SlotHeight::from(slot), validator_indices));
-    }
-
-    Some(slashings)
-}
-
-/// Parse the `proposer_slashings` section of the YAML document.
-fn parse_proposer_slashings(yaml: &Yaml) -> Option<Vec<ProposerSlashingTuple>> {
-    let mut slashings = vec![];
-
-    for slashing in yaml["proposer_slashings"].as_vec()? {
-        let slot = as_u64(slashing, "slot").expect("Incomplete proposer slashing (slot)_");
-        let validator_index = as_u64(slashing, "validator_index")
-            .expect("Incomplete proposer slashing (validator_index)");
-
-        slashings.push((SlotHeight::from(slot), validator_index));
-    }
-
-    Some(slashings)
-}
-
-/// Parse the `deposits` section of the YAML document.
-fn parse_deposits(yaml: &Yaml) -> Option<Vec<DepositTuple>> {
-    let mut deposits = vec![];
-
-    for deposit in yaml["deposits"].as_vec()? {
-        let slot = as_u64(deposit, "slot").expect("Incomplete deposit (slot)");
-        let amount = as_u64(deposit, "amount").expect("Incomplete deposit (amount)");
-
-        deposits.push((SlotHeight::from(slot), amount))
-    }
-
-    Some(deposits)
-}

beacon_node/beacon_chain/test_harness/src/test_case/results.rs

@@ -1,34 +0,0 @@
-use super::state_check::StateCheck;
-use super::yaml_helpers::as_usize;
-use yaml_rust::Yaml;
-
-/// A series of tests to be carried out upon an `ExecutionResult`, returned from executing a
-/// `TestCase`.
-#[derive(Debug)]
-pub struct Results {
-    pub num_skipped_slots: Option<usize>,
-    pub state_checks: Option<Vec<StateCheck>>,
-}
-
-impl Results {
-    /// Load from a YAML document.
-    ///
-    /// Expects the `results` section of the YAML document.
-    pub fn from_yaml(yaml: &Yaml) -> Self {
-        Self {
-            num_skipped_slots: as_usize(yaml, "num_skipped_slots"),
-            state_checks: parse_state_checks(yaml),
-        }
-    }
-}
-
-/// Parse the `state_checks` section of the YAML document.
-fn parse_state_checks(yaml: &Yaml) -> Option<Vec<StateCheck>> {
-    let mut states = vec![];
-
-    for state_yaml in yaml["states"].as_vec()? {
-        states.push(StateCheck::from_yaml(state_yaml));
-    }
-
-    Some(states)
-}

beacon_node/beacon_chain/test_harness/src/test_case/state_check.rs

@@ -1,206 +0,0 @@
-use super::yaml_helpers::{as_u64, as_usize, as_vec_u64};
-use log::info;
-use types::*;
-use yaml_rust::Yaml;
-
-type ValidatorIndex = u64;
-type BalanceGwei = u64;
-
-type BalanceCheckTuple = (ValidatorIndex, String, BalanceGwei);
-
-/// Tests to be conducted upon a `BeaconState` object generated during the execution of a
-/// `TestCase`.
-#[derive(Debug)]
-pub struct StateCheck {
-    /// Checked against `beacon_state.slot`.
-    pub slot: Slot,
-    /// Checked against `beacon_state.validator_registry.len()`.
-    pub num_validators: Option<usize>,
-    /// The number of pending attestations from the previous epoch that should be in the state.
-    pub num_previous_epoch_attestations: Option<usize>,
-    /// The number of pending attestations from the current epoch that should be in the state.
-    pub num_current_epoch_attestations: Option<usize>,
-    /// A list of validator indices which have been penalized. Must be in ascending order.
-    pub slashed_validators: Option<Vec<u64>>,
-    /// A list of validator indices which have been fully exited. Must be in ascending order.
-    pub exited_validators: Option<Vec<u64>>,
-    /// A list of validator indices which have had an exit initiated. Must be in ascending order.
-    pub exit_initiated_validators: Option<Vec<u64>>,
-    /// A list of balances to check.
-    pub balances: Option<Vec<BalanceCheckTuple>>,
-}
-
-impl StateCheck {
-    /// Load from a YAML document.
-    ///
-    /// Expects the `state_check` section of the YAML document.
-    pub fn from_yaml(yaml: &Yaml) -> Self {
-        Self {
-            slot: Slot::from(as_u64(&yaml, "slot").expect("State must specify slot")),
-            num_validators: as_usize(&yaml, "num_validators"),
-            num_previous_epoch_attestations: as_usize(&yaml, "num_previous_epoch_attestations"),
-            num_current_epoch_attestations: as_usize(&yaml, "num_current_epoch_attestations"),
-            slashed_validators: as_vec_u64(&yaml, "slashed_validators"),
-            exited_validators: as_vec_u64(&yaml, "exited_validators"),
-            exit_initiated_validators: as_vec_u64(&yaml, "exit_initiated_validators"),
-            balances: parse_balances(&yaml),
-        }
-    }
-
-    /// Performs all checks against a `BeaconState`
-    ///
-    /// # Panics
-    ///
-    /// Panics with an error message if any test fails.
-    #[allow(clippy::cyclomatic_complexity)]
-    pub fn assert_valid(&self, state: &BeaconState, spec: &ChainSpec) {
-        let state_epoch = state.slot.epoch(spec.slots_per_epoch);
-
-        info!("Running state check for slot height {}.", self.slot);
-
-        // Check the state slot.
-        assert_eq!(
-            self.slot,
-            state.slot - spec.genesis_epoch.start_slot(spec.slots_per_epoch),
-            "State slot is invalid."
-        );
-
-        // Check the validator count
-        if let Some(num_validators) = self.num_validators {
-            assert_eq!(
-                state.validator_registry.len(),
-                num_validators,
-                "State validator count != expected."
-            );
-            info!("OK: num_validators = {}.", num_validators);
-        }
-
-        // Check the previous epoch attestations
-        if let Some(n) = self.num_previous_epoch_attestations {
-            assert_eq!(
-                state.previous_epoch_attestations.len(),
-                n,
-                "previous epoch attestations count != expected."
-            );
-            info!("OK: num_previous_epoch_attestations = {}.", n);
-        }
-
-        // Check the current epoch attestations
-        if let Some(n) = self.num_current_epoch_attestations {
-            assert_eq!(
-                state.current_epoch_attestations.len(),
-                n,
-                "current epoch attestations count != expected."
-            );
-            info!("OK: num_current_epoch_attestations = {}.", n);
-        }
-
-        // Check for slashed validators.
-        if let Some(ref slashed_validators) = self.slashed_validators {
-            let actually_slashed_validators: Vec<u64> = state
-                .validator_registry
-                .iter()
-                .enumerate()
-                .filter_map(|(i, validator)| {
-                    if validator.slashed {
-                        Some(i as u64)
-                    } else {
-                        None
-                    }
-                })
-                .collect();
-            assert_eq!(
-                actually_slashed_validators, *slashed_validators,
-                "Slashed validators != expected."
-            );
-            info!("OK: slashed_validators = {:?}.", slashed_validators);
-        }
-
-        // Check for exited validators.
-        if let Some(ref exited_validators) = self.exited_validators {
-            let actually_exited_validators: Vec<u64> = state
-                .validator_registry
-                .iter()
-                .enumerate()
-                .filter_map(|(i, validator)| {
-                    if validator.is_exited_at(state_epoch) {
-                        Some(i as u64)
-                    } else {
-                        None
-                    }
-                })
-                .collect();
-            assert_eq!(
-                actually_exited_validators, *exited_validators,
-                "Exited validators != expected."
-            );
-            info!("OK: exited_validators = {:?}.", exited_validators);
-        }
-
-        // Check for validators that have initiated exit.
-        if let Some(ref exit_initiated_validators) = self.exit_initiated_validators {
-            let actual: Vec<u64> = state
-                .validator_registry
-                .iter()
-                .enumerate()
-                .filter_map(|(i, validator)| {
-                    if validator.initiated_exit {
-                        Some(i as u64)
-                    } else {
-                        None
-                    }
-                })
-                .collect();
-            assert_eq!(
-                actual, *exit_initiated_validators,
-                "Exit initiated validators != expected."
-            );
-            info!(
-                "OK: exit_initiated_validators = {:?}.",
-                exit_initiated_validators
-            );
-        }
-
-        // Check validator balances.
-        if let Some(ref balances) = self.balances {
-            for (index, comparison, expected) in balances {
-                let actual = *state
-                    .validator_balances
-                    .get(*index as usize)
-                    .expect("Balance check specifies unknown validator");
-
-                let result = match comparison.as_ref() {
-                    "eq" => actual == *expected,
-                    _ => panic!("Unknown balance comparison (use `eq`)"),
-                };
-                assert!(
-                    result,
-                    format!(
-                        "Validator balance for {}: {} !{} {}.",
-                        index, actual, comparison, expected
-                    )
-                );
-                info!("OK: validator balance for {:?}.", index);
-            }
-        }
-    }
-}
-
-/// Parse the `transfers` section of the YAML document.
-fn parse_balances(yaml: &Yaml) -> Option<Vec<BalanceCheckTuple>> {
-    let mut tuples = vec![];
-
-    for exit in yaml["balances"].as_vec()? {
-        let from =
-            as_u64(exit, "validator_index").expect("Incomplete balance check (validator_index)");
-        let comparison = exit["comparison"]
-            .clone()
-            .into_string()
-            .expect("Incomplete balance check (amount)");
-        let balance = as_u64(exit, "balance").expect("Incomplete balance check (balance)");
-
-        tuples.push((from, comparison, balance));
-    }
-
-    Some(tuples)
-}

beacon_node/beacon_chain/test_harness/src/test_case/yaml_helpers.rs

@@ -1,19 +0,0 @@
-use yaml_rust::Yaml;
-
-pub fn as_usize(yaml: &Yaml, key: &str) -> Option<usize> {
-    yaml[key].as_i64().and_then(|n| Some(n as usize))
-}
-
-pub fn as_u64(yaml: &Yaml, key: &str) -> Option<u64> {
-    yaml[key].as_i64().and_then(|n| Some(n as u64))
-}
-
-pub fn as_vec_u64(yaml: &Yaml, key: &str) -> Option<Vec<u64>> {
-    yaml[key].clone().into_vec().and_then(|vec| {
-        Some(
-            vec.iter()
-                .map(|item| item.as_i64().unwrap() as u64)
-                .collect(),
-        )
-    })
-}

beacon_node/beacon_chain/test_harness/src/validator_harness/direct_beacon_node.rs

@@ -1,100 +0,0 @@
-use attester::{
-    BeaconNode as AttesterBeaconNode, BeaconNodeError as NodeError,
-    PublishOutcome as AttestationPublishOutcome,
-};
-use beacon_chain::BeaconChain;
-use block_proposer::{
-    BeaconNode as BeaconBlockNode, BeaconNodeError as BeaconBlockNodeError,
-    PublishOutcome as BlockPublishOutcome,
-};
-use db::ClientDB;
-use fork_choice::ForkChoice;
-use parking_lot::RwLock;
-use slot_clock::SlotClock;
-use std::sync::Arc;
-use types::{AttestationData, BeaconBlock, FreeAttestation, Signature, Slot};
-
-/// Connect directly to a borrowed `BeaconChain` instance so an attester/producer can request/submit
-/// blocks/attestations.
-///
-/// `BeaconBlock`s and `FreeAttestation`s are not actually published to the `BeaconChain`, instead
-/// they are stored inside this struct. This is to allow one to benchmark the submission of the
-/// block/attestation directly, or modify it before submission.
-pub struct DirectBeaconNode<T: ClientDB, U: SlotClock, F: ForkChoice> {
-    beacon_chain: Arc<BeaconChain<T, U, F>>,
-    published_blocks: RwLock<Vec<BeaconBlock>>,
-    published_attestations: RwLock<Vec<FreeAttestation>>,
-}
-
-impl<T: ClientDB, U: SlotClock, F: ForkChoice> DirectBeaconNode<T, U, F> {
-    pub fn new(beacon_chain: Arc<BeaconChain<T, U, F>>) -> Self {
-        Self {
-            beacon_chain,
-            published_blocks: RwLock::new(vec![]),
-            published_attestations: RwLock::new(vec![]),
-        }
-    }
-
-    /// Get the last published block (if any).
-    pub fn last_published_block(&self) -> Option<BeaconBlock> {
-        Some(self.published_blocks.read().last()?.clone())
-    }
-}
-
-impl<T: ClientDB, U: SlotClock, F: ForkChoice> AttesterBeaconNode for DirectBeaconNode<T, U, F> {
-    fn produce_attestation_data(
-        &self,
-        _slot: Slot,
-        shard: u64,
-    ) -> Result<Option<AttestationData>, NodeError> {
-        match self.beacon_chain.produce_attestation_data(shard) {
-            Ok(attestation_data) => Ok(Some(attestation_data)),
-            Err(e) => Err(NodeError::RemoteFailure(format!("{:?}", e))),
-        }
-    }
-
-    fn publish_attestation(
-        &self,
-        free_attestation: FreeAttestation,
-    ) -> Result<AttestationPublishOutcome, NodeError> {
-        self.published_attestations.write().push(free_attestation);
-        Ok(AttestationPublishOutcome::ValidAttestation)
-    }
-}
-
-impl<T: ClientDB, U: SlotClock, F: ForkChoice> BeaconBlockNode for DirectBeaconNode<T, U, F> {
-    /// Requests a new `BeaconBlock from the `BeaconChain`.
-    fn produce_beacon_block(
-        &self,
-        slot: Slot,
-        randao_reveal: &Signature,
-    ) -> Result<Option<BeaconBlock>, BeaconBlockNodeError> {
-        let (block, _state) = self
-            .beacon_chain
-            .produce_block(randao_reveal.clone())
-            .map_err(|e| {
-                BeaconBlockNodeError::RemoteFailure(format!("Did not produce block: {:?}", e))
-            })?;
-
-        if block.slot == slot {
-            Ok(Some(block))
-        } else {
-            Err(BeaconBlockNodeError::RemoteFailure(
-                "Unable to produce at non-current slot.".to_string(),
-            ))
-        }
-    }
-
-    /// A block is not _actually_ published to the `BeaconChain`, instead it is stored in the
-    /// `published_block_vec` and a successful `ValidBlock` is returned to the caller.
-    ///
-    /// The block may be retrieved and then applied to the `BeaconChain` manually, potentially in a
-    /// benchmarking scenario.
-    fn publish_beacon_block(
-        &self,
-        block: BeaconBlock,
-    ) -> Result<BlockPublishOutcome, BeaconBlockNodeError> {
-        self.published_blocks.write().push(block);
-        Ok(BlockPublishOutcome::ValidBlock)
-    }
-}

beacon_node/beacon_chain/test_harness/src/validator_harness/direct_duties.rs

@@ -1,74 +0,0 @@
-use attester::{
-    DutiesReader as AttesterDutiesReader, DutiesReaderError as AttesterDutiesReaderError,
-};
-use beacon_chain::BeaconChain;
-use block_proposer::{
-    DutiesReader as ProducerDutiesReader, DutiesReaderError as ProducerDutiesReaderError,
-};
-use db::ClientDB;
-use fork_choice::ForkChoice;
-use slot_clock::SlotClock;
-use std::sync::Arc;
-use types::{Fork, PublicKey, Slot};
-
-/// Connects directly to a borrowed `BeaconChain` and reads attester/proposer duties directly from
-/// it.
-pub struct DirectDuties<T: ClientDB, U: SlotClock, F: ForkChoice> {
-    beacon_chain: Arc<BeaconChain<T, U, F>>,
-    pubkey: PublicKey,
-}
-
-impl<T: ClientDB, U: SlotClock, F: ForkChoice> DirectDuties<T, U, F> {
-    pub fn new(pubkey: PublicKey, beacon_chain: Arc<BeaconChain<T, U, F>>) -> Self {
-        Self {
-            beacon_chain,
-            pubkey,
-        }
-    }
-}
-
-impl<T: ClientDB, U: SlotClock, F: ForkChoice> ProducerDutiesReader for DirectDuties<T, U, F> {
-    fn is_block_production_slot(&self, slot: Slot) -> Result<bool, ProducerDutiesReaderError> {
-        let validator_index = self
-            .beacon_chain
-            .validator_index(&self.pubkey)
-            .ok_or_else(|| ProducerDutiesReaderError::UnknownValidator)?;
-
-        match self.beacon_chain.block_proposer(slot) {
-            Ok(proposer) if proposer == validator_index => Ok(true),
-            Ok(_) => Ok(false),
-            Err(_) => Err(ProducerDutiesReaderError::UnknownEpoch),
-        }
-    }
-
-    fn fork(&self) -> Result<Fork, ProducerDutiesReaderError> {
-        Ok(self.beacon_chain.state.read().fork.clone())
-    }
-}
-
-impl<T: ClientDB, U: SlotClock, F: ForkChoice> AttesterDutiesReader for DirectDuties<T, U, F> {
-    fn validator_index(&self) -> Option<u64> {
-        match self.beacon_chain.validator_index(&self.pubkey) {
-            Some(index) => Some(index as u64),
-            None => None,
-        }
-    }
-
-    fn attestation_shard(&self, slot: Slot) -> Result<Option<u64>, AttesterDutiesReaderError> {
-        if let Some(validator_index) = self.validator_index() {
-            match self
-                .beacon_chain
-                .validator_attestion_slot_and_shard(validator_index as usize)
-            {
-                Ok(Some((attest_slot, attest_shard))) if attest_slot == slot => {
-                    Ok(Some(attest_shard))
-                }
-                Ok(Some(_)) => Ok(None),
-                Ok(None) => Err(AttesterDutiesReaderError::UnknownEpoch),
-                Err(_) => unreachable!("Error when getting validator attestation shard."),
-            }
-        } else {
-            Err(AttesterDutiesReaderError::UnknownValidator)
-        }
-    }
-}

beacon_node/beacon_chain/test_harness/src/validator_harness/local_signer.rs

@@ -1,36 +0,0 @@
-use attester::Signer as AttesterSigner;
-use block_proposer::Signer as BlockProposerSigner;
-use types::{Keypair, Signature};
-
-/// A test-only struct used to perform signing for a proposer or attester.
-pub struct LocalSigner {
-    keypair: Keypair,
-}
-
-impl LocalSigner {
-    /// Produce a new TestSigner with signing enabled by default.
-    pub fn new(keypair: Keypair) -> Self {
-        Self { keypair }
-    }
-
-    /// Sign some message.
-    fn bls_sign(&self, message: &[u8], domain: u64) -> Option<Signature> {
-        Some(Signature::new(message, domain, &self.keypair.sk))
-    }
-}
-
-impl BlockProposerSigner for LocalSigner {
-    fn sign_block_proposal(&self, message: &[u8], domain: u64) -> Option<Signature> {
-        self.bls_sign(message, domain)
-    }
-
-    fn sign_randao_reveal(&self, message: &[u8], domain: u64) -> Option<Signature> {
-        self.bls_sign(message, domain)
-    }
-}
-
-impl AttesterSigner for LocalSigner {
-    fn sign_attestation_message(&self, message: &[u8], domain: u64) -> Option<Signature> {
-        self.bls_sign(message, domain)
-    }
-}

beacon_node/beacon_chain/test_harness/src/validator_harness/mod.rs

@@ -1,119 +0,0 @@
-mod direct_beacon_node;
-mod direct_duties;
-mod local_signer;
-
-use attester::Attester;
-use beacon_chain::BeaconChain;
-use block_proposer::PollOutcome as BlockPollOutcome;
-use block_proposer::{BlockProducer, Error as BlockPollError};
-use db::MemoryDB;
-use direct_beacon_node::DirectBeaconNode;
-use direct_duties::DirectDuties;
-use fork_choice::BitwiseLMDGhost;
-use local_signer::LocalSigner;
-use slot_clock::TestingSlotClock;
-use std::sync::Arc;
-use types::{BeaconBlock, ChainSpec, Keypair, Slot};
-
-#[derive(Debug, PartialEq)]
-pub enum BlockProduceError {
-    DidNotProduce(BlockPollOutcome),
-    PollError(BlockPollError),
-}
-
-type TestingBlockProducer = BlockProducer<
-    TestingSlotClock,
-    DirectBeaconNode<MemoryDB, TestingSlotClock, BitwiseLMDGhost<MemoryDB>>,
-    DirectDuties<MemoryDB, TestingSlotClock, BitwiseLMDGhost<MemoryDB>>,
-    LocalSigner,
->;
-
-type TestingAttester = Attester<
-    TestingSlotClock,
-    DirectBeaconNode<MemoryDB, TestingSlotClock, BitwiseLMDGhost<MemoryDB>>,
-    DirectDuties<MemoryDB, TestingSlotClock, BitwiseLMDGhost<MemoryDB>>,
-    LocalSigner,
->;
-
-/// A `BlockProducer` and `Attester` which sign using a common keypair.
-///
-/// The test validator connects directly to a borrowed `BeaconChain` struct. It is useful for
-/// testing that the core proposer and attester logic is functioning. Also for supporting beacon
-/// chain tests.
-pub struct ValidatorHarness {
-    pub block_producer: TestingBlockProducer,
-    pub attester: TestingAttester,
-    pub spec: Arc<ChainSpec>,
-    pub epoch_map: Arc<DirectDuties<MemoryDB, TestingSlotClock, BitwiseLMDGhost<MemoryDB>>>,
-    pub keypair: Keypair,
-    pub beacon_node: Arc<DirectBeaconNode<MemoryDB, TestingSlotClock, BitwiseLMDGhost<MemoryDB>>>,
-    pub slot_clock: Arc<TestingSlotClock>,
-    pub signer: Arc<LocalSigner>,
-}
-
-impl ValidatorHarness {
-    /// Create a new ValidatorHarness that signs with the given keypair, operates per the given spec and connects to the
-    /// supplied beacon node.
-    ///
-    /// A `BlockProducer` and `Attester` is created..
-    pub fn new(
-        keypair: Keypair,
-        beacon_chain: Arc<BeaconChain<MemoryDB, TestingSlotClock, BitwiseLMDGhost<MemoryDB>>>,
-        spec: Arc<ChainSpec>,
-    ) -> Self {
-        let slot_clock = Arc::new(TestingSlotClock::new(spec.genesis_slot.as_u64()));
-        let signer = Arc::new(LocalSigner::new(keypair.clone()));
-        let beacon_node = Arc::new(DirectBeaconNode::new(beacon_chain.clone()));
-        let epoch_map = Arc::new(DirectDuties::new(keypair.pk.clone(), beacon_chain.clone()));
-
-        let block_producer = BlockProducer::new(
-            spec.clone(),
-            epoch_map.clone(),
-            slot_clock.clone(),
-            beacon_node.clone(),
-            signer.clone(),
-        );
-
-        let attester = Attester::new(
-            epoch_map.clone(),
-            slot_clock.clone(),
-            beacon_node.clone(),
-            signer.clone(),
-        );
-
-        Self {
-            block_producer,
-            attester,
-            spec,
-            epoch_map,
-            keypair,
-            beacon_node,
-            slot_clock,
-            signer,
-        }
-    }
-
-    /// Run the `poll` function on the `BlockProducer` and produce a block.
-    ///
-    /// An error is returned if the producer refuses to produce.
-    pub fn produce_block(&mut self) -> Result<BeaconBlock, BlockProduceError> {
-        // Using `DirectBeaconNode`, the validator will always return sucessufully if it tries to
-        // publish a block.
-        match self.block_producer.poll() {
-            Ok(BlockPollOutcome::BlockProduced(_)) => {}
-            Ok(outcome) => return Err(BlockProduceError::DidNotProduce(outcome)),
-            Err(error) => return Err(BlockProduceError::PollError(error)),
-        };
-        Ok(self
-            .beacon_node
-            .last_published_block()
-            .expect("Unable to obtain produced block."))
-    }
-
-    /// Set the validators slot clock to the specified slot.
-    ///
-    /// The validators slot clock will always read this value until it is set to something else.
-    pub fn set_slot(&mut self, slot: Slot) {
-        self.slot_clock.set_slot(slot.as_u64())
-    }
-}

beacon_node/beacon_chain/test_harness/tests/chain.rs

@@ -1,46 +0,0 @@
-#![cfg(not(debug_assertions))]
-
-use env_logger::{Builder, Env};
-use log::debug;
-use test_harness::BeaconChainHarness;
-use types::ChainSpec;
-
-#[test]
-fn it_can_build_on_genesis_block() {
-    Builder::from_env(Env::default().default_filter_or("info")).init();
-
-    let spec = ChainSpec::few_validators();
-    let validator_count = 8;
-
-    let mut harness = BeaconChainHarness::new(spec, validator_count as usize);
-
-    harness.advance_chain_with_block();
-}
-
-#[test]
-#[ignore]
-fn it_can_produce_past_first_epoch_boundary() {
-    Builder::from_env(Env::default().default_filter_or("info")).init();
-
-    let spec = ChainSpec::few_validators();
-    let validator_count = 8;
-
-    debug!("Starting harness build...");
-
-    let mut harness = BeaconChainHarness::new(spec, validator_count);
-
-    debug!("Harness built, tests starting..");
-
-    let blocks = harness.spec.slots_per_epoch * 2 + 1;
-
-    for i in 0..blocks {
-        harness.advance_chain_with_block();
-        debug!("Produced block {}/{}.", i + 1, blocks);
-    }
-
-    harness.run_fork_choice();
-
-    let dump = harness.chain_dump().expect("Chain dump failed.");
-
-    assert_eq!(dump.len() as u64, blocks + 1); // + 1 for genesis block.
-}

beacon_node/client/src/client_config.rs

@@ -9,8 +9,8 @@ use std::net::{IpAddr, Ipv4Addr};
 use std::path::PathBuf;
 use types::multiaddr::Protocol;
 use types::multiaddr::ToMultiaddr;
-use types::ChainSpec;
 use types::Multiaddr;
+use types::{ChainSpec, EthSpec, LighthouseTestnetEthSpec};
 
 /// Stores the client configuration for this Lighthouse instance.
 #[derive(Debug, Clone)]
@@ -35,7 +35,7 @@ impl Default for ClientConfig {
         fs::create_dir_all(&data_dir)
             .unwrap_or_else(|_| panic!("Unable to create {:?}", &data_dir));
 
-        let default_spec = ChainSpec::lighthouse_testnet();
+        let default_spec = LighthouseTestnetEthSpec::spec();
         let default_net_conf = NetworkConfig::new(default_spec.boot_nodes.clone());
 
         Self {

beacon_node/client/src/client_types.rs

@@ -1,23 +1,22 @@
-use crate::ClientConfig;
+use crate::{ArcBeaconChain, ClientConfig};
 use beacon_chain::{
     db::{ClientDB, DiskDB, MemoryDB},
     fork_choice::BitwiseLMDGhost,
     initialise,
     slot_clock::{SlotClock, SystemTimeSlotClock},
-    BeaconChain,
 };
 use fork_choice::ForkChoice;
-
+use types::{EthSpec, FewValidatorsEthSpec, FoundationEthSpec};
-use std::sync::Arc;
 
 pub trait ClientTypes {
     type DB: ClientDB + 'static;
     type SlotClock: SlotClock + 'static;
     type ForkChoice: ForkChoice + 'static;
+    type EthSpec: EthSpec + 'static;
 
     fn initialise_beacon_chain(
         config: &ClientConfig,
-    ) -> Arc<BeaconChain<Self::DB, Self::SlotClock, Self::ForkChoice>>;
+    ) -> ArcBeaconChain<Self::DB, Self::SlotClock, Self::ForkChoice, Self::EthSpec>;
 }
 
 pub struct StandardClientType;
@@ -25,11 +24,12 @@ pub struct StandardClientType;
 impl ClientTypes for StandardClientType {
     type DB = DiskDB;
     type SlotClock = SystemTimeSlotClock;
-    type ForkChoice = BitwiseLMDGhost<DiskDB>;
+    type ForkChoice = BitwiseLMDGhost<DiskDB, Self::EthSpec>;
+    type EthSpec = FoundationEthSpec;
 
     fn initialise_beacon_chain(
         config: &ClientConfig,
-    ) -> Arc<BeaconChain<Self::DB, Self::SlotClock, Self::ForkChoice>> {
+    ) -> ArcBeaconChain<Self::DB, Self::SlotClock, Self::ForkChoice, Self::EthSpec> {
         initialise::initialise_beacon_chain(&config.spec, Some(&config.db_name))
     }
 }
@@ -39,11 +39,12 @@ pub struct TestingClientType;
 impl ClientTypes for TestingClientType {
     type DB = MemoryDB;
     type SlotClock = SystemTimeSlotClock;
-    type ForkChoice = BitwiseLMDGhost<MemoryDB>;
+    type ForkChoice = BitwiseLMDGhost<MemoryDB, Self::EthSpec>;
+    type EthSpec = FewValidatorsEthSpec;
 
     fn initialise_beacon_chain(
         config: &ClientConfig,
-    ) -> Arc<BeaconChain<Self::DB, Self::SlotClock, Self::ForkChoice>> {
+    ) -> ArcBeaconChain<Self::DB, Self::SlotClock, Self::ForkChoice, Self::EthSpec> {
         initialise::initialise_test_beacon_chain(&config.spec, None)
     }
 }

beacon_node/client/src/lib.rs

@@ -20,6 +20,9 @@ use std::sync::Arc;
 use std::time::{Duration, Instant};
 use tokio::runtime::TaskExecutor;
 use tokio::timer::Interval;
+use types::EthSpec;
+
+type ArcBeaconChain<D, S, F, B> = Arc<BeaconChain<D, S, F, B>>;
 
 /// Main beacon node client service. This provides the connection and initialisation of the clients
 /// sub-services in multiple threads.
@@ -27,9 +30,9 @@ pub struct Client<T: ClientTypes> {
     /// Configuration for the lighthouse client.
     _config: ClientConfig,
     /// The beacon chain for the running client.
-    _beacon_chain: Arc<BeaconChain<T::DB, T::SlotClock, T::ForkChoice>>,
+    _beacon_chain: ArcBeaconChain<T::DB, T::SlotClock, T::ForkChoice, T::EthSpec>,
     /// Reference to the network service.
-    pub network: Arc<NetworkService>,
+    pub network: Arc<NetworkService<T::EthSpec>>,
     /// Signal to terminate the RPC server.
     pub rpc_exit_signal: Option<Signal>,
     /// Signal to terminate the slot timer.
@@ -141,11 +144,12 @@ impl<TClientType: ClientTypes> Client<TClientType> {
     }
 }
 
-fn do_state_catchup<T, U, F>(chain: &Arc<BeaconChain<T, U, F>>, log: &slog::Logger)
+fn do_state_catchup<T, U, F, E>(chain: &Arc<BeaconChain<T, U, F, E>>, log: &slog::Logger)
 where
     T: ClientDB,
     U: SlotClock,
     F: ForkChoice,
+    E: EthSpec,
 {
     if let Some(genesis_height) = chain.slots_since_genesis() {
         let result = chain.catchup_state();

beacon_node/db/src/disk_db.rs

@@ -97,7 +97,7 @@ impl ClientDB for DiskDB {
             None => Err(DBError {
                 message: "Unknown column".to_string(),
             }),
-            Some(handle) => self.db.put_cf(handle, key, val).map_err(|e| e.into()),
+            Some(handle) => self.db.put_cf(handle, key, val).map_err(Into::into),
         }
     }
 

beacon_node/db/src/stores/beacon_block_store.rs

@@ -1,6 +1,6 @@
 use super::BLOCKS_DB_COLUMN as DB_COLUMN;
 use super::{ClientDB, DBError};
-use ssz::decode;
+use ssz::Decode;
 use std::sync::Arc;
 use types::{BeaconBlock, Hash256, Slot};
 
@@ -30,7 +30,7 @@ impl<T: ClientDB> BeaconBlockStore<T> {
         match self.get(&hash)? {
             None => Ok(None),
             Some(ssz) => {
-                let block = decode::<BeaconBlock>(&ssz).map_err(|_| DBError {
+                let block = BeaconBlock::from_ssz_bytes(&ssz).map_err(|_| DBError {
                     message: "Bad BeaconBlock SSZ.".to_string(),
                 })?;
                 Ok(Some(block))

beacon_node/db/src/stores/beacon_state_store.rs

@@ -1,8 +1,8 @@
 use super::STATES_DB_COLUMN as DB_COLUMN;
 use super::{ClientDB, DBError};
-use ssz::decode;
+use ssz::Decode;
 use std::sync::Arc;
-use types::{BeaconState, Hash256};
+use types::{BeaconState, EthSpec, Hash256};
 
 pub struct BeaconStateStore<T>
 where
@@ -19,11 +19,14 @@ impl<T: ClientDB> BeaconStateStore<T> {
         Self { db }
     }
 
-    pub fn get_deserialized(&self, hash: &Hash256) -> Result<Option<BeaconState>, DBError> {
+    pub fn get_deserialized<E: EthSpec>(
+        &self,
+        hash: &Hash256,
+    ) -> Result<Option<BeaconState<E>>, DBError> {
         match self.get(&hash)? {
             None => Ok(None),
             Some(ssz) => {
-                let state = decode::<BeaconState>(&ssz).map_err(|_| DBError {
+                let state = BeaconState::from_ssz_bytes(&ssz).map_err(|_| DBError {
                     message: "Bad State SSZ.".to_string(),
                 })?;
                 Ok(Some(state))
@@ -40,7 +43,7 @@ mod tests {
     use ssz::ssz_encode;
     use std::sync::Arc;
     use types::test_utils::{SeedableRng, TestRandom, XorShiftRng};
-    use types::Hash256;
+    use types::{FoundationBeaconState, Hash256};
 
     test_crud_for_store!(BeaconStateStore, DB_COLUMN);
 
@@ -50,7 +53,7 @@ mod tests {
         let store = BeaconStateStore::new(db.clone());
 
         let mut rng = XorShiftRng::from_seed([42; 16]);
-        let state = BeaconState::random_for_test(&mut rng);
+        let state: FoundationBeaconState = BeaconState::random_for_test(&mut rng);
         let state_root = state.canonical_root();
 
         store.put(&state_root, &ssz_encode(&state)).unwrap();

beacon_node/db/src/stores/validator_store.rs

@@ -4,7 +4,7 @@ use self::bytes::{BufMut, BytesMut};
 use super::VALIDATOR_DB_COLUMN as DB_COLUMN;
 use super::{ClientDB, DBError};
 use bls::PublicKey;
-use ssz::{decode, ssz_encode};
+use ssz::{Decode, Encode};
 use std::sync::Arc;
 
 #[derive(Debug, PartialEq)]
@@ -55,7 +55,7 @@ impl<T: ClientDB> ValidatorStore<T> {
         public_key: &PublicKey,
     ) -> Result<(), ValidatorStoreError> {
         let key = self.get_db_key_for_index(&KeyPrefixes::PublicKey, index);
-        let val = ssz_encode(public_key);
+        let val = public_key.as_ssz_bytes();
         self.db
             .put(DB_COLUMN, &key[..], &val[..])
             .map_err(ValidatorStoreError::from)
@@ -69,7 +69,7 @@ impl<T: ClientDB> ValidatorStore<T> {
         let val = self.db.get(DB_COLUMN, &key[..])?;
         match val {
             None => Ok(None),
-            Some(val) => match decode::<PublicKey>(&val) {
+            Some(val) => match PublicKey::from_ssz_bytes(&val) {
                 Ok(key) => Ok(Some(key)),
                 Err(_) => Err(ValidatorStoreError::DecodeError),
             },
@@ -125,7 +125,7 @@ mod tests {
             .unwrap()
             .unwrap();
 
-        assert_eq!(public_key_at_index, ssz_encode(&public_key));
+        assert_eq!(public_key_at_index, public_key.as_ssz_bytes());
     }
 
     #[test]
@@ -139,7 +139,7 @@ mod tests {
         db.put(
             DB_COLUMN,
             &store.get_db_key_for_index(&KeyPrefixes::PublicKey, index)[..],
-            &ssz_encode(&public_key)[..],
+            &public_key.as_ssz_bytes(),
         )
         .unwrap();
 
@@ -157,7 +157,7 @@ mod tests {
         db.put(
             DB_COLUMN,
             &store.get_db_key_for_index(&KeyPrefixes::PublicKey, 3)[..],
-            &ssz_encode(&public_key)[..],
+            &public_key.as_ssz_bytes(),
         )
         .unwrap();
 

beacon_node/eth2-libp2p/src/behaviour.rs

@@ -13,7 +13,7 @@ use libp2p::{
     NetworkBehaviour, PeerId,
 };
 use slog::{debug, o, trace, warn};
-use ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream};
+use ssz::{ssz_encode, Decode, DecodeError, Encode};
 use types::{Attestation, BeaconBlock};
 use types::{Topic, TopicHash};
 
@@ -49,7 +49,7 @@ impl<TSubstream: AsyncRead + AsyncWrite> NetworkBehaviourEventProcess<GossipsubE
             GossipsubEvent::Message(gs_msg) => {
                 trace!(self.log, "Received GossipEvent"; "msg" => format!("{:?}", gs_msg));
 
-                let pubsub_message = match PubsubMessage::ssz_decode(&gs_msg.data, 0) {
+                let pubsub_message = match PubsubMessage::from_ssz_bytes(&gs_msg.data) {
                     //TODO: Punish peer on error
                     Err(e) => {
                         warn!(
@@ -59,7 +59,7 @@ impl<TSubstream: AsyncRead + AsyncWrite> NetworkBehaviourEventProcess<GossipsubE
                         );
                         return;
                     }
-                    Ok((msg, _index)) => msg,
+                    Ok(msg) => msg,
                 };
 
                 self.events.push(BehaviourEvent::GossipMessage {
@@ -197,34 +197,59 @@ pub enum PubsubMessage {
 }
 
 //TODO: Correctly encode/decode enums. Prefixing with integer for now.
-impl Encodable for PubsubMessage {
+impl Encode for PubsubMessage {
-    fn ssz_append(&self, s: &mut SszStream) {
+    fn is_ssz_fixed_len() -> bool {
+        false
+    }
+
+    fn ssz_append(&self, buf: &mut Vec<u8>) {
+        let offset = <u32 as Encode>::ssz_fixed_len() + <Vec<u8> as Encode>::ssz_fixed_len();
+
+        let mut encoder = ssz::SszEncoder::container(buf, offset);
+
         match self {
             PubsubMessage::Block(block_gossip) => {
-                0u32.ssz_append(s);
+                encoder.append(&0_u32);
-                block_gossip.ssz_append(s);
+
+                // Encode the gossip as a Vec<u8>;
+                encoder.append(&block_gossip.as_ssz_bytes());
             }
             PubsubMessage::Attestation(attestation_gossip) => {
-                1u32.ssz_append(s);
+                encoder.append(&1_u32);
-                attestation_gossip.ssz_append(s);
+
+                // Encode the gossip as a Vec<u8>;
+                encoder.append(&attestation_gossip.as_ssz_bytes());
             }
         }
+
+        encoder.finalize();
     }
 }
 
-impl Decodable for PubsubMessage {
+impl Decode for PubsubMessage {
-    fn ssz_decode(bytes: &[u8], index: usize) -> Result<(Self, usize), DecodeError> {
+    fn is_ssz_fixed_len() -> bool {
-        let (id, index) = u32::ssz_decode(bytes, index)?;
+        false
+    }
+
+    fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, ssz::DecodeError> {
+        let mut builder = ssz::SszDecoderBuilder::new(&bytes);
+
+        builder.register_type::<u32>()?;
+        builder.register_type::<Vec<u8>>()?;
+
+        let mut decoder = builder.build()?;
+
+        let id: u32 = decoder.decode_next()?;
+        let body: Vec<u8> = decoder.decode_next()?;
+
         match id {
-            0 => {
+            0 => Ok(PubsubMessage::Block(BeaconBlock::from_ssz_bytes(&body)?)),
-                let (block, index) = BeaconBlock::ssz_decode(bytes, index)?;
+            1 => Ok(PubsubMessage::Attestation(Attestation::from_ssz_bytes(
-                Ok((PubsubMessage::Block(block), index))
+                &body,
-            }
+            )?)),
-            1 => {
+            _ => Err(DecodeError::BytesInvalid(
-                let (attestation, index) = Attestation::ssz_decode(bytes, index)?;
+                "Invalid PubsubMessage id".to_string(),
-                Ok((PubsubMessage::Attestation(attestation), index))
+            )),
-            }
-            _ => Err(DecodeError::Invalid),
         }
     }
 }
@@ -236,13 +261,11 @@ mod test {
 
     #[test]
     fn ssz_encoding() {
-        let original = PubsubMessage::Block(BeaconBlock::empty(&ChainSpec::foundation()));
+        let original = PubsubMessage::Block(BeaconBlock::empty(&FoundationEthSpec::spec()));
 
         let encoded = ssz_encode(&original);
 
-        println!("{:?}", encoded);
+        let decoded = PubsubMessage::from_ssz_bytes(&encoded).unwrap();
-
-        let (decoded, _i) = PubsubMessage::ssz_decode(&encoded, 0).unwrap();
 
         assert_eq!(original, decoded);
     }

beacon_node/eth2-libp2p/src/rpc/methods.rs

@@ -1,5 +1,6 @@
-use ssz::{Decodable, DecodeError, Encodable, SszStream};
+//!Available RPC methods types and ids.
-/// Available RPC methods types and ids.
+
+use ssz::{impl_decode_via_from, impl_encode_via_from};
 use ssz_derive::{Decode, Encode};
 use types::{BeaconBlockBody, BeaconBlockHeader, Epoch, Hash256, Slot};
 
@@ -149,19 +150,8 @@ impl Into<u64> for GoodbyeReason {
     }
 }
 
-impl Encodable for GoodbyeReason {
+impl_encode_via_from!(GoodbyeReason, u64);
-    fn ssz_append(&self, s: &mut SszStream) {
+impl_decode_via_from!(GoodbyeReason, u64);
-        let id: u64 = (*self).clone().into();
-        id.ssz_append(s);
-    }
-}
-
-impl Decodable for GoodbyeReason {
-    fn ssz_decode(bytes: &[u8], index: usize) -> Result<(Self, usize), DecodeError> {
-        let (id, index) = u64::ssz_decode(bytes, index)?;
-        Ok((Self::from(id), index))
-    }
-}
 
 /// Request a number of beacon block roots from a peer.
 #[derive(Encode, Decode, Clone, Debug, PartialEq)]

beacon_node/eth2-libp2p/src/rpc/protocol.rs

@@ -1,6 +1,6 @@
 use super::methods::*;
 use libp2p::core::{upgrade, InboundUpgrade, OutboundUpgrade, UpgradeInfo};
-use ssz::{ssz_encode, Decodable, DecodeError as SSZDecodeError, Encodable, SszStream};
+use ssz::{impl_decode_via_from, impl_encode_via_from, ssz_encode, Decode, Encode};
 use std::hash::{Hash, Hasher};
 use std::io;
 use std::iter;
@@ -72,18 +72,8 @@ impl Into<u64> for RequestId {
     }
 }
 
-impl Encodable for RequestId {
+impl_encode_via_from!(RequestId, u64);
-    fn ssz_append(&self, s: &mut SszStream) {
+impl_decode_via_from!(RequestId, u64);
-        self.0.ssz_append(s);
-    }
-}
-
-impl Decodable for RequestId {
-    fn ssz_decode(bytes: &[u8], index: usize) -> Result<(Self, usize), SSZDecodeError> {
-        let (id, index) = u64::ssz_decode(bytes, index)?;
-        Ok((Self::from(id), index))
-    }
-}
 
 /// The RPC types which are sent/received in this protocol.
 #[derive(Debug, Clone)]
@@ -125,42 +115,40 @@ where
     }
 }
 
+// NOTE!
+//
+// This code has not been tested, it is a placeholder until we can update to the new libp2p
+// spec.
 fn decode(packet: Vec<u8>) -> Result<RPCEvent, DecodeError> {
-    // decode the header of the rpc
+    let mut builder = ssz::SszDecoderBuilder::new(&packet);
-    // request/response
+
-    let (request, index) = bool::ssz_decode(&packet, 0)?;
+    builder.register_type::<bool>()?;
-    let (id, index) = RequestId::ssz_decode(&packet, index)?;
+    builder.register_type::<RequestId>()?;
-    let (method_id, index) = u16::ssz_decode(&packet, index)?;
+    builder.register_type::<u16>()?;
+    builder.register_type::<Vec<u8>>()?;
+
+    let mut decoder = builder.build()?;
+
+    let request: bool = decoder.decode_next()?;
+    let id: RequestId = decoder.decode_next()?;
+    let method_id: u16 = decoder.decode_next()?;
+    let bytes: Vec<u8> = decoder.decode_next()?;
 
     if request {
         let body = match RPCMethod::from(method_id) {
-            RPCMethod::Hello => {
+            RPCMethod::Hello => RPCRequest::Hello(HelloMessage::from_ssz_bytes(&bytes)?),
-                let (hello_body, _index) = HelloMessage::ssz_decode(&packet, index)?;
+            RPCMethod::Goodbye => RPCRequest::Goodbye(GoodbyeReason::from_ssz_bytes(&bytes)?),
-                RPCRequest::Hello(hello_body)
-            }
-            RPCMethod::Goodbye => {
-                let (goodbye_reason, _index) = GoodbyeReason::ssz_decode(&packet, index)?;
-                RPCRequest::Goodbye(goodbye_reason)
-            }
             RPCMethod::BeaconBlockRoots => {
-                let (block_roots_request, _index) =
+                RPCRequest::BeaconBlockRoots(BeaconBlockRootsRequest::from_ssz_bytes(&bytes)?)
-                    BeaconBlockRootsRequest::ssz_decode(&packet, index)?;
-                RPCRequest::BeaconBlockRoots(block_roots_request)
             }
             RPCMethod::BeaconBlockHeaders => {
-                let (block_headers_request, _index) =
+                RPCRequest::BeaconBlockHeaders(BeaconBlockHeadersRequest::from_ssz_bytes(&bytes)?)
-                    BeaconBlockHeadersRequest::ssz_decode(&packet, index)?;
-                RPCRequest::BeaconBlockHeaders(block_headers_request)
             }
             RPCMethod::BeaconBlockBodies => {
-                let (block_bodies_request, _index) =
+                RPCRequest::BeaconBlockBodies(BeaconBlockBodiesRequest::from_ssz_bytes(&bytes)?)
-                    BeaconBlockBodiesRequest::ssz_decode(&packet, index)?;
-                RPCRequest::BeaconBlockBodies(block_bodies_request)
             }
             RPCMethod::BeaconChainState => {
-                let (chain_state_request, _index) =
+                RPCRequest::BeaconChainState(BeaconChainStateRequest::from_ssz_bytes(&bytes)?)
-                    BeaconChainStateRequest::ssz_decode(&packet, index)?;
-                RPCRequest::BeaconChainState(chain_state_request)
             }
             RPCMethod::Unknown => return Err(DecodeError::UnknownRPCMethod),
         };
@@ -174,29 +162,24 @@ fn decode(packet: Vec<u8>) -> Result<RPCEvent, DecodeError> {
     // we have received a response
     else {
         let result = match RPCMethod::from(method_id) {
-            RPCMethod::Hello => {
+            RPCMethod::Hello => RPCResponse::Hello(HelloMessage::from_ssz_bytes(&bytes)?),
-                let (body, _index) = HelloMessage::ssz_decode(&packet, index)?;
-                RPCResponse::Hello(body)
-            }
-            RPCMethod::Goodbye => unreachable!("Should never receive a goodbye response"),
             RPCMethod::BeaconBlockRoots => {
-                let (body, _index) = BeaconBlockRootsResponse::ssz_decode(&packet, index)?;
+                RPCResponse::BeaconBlockRoots(BeaconBlockRootsResponse::from_ssz_bytes(&bytes)?)
-                RPCResponse::BeaconBlockRoots(body)
             }
             RPCMethod::BeaconBlockHeaders => {
-                let (body, _index) = BeaconBlockHeadersResponse::ssz_decode(&packet, index)?;
+                RPCResponse::BeaconBlockHeaders(BeaconBlockHeadersResponse::from_ssz_bytes(&bytes)?)
-                RPCResponse::BeaconBlockHeaders(body)
             }
             RPCMethod::BeaconBlockBodies => {
-                let (body, _index) = BeaconBlockBodiesResponse::ssz_decode(&packet, index)?;
+                RPCResponse::BeaconBlockBodies(BeaconBlockBodiesResponse::from_ssz_bytes(&packet)?)
-                RPCResponse::BeaconBlockBodies(body)
             }
             RPCMethod::BeaconChainState => {
-                let (body, _index) = BeaconChainStateResponse::ssz_decode(&packet, index)?;
+                RPCResponse::BeaconChainState(BeaconChainStateResponse::from_ssz_bytes(&packet)?)
-                RPCResponse::BeaconChainState(body)
             }
+            // We should never receive a goodbye response; it is invalid.
+            RPCMethod::Goodbye => return Err(DecodeError::UnknownRPCMethod),
             RPCMethod::Unknown => return Err(DecodeError::UnknownRPCMethod),
         };
+
         Ok(RPCEvent::Response {
             id,
             method_id,
@@ -220,35 +203,51 @@ where
     }
 }
 
-impl Encodable for RPCEvent {
+impl Encode for RPCEvent {
-    fn ssz_append(&self, s: &mut SszStream) {
+    fn is_ssz_fixed_len() -> bool {
+        false
+    }
+
+    // NOTE!
+    //
+    // This code has not been tested, it is a placeholder until we can update to the new libp2p
+    // spec.
+    fn ssz_append(&self, buf: &mut Vec<u8>) {
+        let offset = <bool as Encode>::ssz_fixed_len()
+            + <u16 as Encode>::ssz_fixed_len()
+            + <Vec<u8> as Encode>::ssz_fixed_len();
+
+        let mut encoder = ssz::SszEncoder::container(buf, offset);
+
         match self {
             RPCEvent::Request {
                 id,
                 method_id,
                 body,
             } => {
-                s.append(&true);
+                encoder.append(&true);
-                s.append(id);
+                encoder.append(id);
-                s.append(method_id);
+                encoder.append(method_id);
+
+                // Encode the `body` as a `Vec<u8>`.
                 match body {
                     RPCRequest::Hello(body) => {
-                        s.append(body);
+                        encoder.append(&body.as_ssz_bytes());
                     }
                     RPCRequest::Goodbye(body) => {
-                        s.append(body);
+                        encoder.append(&body.as_ssz_bytes());
                     }
                     RPCRequest::BeaconBlockRoots(body) => {
-                        s.append(body);
+                        encoder.append(&body.as_ssz_bytes());
                     }
                     RPCRequest::BeaconBlockHeaders(body) => {
-                        s.append(body);
+                        encoder.append(&body.as_ssz_bytes());
                     }
                     RPCRequest::BeaconBlockBodies(body) => {
-                        s.append(body);
+                        encoder.append(&body.as_ssz_bytes());
                     }
                     RPCRequest::BeaconChainState(body) => {
-                        s.append(body);
+                        encoder.append(&body.as_ssz_bytes());
                     }
                 }
             }
@@ -257,28 +256,32 @@ impl Encodable for RPCEvent {
                 method_id,
                 result,
             } => {
-                s.append(&false);
+                encoder.append(&true);
-                s.append(id);
+                encoder.append(id);
-                s.append(method_id);
+                encoder.append(method_id);
+
                 match result {
                     RPCResponse::Hello(response) => {
-                        s.append(response);
+                        encoder.append(&response.as_ssz_bytes());
                     }
                     RPCResponse::BeaconBlockRoots(response) => {
-                        s.append(response);
+                        encoder.append(&response.as_ssz_bytes());
                     }
                     RPCResponse::BeaconBlockHeaders(response) => {
-                        s.append(response);
+                        encoder.append(&response.as_ssz_bytes());
                     }
                     RPCResponse::BeaconBlockBodies(response) => {
-                        s.append(response);
+                        encoder.append(&response.as_ssz_bytes());
                     }
                     RPCResponse::BeaconChainState(response) => {
-                        s.append(response);
+                        encoder.append(&response.as_ssz_bytes());
                     }
                 }
             }
         }
+
+        // Finalize the encoder, writing to `buf`.
+        encoder.finalize();
     }
 }
 

beacon_node/network/Cargo.toml

@@ -5,7 +5,6 @@ authors = ["Age Manning <Age@AgeManning.com>"]
 edition = "2018"
 
 [dev-dependencies]
-test_harness = { path = "../beacon_chain/test_harness" }
 sloggers = "0.3.2"
 
 [dependencies]
@@ -15,6 +14,7 @@ version = { path = "../version" }
 types = { path = "../../eth2/types" }
 slog = { version = "^2.2.3" , features = ["max_level_trace", "release_max_level_debug"] }
 ssz = { path = "../../eth2/utils/ssz" }
+tree_hash = { path = "../../eth2/utils/tree_hash" }
 futures = "0.1.25"
 error-chain = "0.12.0"
 crossbeam-channel = "0.3.8"

beacon_node/network/src/beacon_chain.rs

@@ -8,19 +8,21 @@ use beacon_chain::{
     AttestationValidationError, CheckPoint,
 };
 use eth2_libp2p::rpc::HelloMessage;
-use types::{Attestation, BeaconBlock, BeaconBlockBody, BeaconBlockHeader, Epoch, Hash256, Slot};
+use types::{
+    Attestation, BeaconBlock, BeaconBlockBody, BeaconBlockHeader, Epoch, EthSpec, Hash256, Slot,
+};
 
 pub use beacon_chain::{BeaconChainError, BlockProcessingOutcome, InvalidBlock};
 
 /// The network's API to the beacon chain.
-pub trait BeaconChain: Send + Sync {
+pub trait BeaconChain<E: EthSpec>: Send + Sync {
     fn get_spec(&self) -> &ChainSpec;
 
-    fn get_state(&self) -> RwLockReadGuard<BeaconState>;
+    fn get_state(&self) -> RwLockReadGuard<BeaconState<E>>;
 
     fn slot(&self) -> Slot;
 
-    fn head(&self) -> RwLockReadGuard<CheckPoint>;
+    fn head(&self) -> RwLockReadGuard<CheckPoint<E>>;
 
     fn get_block(&self, block_root: &Hash256) -> Result<Option<BeaconBlock>, BeaconChainError>;
 
@@ -28,7 +30,7 @@ pub trait BeaconChain: Send + Sync {
 
     fn best_block_root(&self) -> Hash256;
 
-    fn finalized_head(&self) -> RwLockReadGuard<CheckPoint>;
+    fn finalized_head(&self) -> RwLockReadGuard<CheckPoint<E>>;
 
     fn finalized_epoch(&self) -> Epoch;
 
@@ -62,17 +64,18 @@ pub trait BeaconChain: Send + Sync {
     fn is_new_block_root(&self, beacon_block_root: &Hash256) -> Result<bool, BeaconChainError>;
 }
 
-impl<T, U, F> BeaconChain for RawBeaconChain<T, U, F>
+impl<T, U, F, E> BeaconChain<E> for RawBeaconChain<T, U, F, E>
 where
     T: ClientDB + Sized,
     U: SlotClock,
     F: ForkChoice,
+    E: EthSpec,
 {
     fn get_spec(&self) -> &ChainSpec {
         &self.spec
     }
 
-    fn get_state(&self) -> RwLockReadGuard<BeaconState> {
+    fn get_state(&self) -> RwLockReadGuard<BeaconState<E>> {
         self.state.read()
     }
 
@@ -80,7 +83,7 @@ where
         self.get_state().slot
     }
 
-    fn head(&self) -> RwLockReadGuard<CheckPoint> {
+    fn head(&self) -> RwLockReadGuard<CheckPoint<E>> {
         self.head()
     }
 
@@ -92,7 +95,7 @@ where
         self.get_state().finalized_epoch
     }
 
-    fn finalized_head(&self) -> RwLockReadGuard<CheckPoint> {
+    fn finalized_head(&self) -> RwLockReadGuard<CheckPoint<E>> {
         self.finalized_head()
     }
 

beacon_node/network/src/message_handler.rs

@@ -13,6 +13,7 @@ use slog::{debug, warn};
 use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::Instant;
+use types::EthSpec;
 
 /// Timeout for RPC requests.
 // const REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
@@ -20,11 +21,11 @@ use std::time::Instant;
 // const HELLO_TIMEOUT: Duration = Duration::from_secs(30);
 
 /// Handles messages received from the network and client and organises syncing.
-pub struct MessageHandler {
+pub struct MessageHandler<E: EthSpec> {
     /// Currently loaded and initialised beacon chain.
-    _chain: Arc<BeaconChain>,
+    _chain: Arc<BeaconChain<E>>,
     /// The syncing framework.
-    sync: SimpleSync,
+    sync: SimpleSync<E>,
     /// The context required to send messages to, and process messages from peers.
     network_context: NetworkContext,
     /// The `MessageHandler` logger.
@@ -44,10 +45,10 @@ pub enum HandlerMessage {
     PubsubMessage(PeerId, Box<PubsubMessage>),
 }
 
-impl MessageHandler {
+impl<E: EthSpec> MessageHandler<E> {
     /// Initializes and runs the MessageHandler.
     pub fn spawn(
-        beacon_chain: Arc<BeaconChain>,
+        beacon_chain: Arc<BeaconChain<E>>,
         network_send: crossbeam_channel::Sender<NetworkMessage>,
         executor: &tokio::runtime::TaskExecutor,
         log: slog::Logger,
@@ -299,7 +300,7 @@ impl NetworkContext {
         let next_id = self
             .outgoing_request_ids
             .entry(peer_id.clone())
-            .and_modify(|id| id.increment())
+            .and_modify(RequestId::increment)
             .or_insert_with(|| RequestId::from(1));
 
         next_id.previous()

beacon_node/network/src/service.rs

@@ -10,22 +10,23 @@ use futures::prelude::*;
 use futures::sync::oneshot;
 use futures::Stream;
 use slog::{debug, info, o, trace};
+use std::marker::PhantomData;
 use std::sync::Arc;
 use tokio::runtime::TaskExecutor;
-use types::Topic;
+use types::{EthSpec, Topic};
 
 /// Service that handles communication between internal services and the eth2_libp2p network service.
-pub struct Service {
+pub struct Service<E: EthSpec> {
     //libp2p_service: Arc<Mutex<LibP2PService>>,
     _libp2p_exit: oneshot::Sender<()>,
     network_send: crossbeam_channel::Sender<NetworkMessage>,
-    //message_handler: MessageHandler,
+    _phantom: PhantomData<E>, //message_handler: MessageHandler,
-    //message_handler_send: Sender<HandlerMessage>,
+                              //message_handler_send: Sender<HandlerMessage>
 }
 
-impl Service {
+impl<E: EthSpec> Service<E> {
     pub fn new(
-        beacon_chain: Arc<BeaconChain>,
+        beacon_chain: Arc<BeaconChain<E>>,
         config: &NetworkConfig,
         executor: &TaskExecutor,
         log: slog::Logger,
@@ -56,6 +57,7 @@ impl Service {
         let network_service = Service {
             _libp2p_exit: libp2p_exit,
             network_send: network_send.clone(),
+            _phantom: PhantomData,
         };
 
         Ok((Arc::new(network_service), network_send))

beacon_node/network/src/sync/import_queue.rs

@@ -2,10 +2,10 @@ use crate::beacon_chain::BeaconChain;
 use eth2_libp2p::rpc::methods::*;
 use eth2_libp2p::PeerId;
 use slog::{debug, error};
-use ssz::TreeHash;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
-use types::{BeaconBlock, BeaconBlockBody, BeaconBlockHeader, Hash256, Slot};
+use tree_hash::TreeHash;
+use types::{BeaconBlock, BeaconBlockBody, BeaconBlockHeader, EthSpec, Hash256, Slot};
 
 /// Provides a queue for fully and partially built `BeaconBlock`s.
 ///
@@ -15,12 +15,12 @@ use types::{BeaconBlock, BeaconBlockBody, BeaconBlockHeader, Hash256, Slot};
 ///
 /// - When we receive a `BeaconBlockBody`, the only way we can find it's matching
 /// `BeaconBlockHeader` is to find a header such that `header.beacon_block_body ==
-/// hash_tree_root(body)`. Therefore, if we used a `HashMap` we would need to use the root of
+/// tree_hash_root(body)`. Therefore, if we used a `HashMap` we would need to use the root of
 /// `BeaconBlockBody` as the key.
 /// - It is possible for multiple distinct blocks to have identical `BeaconBlockBodies`. Therefore
 /// we cannot use a `HashMap` keyed by the root of `BeaconBlockBody`.
-pub struct ImportQueue {
+pub struct ImportQueue<E: EthSpec> {
-    pub chain: Arc<BeaconChain>,
+    pub chain: Arc<BeaconChain<E>>,
     /// Partially imported blocks, keyed by the root of `BeaconBlockBody`.
     pub partials: Vec<PartialBeaconBlock>,
     /// Time before a queue entry is considered state.
@@ -29,9 +29,9 @@ pub struct ImportQueue {
     log: slog::Logger,
 }
 
-impl ImportQueue {
+impl<E: EthSpec> ImportQueue<E> {
     /// Return a new, empty queue.
-    pub fn new(chain: Arc<BeaconChain>, stale_time: Duration, log: slog::Logger) -> Self {
+    pub fn new(chain: Arc<BeaconChain<E>>, stale_time: Duration, log: slog::Logger) -> Self {
         Self {
             chain,
             partials: vec![],
@@ -166,7 +166,7 @@ impl ImportQueue {
         let mut required_bodies: Vec<Hash256> = vec![];
 
         for header in headers {
-            let block_root = Hash256::from_slice(&header.hash_tree_root()[..]);
+            let block_root = Hash256::from_slice(&header.tree_hash_root()[..]);
 
             if self.chain_has_not_seen_block(&block_root) {
                 self.insert_header(block_root, header, sender.clone());
@@ -230,7 +230,7 @@ impl ImportQueue {
     ///
     /// If the body already existed, the `inserted` time is set to `now`.
     fn insert_body(&mut self, body: BeaconBlockBody, sender: PeerId) {
-        let body_root = Hash256::from_slice(&body.hash_tree_root()[..]);
+        let body_root = Hash256::from_slice(&body.tree_hash_root()[..]);
 
         self.partials.iter_mut().for_each(|mut p| {
             if let Some(header) = &mut p.header {
@@ -250,7 +250,7 @@ impl ImportQueue {
     ///
     /// If the partial already existed, the `inserted` time is set to `now`.
     fn insert_full_block(&mut self, block: BeaconBlock, sender: PeerId) {
-        let block_root = Hash256::from_slice(&block.hash_tree_root()[..]);
+        let block_root = Hash256::from_slice(&block.tree_hash_root()[..]);
 
         let partial = PartialBeaconBlock {
             slot: block.slot,

beacon_node/network/src/sync/simple_sync.rs

@@ -5,11 +5,11 @@ use eth2_libp2p::rpc::methods::*;
 use eth2_libp2p::rpc::{RPCRequest, RPCResponse, RequestId};
 use eth2_libp2p::PeerId;
 use slog::{debug, error, info, o, warn};
-use ssz::TreeHash;
 use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::Duration;
-use types::{Attestation, BeaconBlock, Epoch, Hash256, Slot};
+use tree_hash::TreeHash;
+use types::{Attestation, BeaconBlock, Epoch, EthSpec, Hash256, Slot};
 
 /// The number of slots that we can import blocks ahead of us, before going into full Sync mode.
 const SLOT_IMPORT_TOLERANCE: u64 = 100;
@@ -88,8 +88,8 @@ impl From<HelloMessage> for PeerSyncInfo {
     }
 }
 
-impl From<&Arc<BeaconChain>> for PeerSyncInfo {
+impl<E: EthSpec> From<&Arc<BeaconChain<E>>> for PeerSyncInfo {
-    fn from(chain: &Arc<BeaconChain>) -> PeerSyncInfo {
+    fn from(chain: &Arc<BeaconChain<E>>) -> PeerSyncInfo {
         Self::from(chain.hello_message())
     }
 }
@@ -103,22 +103,22 @@ pub enum SyncState {
 }
 
 /// Simple Syncing protocol.
-pub struct SimpleSync {
+pub struct SimpleSync<E: EthSpec> {
     /// A reference to the underlying beacon chain.
-    chain: Arc<BeaconChain>,
+    chain: Arc<BeaconChain<E>>,
     /// A mapping of Peers to their respective PeerSyncInfo.
     known_peers: HashMap<PeerId, PeerSyncInfo>,
     /// A queue to allow importing of blocks
-    import_queue: ImportQueue,
+    import_queue: ImportQueue<E>,
     /// The current state of the syncing protocol.
     state: SyncState,
     /// Sync logger.
     log: slog::Logger,
 }
 
-impl SimpleSync {
+impl<E: EthSpec> SimpleSync<E> {
     /// Instantiate a `SimpleSync` instance, with no peers and an empty queue.
-    pub fn new(beacon_chain: Arc<BeaconChain>, log: &slog::Logger) -> Self {
+    pub fn new(beacon_chain: Arc<BeaconChain<E>>, log: &slog::Logger) -> Self {
         let sync_logger = log.new(o!("Service"=> "Sync"));
 
         let queue_item_stale_time = Duration::from_secs(QUEUE_STALE_SECS);
@@ -565,7 +565,7 @@ impl SimpleSync {
             return false;
         }
 
-        let block_root = Hash256::from_slice(&block.hash_tree_root());
+        let block_root = Hash256::from_slice(&block.tree_hash_root());
 
         // Ignore any block that the chain already knows about.
         if self.chain_has_seen_block(&block_root) {

beacon_node/network/tests/tests.rs

@@ -1,570 +0,0 @@
-use crossbeam_channel::{unbounded, Receiver, RecvTimeoutError, Sender};
-use eth2_libp2p::rpc::methods::*;
-use eth2_libp2p::rpc::{RPCMethod, RPCRequest, RPCResponse, RequestId};
-use eth2_libp2p::{PeerId, RPCEvent};
-use network::beacon_chain::BeaconChain as NetworkBeaconChain;
-use network::message_handler::{HandlerMessage, MessageHandler};
-use network::service::{NetworkMessage, OutgoingMessage};
-use sloggers::terminal::{Destination, TerminalLoggerBuilder};
-use sloggers::types::Severity;
-use sloggers::Build;
-use std::time::Duration;
-use test_harness::BeaconChainHarness;
-use tokio::runtime::TaskExecutor;
-use types::{test_utils::TestingBeaconStateBuilder, *};
-
-pub struct SyncNode {
-    pub id: usize,
-    sender: Sender<HandlerMessage>,
-    receiver: Receiver<NetworkMessage>,
-    peer_id: PeerId,
-    harness: BeaconChainHarness,
-}
-
-impl SyncNode {
-    fn from_beacon_state_builder(
-        id: usize,
-        executor: &TaskExecutor,
-        state_builder: TestingBeaconStateBuilder,
-        spec: &ChainSpec,
-        logger: slog::Logger,
-    ) -> Self {
-        let harness = BeaconChainHarness::from_beacon_state_builder(state_builder, spec.clone());
-
-        let (network_sender, network_receiver) = unbounded();
-        let message_handler_sender = MessageHandler::spawn(
-            harness.beacon_chain.clone(),
-            network_sender,
-            executor,
-            logger,
-        )
-        .unwrap();
-
-        Self {
-            id,
-            sender: message_handler_sender,
-            receiver: network_receiver,
-            peer_id: PeerId::random(),
-            harness,
-        }
-    }
-
-    fn increment_beacon_chain_slot(&mut self) {
-        self.harness.increment_beacon_chain_slot();
-    }
-
-    fn send(&self, message: HandlerMessage) {
-        self.sender.send(message).unwrap();
-    }
-
-    fn recv(&self) -> Result<NetworkMessage, RecvTimeoutError> {
-        self.receiver.recv_timeout(Duration::from_millis(500))
-    }
-
-    fn hello_message(&self) -> HelloMessage {
-        self.harness.beacon_chain.hello_message()
-    }
-
-    pub fn connect_to(&mut self, node: &SyncNode) {
-        let message = HandlerMessage::PeerDialed(self.peer_id.clone());
-        node.send(message);
-    }
-
-    /// Reads the receive queue from one node and passes the message to the other. Also returns a
-    /// copy of the message.
-    ///
-    /// self -----> node
-    ///        |
-    ///        us
-    ///
-    /// Named after the unix `tee` command.
-    fn tee(&mut self, node: &SyncNode) -> NetworkMessage {
-        let network_message = self.recv().expect("Timeout on tee");
-
-        let handler_message = match network_message.clone() {
-            NetworkMessage::Send(_to_peer_id, OutgoingMessage::RPC(event)) => {
-                HandlerMessage::RPC(self.peer_id.clone(), event)
-            }
-            _ => panic!("tee cannot parse {:?}", network_message),
-        };
-
-        node.send(handler_message);
-
-        network_message
-    }
-
-    fn tee_hello_request(&mut self, node: &SyncNode) -> HelloMessage {
-        let request = self.tee_rpc_request(node);
-
-        match request {
-            RPCRequest::Hello(message) => message,
-            _ => panic!("tee_hello_request got: {:?}", request),
-        }
-    }
-
-    fn tee_hello_response(&mut self, node: &SyncNode) -> HelloMessage {
-        let response = self.tee_rpc_response(node);
-
-        match response {
-            RPCResponse::Hello(message) => message,
-            _ => panic!("tee_hello_response got: {:?}", response),
-        }
-    }
-
-    fn tee_block_root_request(&mut self, node: &SyncNode) -> BeaconBlockRootsRequest {
-        let msg = self.tee_rpc_request(node);
-
-        match msg {
-            RPCRequest::BeaconBlockRoots(data) => data,
-            _ => panic!("tee_block_root_request got: {:?}", msg),
-        }
-    }
-
-    fn tee_block_root_response(&mut self, node: &SyncNode) -> BeaconBlockRootsResponse {
-        let msg = self.tee_rpc_response(node);
-
-        match msg {
-            RPCResponse::BeaconBlockRoots(data) => data,
-            _ => panic!("tee_block_root_response got: {:?}", msg),
-        }
-    }
-
-    fn tee_block_header_request(&mut self, node: &SyncNode) -> BeaconBlockHeadersRequest {
-        let msg = self.tee_rpc_request(node);
-
-        match msg {
-            RPCRequest::BeaconBlockHeaders(data) => data,
-            _ => panic!("tee_block_header_request got: {:?}", msg),
-        }
-    }
-
-    fn tee_block_header_response(&mut self, node: &SyncNode) -> BeaconBlockHeadersResponse {
-        let msg = self.tee_rpc_response(node);
-
-        match msg {
-            RPCResponse::BeaconBlockHeaders(data) => data,
-            _ => panic!("tee_block_header_response got: {:?}", msg),
-        }
-    }
-
-    fn tee_block_body_request(&mut self, node: &SyncNode) -> BeaconBlockBodiesRequest {
-        let msg = self.tee_rpc_request(node);
-
-        match msg {
-            RPCRequest::BeaconBlockBodies(data) => data,
-            _ => panic!("tee_block_body_request got: {:?}", msg),
-        }
-    }
-
-    fn tee_block_body_response(&mut self, node: &SyncNode) -> BeaconBlockBodiesResponse {
-        let msg = self.tee_rpc_response(node);
-
-        match msg {
-            RPCResponse::BeaconBlockBodies(data) => data,
-            _ => panic!("tee_block_body_response got: {:?}", msg),
-        }
-    }
-
-    fn tee_rpc_request(&mut self, node: &SyncNode) -> RPCRequest {
-        let network_message = self.tee(node);
-
-        match network_message {
-            NetworkMessage::Send(
-                _peer_id,
-                OutgoingMessage::RPC(RPCEvent::Request {
-                    id: _,
-                    method_id: _,
-                    body,
-                }),
-            ) => body,
-            _ => panic!("tee_rpc_request failed! got {:?}", network_message),
-        }
-    }
-
-    fn tee_rpc_response(&mut self, node: &SyncNode) -> RPCResponse {
-        let network_message = self.tee(node);
-
-        match network_message {
-            NetworkMessage::Send(
-                _peer_id,
-                OutgoingMessage::RPC(RPCEvent::Response {
-                    id: _,
-                    method_id: _,
-                    result,
-                }),
-            ) => result,
-            _ => panic!("tee_rpc_response failed! got {:?}", network_message),
-        }
-    }
-
-    pub fn get_block_root_request(&self) -> BeaconBlockRootsRequest {
-        let request = self.recv_rpc_request().expect("No block root request");
-
-        match request {
-            RPCRequest::BeaconBlockRoots(request) => request,
-            _ => panic!("Did not get block root request"),
-        }
-    }
-
-    pub fn get_block_headers_request(&self) -> BeaconBlockHeadersRequest {
-        let request = self.recv_rpc_request().expect("No block headers request");
-
-        match request {
-            RPCRequest::BeaconBlockHeaders(request) => request,
-            _ => panic!("Did not get block headers request"),
-        }
-    }
-
-    pub fn get_block_bodies_request(&self) -> BeaconBlockBodiesRequest {
-        let request = self.recv_rpc_request().expect("No block bodies request");
-
-        match request {
-            RPCRequest::BeaconBlockBodies(request) => request,
-            _ => panic!("Did not get block bodies request"),
-        }
-    }
-
-    fn _recv_rpc_response(&self) -> Result<RPCResponse, RecvTimeoutError> {
-        let network_message = self.recv()?;
-        Ok(match network_message {
-            NetworkMessage::Send(
-                _peer_id,
-                OutgoingMessage::RPC(RPCEvent::Response {
-                    id: _,
-                    method_id: _,
-                    result,
-                }),
-            ) => result,
-            _ => panic!("get_rpc_response failed! got {:?}", network_message),
-        })
-    }
-
-    fn recv_rpc_request(&self) -> Result<RPCRequest, RecvTimeoutError> {
-        let network_message = self.recv()?;
-        Ok(match network_message {
-            NetworkMessage::Send(
-                _peer_id,
-                OutgoingMessage::RPC(RPCEvent::Request {
-                    id: _,
-                    method_id: _,
-                    body,
-                }),
-            ) => body,
-            _ => panic!("get_rpc_request failed! got {:?}", network_message),
-        })
-    }
-}
-
-fn get_logger() -> slog::Logger {
-    let mut builder = TerminalLoggerBuilder::new();
-    builder.level(Severity::Debug);
-    builder.destination(Destination::Stderr);
-    builder.build().unwrap()
-}
-
-pub struct SyncMaster {
-    harness: BeaconChainHarness,
-    peer_id: PeerId,
-    response_ids: Vec<RequestId>,
-}
-
-impl SyncMaster {
-    fn from_beacon_state_builder(
-        state_builder: TestingBeaconStateBuilder,
-        node_count: usize,
-        spec: &ChainSpec,
-    ) -> Self {
-        let harness = BeaconChainHarness::from_beacon_state_builder(state_builder, spec.clone());
-        let peer_id = PeerId::random();
-        let response_ids = vec![RequestId::from(0); node_count];
-
-        Self {
-            harness,
-            peer_id,
-            response_ids,
-        }
-    }
-
-    pub fn response_id(&mut self, node: &SyncNode) -> RequestId {
-        let id = self.response_ids[node.id].clone();
-        self.response_ids[node.id].increment();
-        id
-    }
-
-    pub fn do_hello_with(&mut self, node: &SyncNode) {
-        let message = HandlerMessage::PeerDialed(self.peer_id.clone());
-        node.send(message);
-
-        let request = node.recv_rpc_request().expect("No hello response");
-
-        match request {
-            RPCRequest::Hello(_hello) => {
-                let hello = self.harness.beacon_chain.hello_message();
-                let response = self.rpc_response(node, RPCResponse::Hello(hello));
-                node.send(response);
-            }
-            _ => panic!("Got message other than hello from node."),
-        }
-    }
-
-    pub fn respond_to_block_roots_request(
-        &mut self,
-        node: &SyncNode,
-        request: BeaconBlockRootsRequest,
-    ) {
-        let roots = self
-            .harness
-            .beacon_chain
-            .get_block_roots(request.start_slot, request.count as usize, 0)
-            .expect("Beacon chain did not give block roots")
-            .iter()
-            .enumerate()
-            .map(|(i, root)| BlockRootSlot {
-                block_root: *root,
-                slot: Slot::from(i) + request.start_slot,
-            })
-            .collect();
-
-        let response = RPCResponse::BeaconBlockRoots(BeaconBlockRootsResponse { roots });
-        self.send_rpc_response(node, response)
-    }
-
-    pub fn respond_to_block_headers_request(
-        &mut self,
-        node: &SyncNode,
-        request: BeaconBlockHeadersRequest,
-    ) {
-        let roots = self
-            .harness
-            .beacon_chain
-            .get_block_roots(
-                request.start_slot,
-                request.max_headers as usize,
-                request.skip_slots as usize,
-            )
-            .expect("Beacon chain did not give blocks");
-
-        if roots.is_empty() {
-            panic!("Roots was empty when trying to get headers.")
-        }
-
-        assert_eq!(
-            roots[0], request.start_root,
-            "Got the wrong start root when getting headers"
-        );
-
-        let headers: Vec<BeaconBlockHeader> = roots
-            .iter()
-            .map(|root| {
-                let block = self
-                    .harness
-                    .beacon_chain
-                    .get_block(root)
-                    .expect("Failed to load block")
-                    .expect("Block did not exist");
-                block.block_header()
-            })
-            .collect();
-
-        let response = RPCResponse::BeaconBlockHeaders(BeaconBlockHeadersResponse { headers });
-        self.send_rpc_response(node, response)
-    }
-
-    pub fn respond_to_block_bodies_request(
-        &mut self,
-        node: &SyncNode,
-        request: BeaconBlockBodiesRequest,
-    ) {
-        let block_bodies: Vec<BeaconBlockBody> = request
-            .block_roots
-            .iter()
-            .map(|root| {
-                let block = self
-                    .harness
-                    .beacon_chain
-                    .get_block(root)
-                    .expect("Failed to load block")
-                    .expect("Block did not exist");
-                block.body
-            })
-            .collect();
-
-        let response = RPCResponse::BeaconBlockBodies(BeaconBlockBodiesResponse { block_bodies });
-        self.send_rpc_response(node, response)
-    }
-
-    fn send_rpc_response(&mut self, node: &SyncNode, rpc_response: RPCResponse) {
-        node.send(self.rpc_response(node, rpc_response));
-    }
-
-    fn rpc_response(&mut self, node: &SyncNode, rpc_response: RPCResponse) -> HandlerMessage {
-        HandlerMessage::RPC(
-            self.peer_id.clone(),
-            RPCEvent::Response {
-                id: self.response_id(node),
-                method_id: RPCMethod::Hello.into(),
-                result: rpc_response,
-            },
-        )
-    }
-}
-
-fn test_setup(
-    state_builder: TestingBeaconStateBuilder,
-    node_count: usize,
-    spec: &ChainSpec,
-    logger: slog::Logger,
-) -> (tokio::runtime::Runtime, SyncMaster, Vec<SyncNode>) {
-    let runtime = tokio::runtime::Runtime::new().unwrap();
-
-    let mut nodes = Vec::with_capacity(node_count);
-    for id in 0..node_count {
-        let node = SyncNode::from_beacon_state_builder(
-            id,
-            &runtime.executor(),
-            state_builder.clone(),
-            &spec,
-            logger.clone(),
-        );
-
-        nodes.push(node);
-    }
-
-    let master = SyncMaster::from_beacon_state_builder(state_builder, node_count, &spec);
-
-    (runtime, master, nodes)
-}
-
-pub fn build_blocks(blocks: usize, master: &mut SyncMaster, nodes: &mut Vec<SyncNode>) {
-    for _ in 0..blocks {
-        master.harness.advance_chain_with_block();
-        for i in 0..nodes.len() {
-            nodes[i].increment_beacon_chain_slot();
-        }
-    }
-    master.harness.run_fork_choice();
-
-    for i in 0..nodes.len() {
-        nodes[i].harness.run_fork_choice();
-    }
-}
-
-#[test]
-#[ignore]
-fn sync_node_with_master() {
-    let logger = get_logger();
-    let spec = ChainSpec::few_validators();
-    let validator_count = 8;
-    let node_count = 1;
-
-    let state_builder =
-        TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(validator_count, &spec);
-
-    let (runtime, mut master, mut nodes) =
-        test_setup(state_builder, node_count, &spec, logger.clone());
-
-    let original_node_slot = nodes[0].hello_message().best_slot;
-
-    build_blocks(2, &mut master, &mut nodes);
-
-    master.do_hello_with(&nodes[0]);
-
-    let roots_request = nodes[0].get_block_root_request();
-    assert_eq!(roots_request.start_slot, original_node_slot + 1);
-    assert_eq!(roots_request.count, 2);
-
-    master.respond_to_block_roots_request(&nodes[0], roots_request);
-
-    let headers_request = nodes[0].get_block_headers_request();
-    assert_eq!(headers_request.start_slot, original_node_slot + 1);
-    assert_eq!(headers_request.max_headers, 2);
-    assert_eq!(headers_request.skip_slots, 0);
-
-    master.respond_to_block_headers_request(&nodes[0], headers_request);
-
-    let bodies_request = nodes[0].get_block_bodies_request();
-    assert_eq!(bodies_request.block_roots.len(), 2);
-
-    master.respond_to_block_bodies_request(&nodes[0], bodies_request);
-
-    std::thread::sleep(Duration::from_millis(10000));
-    runtime.shutdown_now();
-}
-
-#[test]
-#[ignore]
-fn sync_two_nodes() {
-    let logger = get_logger();
-    let spec = ChainSpec::few_validators();
-    let validator_count = 8;
-    let node_count = 2;
-
-    let state_builder =
-        TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(validator_count, &spec);
-
-    let (runtime, _master, mut nodes) =
-        test_setup(state_builder, node_count, &spec, logger.clone());
-
-    // let original_node_slot = nodes[0].hello_message().best_slot;
-    let mut node_a = nodes.remove(0);
-    let mut node_b = nodes.remove(0);
-
-    let blocks = 2;
-
-    // Node A builds out a longer, better chain.
-    for _ in 0..blocks {
-        // Node A should build a block.
-        node_a.harness.advance_chain_with_block();
-        // Node B should just increment it's slot without a block.
-        node_b.harness.increment_beacon_chain_slot();
-    }
-    node_a.harness.run_fork_choice();
-
-    // A connects to B.
-    node_a.connect_to(&node_b);
-
-    // B says hello to A.
-    node_b.tee_hello_request(&node_a);
-    // A says hello back.
-    node_a.tee_hello_response(&node_b);
-
-    // B requests block roots from A.
-    node_b.tee_block_root_request(&node_a);
-    // A provides block roots to A.
-    node_a.tee_block_root_response(&node_b);
-
-    // B requests block headers from A.
-    node_b.tee_block_header_request(&node_a);
-    // A provides block headers to B.
-    node_a.tee_block_header_response(&node_b);
-
-    // B requests block bodies from A.
-    node_b.tee_block_body_request(&node_a);
-    // A provides block bodies to B.
-    node_a.tee_block_body_response(&node_b);
-
-    std::thread::sleep(Duration::from_secs(20));
-
-    node_b.harness.run_fork_choice();
-
-    let node_a_chain = node_a
-        .harness
-        .beacon_chain
-        .chain_dump()
-        .expect("Can't dump node a chain");
-
-    let node_b_chain = node_b
-        .harness
-        .beacon_chain
-        .chain_dump()
-        .expect("Can't dump node b chain");
-
-    assert_eq!(
-        node_a_chain.len(),
-        node_b_chain.len(),
-        "Chains should be equal length"
-    );
-    assert_eq!(node_a_chain, node_b_chain, "Chains should be identical");
-
-    runtime.shutdown_now();
-}

beacon_node/rpc/src/attestation.rs

@@ -7,17 +7,17 @@ use protos::services::{
 };
 use protos::services_grpc::AttestationService;
 use slog::{error, info, trace, warn};
-use ssz::{ssz_encode, Decodable};
+use ssz::{ssz_encode, Decode};
 use std::sync::Arc;
-use types::Attestation;
+use types::{Attestation, EthSpec};
 
 #[derive(Clone)]
-pub struct AttestationServiceInstance {
+pub struct AttestationServiceInstance<E: EthSpec> {
-    pub chain: Arc<BeaconChain>,
+    pub chain: Arc<BeaconChain<E>>,
     pub log: slog::Logger,
 }
 
-impl AttestationService for AttestationServiceInstance {
+impl<E: EthSpec> AttestationService for AttestationServiceInstance<E> {
     /// Produce the `AttestationData` for signing by a validator.
     fn produce_attestation_data(
         &mut self,
@@ -110,8 +110,8 @@ impl AttestationService for AttestationServiceInstance {
         let mut resp = PublishAttestationResponse::new();
         let ssz_serialized_attestation = req.get_attestation().get_ssz();
 
-        let attestation = match Attestation::ssz_decode(ssz_serialized_attestation, 0) {
+        let attestation = match Attestation::from_ssz_bytes(ssz_serialized_attestation) {
-            Ok((v, _index)) => v,
+            Ok(v) => v,
             Err(_) => {
                 let log_clone = self.log.clone();
                 let f = sink

beacon_node/rpc/src/beacon_block.rs

@@ -11,18 +11,18 @@ use protos::services::{
 use protos::services_grpc::BeaconBlockService;
 use slog::Logger;
 use slog::{error, info, trace, warn};
-use ssz::{ssz_encode, Decodable};
+use ssz::{ssz_encode, Decode};
 use std::sync::Arc;
-use types::{BeaconBlock, Signature, Slot};
+use types::{BeaconBlock, EthSpec, Signature, Slot};
 
 #[derive(Clone)]
-pub struct BeaconBlockServiceInstance {
+pub struct BeaconBlockServiceInstance<E: EthSpec> {
-    pub chain: Arc<BeaconChain>,
+    pub chain: Arc<BeaconChain<E>>,
     pub network_chan: crossbeam_channel::Sender<NetworkMessage>,
     pub log: Logger,
 }
 
-impl BeaconBlockService for BeaconBlockServiceInstance {
+impl<E: EthSpec> BeaconBlockService for BeaconBlockServiceInstance<E> {
     /// Produce a `BeaconBlock` for signing by a validator.
     fn produce_beacon_block(
         &mut self,
@@ -35,8 +35,8 @@ impl BeaconBlockService for BeaconBlockServiceInstance {
         // decode the request
         // TODO: requested slot currently unused, see: https://github.com/sigp/lighthouse/issues/336
         let _requested_slot = Slot::from(req.get_slot());
-        let randao_reveal = match Signature::ssz_decode(req.get_randao_reveal(), 0) {
+        let randao_reveal = match Signature::from_ssz_bytes(req.get_randao_reveal()) {
-            Ok((reveal, _index)) => reveal,
+            Ok(reveal) => reveal,
             Err(_) => {
                 // decode error, incorrect signature
                 let log_clone = self.log.clone();
@@ -91,8 +91,8 @@ impl BeaconBlockService for BeaconBlockServiceInstance {
 
         let ssz_serialized_block = req.get_block().get_ssz();
 
-        match BeaconBlock::ssz_decode(ssz_serialized_block, 0) {
+        match BeaconBlock::from_ssz_bytes(ssz_serialized_block) {
-            Ok((block, _i)) => {
+            Ok(block) => {
                 match self.chain.process_block(block.clone()) {
                     Ok(outcome) => {
                         if outcome.sucessfully_processed() {

beacon_node/rpc/src/beacon_chain.rs

@@ -8,15 +8,15 @@ use beacon_chain::{
     AttestationValidationError, BlockProductionError,
 };
 pub use beacon_chain::{BeaconChainError, BlockProcessingOutcome};
-use types::{Attestation, AttestationData, BeaconBlock};
+use types::{Attestation, AttestationData, BeaconBlock, EthSpec};
 
 /// The RPC's API to the beacon chain.
-pub trait BeaconChain: Send + Sync {
+pub trait BeaconChain<E: EthSpec>: Send + Sync {
     fn get_spec(&self) -> &ChainSpec;
 
-    fn get_state(&self) -> RwLockReadGuard<BeaconState>;
+    fn get_state(&self) -> RwLockReadGuard<BeaconState<E>>;
 
-    fn get_mut_state(&self) -> RwLockWriteGuard<BeaconState>;
+    fn get_mut_state(&self) -> RwLockWriteGuard<BeaconState<E>>;
 
     fn process_block(&self, block: BeaconBlock)
         -> Result<BlockProcessingOutcome, BeaconChainError>;
@@ -24,7 +24,7 @@ pub trait BeaconChain: Send + Sync {
     fn produce_block(
         &self,
         randao_reveal: Signature,
-    ) -> Result<(BeaconBlock, BeaconState), BlockProductionError>;
+    ) -> Result<(BeaconBlock, BeaconState<E>), BlockProductionError>;
 
     fn produce_attestation_data(&self, shard: u64) -> Result<AttestationData, BeaconChainError>;
 
@@ -34,21 +34,22 @@ pub trait BeaconChain: Send + Sync {
     ) -> Result<(), AttestationValidationError>;
 }
 
-impl<T, U, F> BeaconChain for RawBeaconChain<T, U, F>
+impl<T, U, F, E> BeaconChain<E> for RawBeaconChain<T, U, F, E>
 where
     T: ClientDB + Sized,
     U: SlotClock,
     F: ForkChoice,
+    E: EthSpec,
 {
     fn get_spec(&self) -> &ChainSpec {
         &self.spec
     }
 
-    fn get_state(&self) -> RwLockReadGuard<BeaconState> {
+    fn get_state(&self) -> RwLockReadGuard<BeaconState<E>> {
         self.state.read()
     }
 
-    fn get_mut_state(&self) -> RwLockWriteGuard<BeaconState> {
+    fn get_mut_state(&self) -> RwLockWriteGuard<BeaconState<E>> {
         self.state.write()
     }
 
@@ -62,7 +63,7 @@ where
     fn produce_block(
         &self,
         randao_reveal: Signature,
-    ) -> Result<(BeaconBlock, BeaconState), BlockProductionError> {
+    ) -> Result<(BeaconBlock, BeaconState<E>), BlockProductionError> {
         self.produce_block(randao_reveal)
     }
 

beacon_node/rpc/src/beacon_node.rs

@@ -5,14 +5,15 @@ use protos::services::{Empty, Fork, NodeInfoResponse};
 use protos::services_grpc::BeaconNodeService;
 use slog::{trace, warn};
 use std::sync::Arc;
+use types::EthSpec;
 
 #[derive(Clone)]
-pub struct BeaconNodeServiceInstance {
+pub struct BeaconNodeServiceInstance<E: EthSpec> {
-    pub chain: Arc<BeaconChain>,
+    pub chain: Arc<BeaconChain<E>>,
     pub log: slog::Logger,
 }
 
-impl BeaconNodeService for BeaconNodeServiceInstance {
+impl<E: EthSpec> BeaconNodeService for BeaconNodeServiceInstance<E> {
     /// Provides basic node information.
     fn info(&mut self, ctx: RpcContext, _req: Empty, sink: UnarySink<NodeInfoResponse>) {
         trace!(self.log, "Node info requested via RPC");

beacon_node/rpc/src/lib.rs

@@ -21,12 +21,13 @@ use protos::services_grpc::{
 use slog::{info, o, warn};
 use std::sync::Arc;
 use tokio::runtime::TaskExecutor;
+use types::EthSpec;
 
-pub fn start_server(
+pub fn start_server<E: EthSpec>(
     config: &RPCConfig,
     executor: &TaskExecutor,
     network_chan: crossbeam_channel::Sender<NetworkMessage>,
-    beacon_chain: Arc<BeaconChain>,
+    beacon_chain: Arc<BeaconChain<E>>,
     log: &slog::Logger,
 ) -> exit_future::Signal {
     let log = log.new(o!("Service"=>"RPC"));

beacon_node/rpc/src/validator.rs

@@ -5,18 +5,18 @@ use grpcio::{RpcContext, RpcStatus, RpcStatusCode, UnarySink};
 use protos::services::{ActiveValidator, GetDutiesRequest, GetDutiesResponse, ValidatorDuty};
 use protos::services_grpc::ValidatorService;
 use slog::{trace, warn};
-use ssz::decode;
+use ssz::Decode;
 use std::sync::Arc;
-use types::{Epoch, RelativeEpoch};
+use types::{Epoch, EthSpec, RelativeEpoch};
 
 #[derive(Clone)]
-pub struct ValidatorServiceInstance {
+pub struct ValidatorServiceInstance<E: EthSpec> {
-    pub chain: Arc<BeaconChain>,
+    pub chain: Arc<BeaconChain<E>>,
     pub log: slog::Logger,
 }
 //TODO: Refactor Errors
 
-impl ValidatorService for ValidatorServiceInstance {
+impl<E: EthSpec> ValidatorService for ValidatorServiceInstance<E> {
     /// For a list of validator public keys, this function returns the slot at which each
     /// validator must propose a block, attest to a shard, their shard committee and the shard they
     /// need to attest to.
@@ -74,7 +74,7 @@ impl ValidatorService for ValidatorServiceInstance {
         for validator_pk in validators.get_public_keys() {
             let mut active_validator = ActiveValidator::new();
 
-            let public_key = match decode::<PublicKey>(validator_pk) {
+            let public_key = match PublicKey::from_ssz_bytes(validator_pk) {
                 Ok(v) => v,
                 Err(_) => {
                     let log_clone = self.log.clone();

docs/documentation.md

@@ -0,0 +1,14 @@
+# Lighthouse Technical Documentation
+
+The technical documentation, as generated by Rust, is available at [lighthouse-docs.sigmaprime.io](http://lighthouse-docs.sigmaprime.io/).
+
+This documentation is generated from Lighthouse and updated regularly.
+
+
+### How to update:
+
+- `cargo doc`: builds the docs inside the `target/doc/` directory.
+- `aws s3 sync target/doc/ s3://lighthouse-docs.sigmaprime.io/`: Uploads all of the docs, as generated with `cargo doc`, to the S3 bucket.
+
+**Note**: You will need appropriate credentials to make the upload.
+

eth2/README.md

@@ -30,7 +30,7 @@ Rust crates containing logic common across the Lighthouse project.
 	- [`ssz`](utils/ssz/): an implementation of the SimpleSerialize 
 	    serialization/deserialization protocol used by Eth 2.0.
 	- [`ssz_derive`](utils/ssz_derive/): provides procedural macros for
-		deriving SSZ `Encodable`, `Decodable`, and `TreeHash` methods.
+		deriving SSZ `Encode`, `Decode`, and `TreeHash` methods.
 	- [`swap_or_not_shuffle`](utils/swap_or_not_shuffle/): a list-shuffling
 		method which is slow, but allows for a subset of indices to be shuffled.
 	- [`test_random_derive`](utils/test_random_derive/): provides procedural

eth2/attester/Cargo.toml

@@ -1,10 +0,0 @@
-[package]
-name = "attester"
-version = "0.1.0"
-authors = ["Paul Hauner <paul@paulhauner.com>"]
-edition = "2018"
-
-[dependencies]
-slot_clock = { path = "../../eth2/utils/slot_clock" }
-ssz = { path = "../../eth2/utils/ssz" }
-types = { path = "../../eth2/types" }

eth2/attester/src/lib.rs

@@ -1,257 +0,0 @@
-pub mod test_utils;
-mod traits;
-
-use slot_clock::SlotClock;
-use ssz::TreeHash;
-use std::sync::Arc;
-use types::{AttestationData, AttestationDataAndCustodyBit, FreeAttestation, Signature, Slot};
-
-pub use self::traits::{
-    BeaconNode, BeaconNodeError, DutiesReader, DutiesReaderError, PublishOutcome, Signer,
-};
-
-const PHASE_0_CUSTODY_BIT: bool = false;
-const DOMAIN_ATTESTATION: u64 = 1;
-
-#[derive(Debug, PartialEq)]
-pub enum PollOutcome {
-    AttestationProduced(Slot),
-    AttestationNotRequired(Slot),
-    SlashableAttestationNotProduced(Slot),
-    BeaconNodeUnableToProduceAttestation(Slot),
-    ProducerDutiesUnknown(Slot),
-    SlotAlreadyProcessed(Slot),
-    SignerRejection(Slot),
-    ValidatorIsUnknown(Slot),
-}
-
-#[derive(Debug, PartialEq)]
-pub enum Error {
-    SlotClockError,
-    SlotUnknowable,
-    EpochMapPoisoned,
-    SlotClockPoisoned,
-    EpochLengthIsZero,
-    BeaconNodeError(BeaconNodeError),
-}
-
-/// A polling state machine which performs block production duties, based upon some epoch duties
-/// (`EpochDutiesMap`) and a concept of time (`SlotClock`).
-///
-/// Ensures that messages are not slashable.
-///
-/// Relies upon an external service to keep the `EpochDutiesMap` updated.
-pub struct Attester<T: SlotClock, U: BeaconNode, V: DutiesReader, W: Signer> {
-    pub last_processed_slot: Option<Slot>,
-    duties: Arc<V>,
-    slot_clock: Arc<T>,
-    beacon_node: Arc<U>,
-    signer: Arc<W>,
-}
-
-impl<T: SlotClock, U: BeaconNode, V: DutiesReader, W: Signer> Attester<T, U, V, W> {
-    /// Returns a new instance where `last_processed_slot == 0`.
-    pub fn new(duties: Arc<V>, slot_clock: Arc<T>, beacon_node: Arc<U>, signer: Arc<W>) -> Self {
-        Self {
-            last_processed_slot: None,
-            duties,
-            slot_clock,
-            beacon_node,
-            signer,
-        }
-    }
-}
-
-impl<T: SlotClock, U: BeaconNode, V: DutiesReader, W: Signer> Attester<T, U, V, W> {
-    /// Poll the `BeaconNode` and produce an attestation if required.
-    pub fn poll(&mut self) -> Result<PollOutcome, Error> {
-        let slot = self
-            .slot_clock
-            .present_slot()
-            .map_err(|_| Error::SlotClockError)?
-            .ok_or(Error::SlotUnknowable)?;
-
-        if !self.is_processed_slot(slot) {
-            self.last_processed_slot = Some(slot);
-
-            let shard = match self.duties.attestation_shard(slot) {
-                Ok(Some(result)) => result,
-                Ok(None) => return Ok(PollOutcome::AttestationNotRequired(slot)),
-                Err(DutiesReaderError::UnknownEpoch) => {
-                    return Ok(PollOutcome::ProducerDutiesUnknown(slot));
-                }
-                Err(DutiesReaderError::UnknownValidator) => {
-                    return Ok(PollOutcome::ValidatorIsUnknown(slot));
-                }
-                Err(DutiesReaderError::EpochLengthIsZero) => return Err(Error::EpochLengthIsZero),
-                Err(DutiesReaderError::Poisoned) => return Err(Error::EpochMapPoisoned),
-            };
-
-            self.produce_attestation(slot, shard)
-        } else {
-            Ok(PollOutcome::SlotAlreadyProcessed(slot))
-        }
-    }
-
-    fn produce_attestation(&mut self, slot: Slot, shard: u64) -> Result<PollOutcome, Error> {
-        let attestation_data = match self.beacon_node.produce_attestation_data(slot, shard)? {
-            Some(attestation_data) => attestation_data,
-            None => return Ok(PollOutcome::BeaconNodeUnableToProduceAttestation(slot)),
-        };
-
-        if !self.safe_to_produce(&attestation_data) {
-            return Ok(PollOutcome::SlashableAttestationNotProduced(slot));
-        }
-
-        let signature = match self.sign_attestation_data(&attestation_data) {
-            Some(signature) => signature,
-            None => return Ok(PollOutcome::SignerRejection(slot)),
-        };
-
-        let validator_index = match self.duties.validator_index() {
-            Some(validator_index) => validator_index,
-            None => return Ok(PollOutcome::ValidatorIsUnknown(slot)),
-        };
-
-        let free_attestation = FreeAttestation {
-            data: attestation_data,
-            signature,
-            validator_index,
-        };
-
-        self.beacon_node.publish_attestation(free_attestation)?;
-        Ok(PollOutcome::AttestationProduced(slot))
-    }
-
-    fn is_processed_slot(&self, slot: Slot) -> bool {
-        match self.last_processed_slot {
-            Some(processed_slot) if slot <= processed_slot => true,
-            _ => false,
-        }
-    }
-
-    /// Consumes a block, returning that block signed by the validators private key.
-    ///
-    /// Important: this function will not check to ensure the block is not slashable. This must be
-    /// done upstream.
-    fn sign_attestation_data(&mut self, attestation_data: &AttestationData) -> Option<Signature> {
-        self.store_produce(attestation_data);
-
-        let message = AttestationDataAndCustodyBit {
-            data: attestation_data.clone(),
-            custody_bit: PHASE_0_CUSTODY_BIT,
-        }
-        .hash_tree_root();
-
-        self.signer
-            .sign_attestation_message(&message[..], DOMAIN_ATTESTATION)
-    }
-
-    /// Returns `true` if signing some attestation_data is safe (non-slashable).
-    ///
-    /// !!! UNSAFE !!!
-    ///
-    /// Important: this function is presently stubbed-out. It provides ZERO SAFETY.
-    fn safe_to_produce(&self, _attestation_data: &AttestationData) -> bool {
-        // TODO: ensure the producer doesn't produce slashable blocks.
-        // https://github.com/sigp/lighthouse/issues/160
-        true
-    }
-
-    /// Record that a block was produced so that slashable votes may not be made in the future.
-    ///
-    /// !!! UNSAFE !!!
-    ///
-    /// Important: this function is presently stubbed-out. It provides ZERO SAFETY.
-    fn store_produce(&mut self, _block: &AttestationData) {
-        // TODO: record this block production to prevent future slashings.
-        // https://github.com/sigp/lighthouse/issues/160
-    }
-}
-
-impl From<BeaconNodeError> for Error {
-    fn from(e: BeaconNodeError) -> Error {
-        Error::BeaconNodeError(e)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::test_utils::{EpochMap, LocalSigner, SimulatedBeaconNode};
-    use super::*;
-    use slot_clock::TestingSlotClock;
-    use types::{
-        test_utils::{SeedableRng, TestRandom, XorShiftRng},
-        ChainSpec, Keypair,
-    };
-
-    // TODO: implement more thorough testing.
-    // https://github.com/sigp/lighthouse/issues/160
-    //
-    // These tests should serve as a good example for future tests.
-
-    #[test]
-    pub fn polling() {
-        let mut rng = XorShiftRng::from_seed([42; 16]);
-
-        let spec = Arc::new(ChainSpec::foundation());
-        let slot_clock = Arc::new(TestingSlotClock::new(0));
-        let beacon_node = Arc::new(SimulatedBeaconNode::default());
-        let signer = Arc::new(LocalSigner::new(Keypair::random()));
-
-        let mut duties = EpochMap::new(spec.slots_per_epoch);
-        let attest_slot = Slot::new(100);
-        let attest_epoch = attest_slot / spec.slots_per_epoch;
-        let attest_shard = 12;
-        duties.insert_attestation_shard(attest_slot, attest_shard);
-        duties.set_validator_index(Some(2));
-        let duties = Arc::new(duties);
-
-        let mut attester = Attester::new(
-            duties.clone(),
-            slot_clock.clone(),
-            beacon_node.clone(),
-            signer.clone(),
-        );
-
-        // Configure responses from the BeaconNode.
-        beacon_node.set_next_produce_result(Ok(Some(AttestationData::random_for_test(&mut rng))));
-        beacon_node.set_next_publish_result(Ok(PublishOutcome::ValidAttestation));
-
-        // One slot before attestation slot...
-        slot_clock.set_slot(attest_slot.as_u64() - 1);
-        assert_eq!(
-            attester.poll(),
-            Ok(PollOutcome::AttestationNotRequired(attest_slot - 1))
-        );
-
-        // On the attest slot...
-        slot_clock.set_slot(attest_slot.as_u64());
-        assert_eq!(
-            attester.poll(),
-            Ok(PollOutcome::AttestationProduced(attest_slot))
-        );
-
-        // Trying the same attest slot again...
-        slot_clock.set_slot(attest_slot.as_u64());
-        assert_eq!(
-            attester.poll(),
-            Ok(PollOutcome::SlotAlreadyProcessed(attest_slot))
-        );
-
-        // One slot after the attest slot...
-        slot_clock.set_slot(attest_slot.as_u64() + 1);
-        assert_eq!(
-            attester.poll(),
-            Ok(PollOutcome::AttestationNotRequired(attest_slot + 1))
-        );
-
-        // In an epoch without known duties...
-        let slot = (attest_epoch + 1) * spec.slots_per_epoch;
-        slot_clock.set_slot(slot.into());
-        assert_eq!(
-            attester.poll(),
-            Ok(PollOutcome::ProducerDutiesUnknown(slot))
-        );
-    }
-}

eth2/attester/src/test_utils/epoch_map.rs

@@ -1,44 +0,0 @@
-use crate::{DutiesReader, DutiesReaderError};
-use std::collections::HashMap;
-use types::{Epoch, Slot};
-
-pub struct EpochMap {
-    slots_per_epoch: u64,
-    validator_index: Option<u64>,
-    map: HashMap<Epoch, (Slot, u64)>,
-}
-
-impl EpochMap {
-    pub fn new(slots_per_epoch: u64) -> Self {
-        Self {
-            slots_per_epoch,
-            validator_index: None,
-            map: HashMap::new(),
-        }
-    }
-
-    pub fn insert_attestation_shard(&mut self, slot: Slot, shard: u64) {
-        let epoch = slot.epoch(self.slots_per_epoch);
-        self.map.insert(epoch, (slot, shard));
-    }
-
-    pub fn set_validator_index(&mut self, index: Option<u64>) {
-        self.validator_index = index;
-    }
-}
-
-impl DutiesReader for EpochMap {
-    fn attestation_shard(&self, slot: Slot) -> Result<Option<u64>, DutiesReaderError> {
-        let epoch = slot.epoch(self.slots_per_epoch);
-
-        match self.map.get(&epoch) {
-            Some((attest_slot, attest_shard)) if *attest_slot == slot => Ok(Some(*attest_shard)),
-            Some((attest_slot, _attest_shard)) if *attest_slot != slot => Ok(None),
-            _ => Err(DutiesReaderError::UnknownEpoch),
-        }
-    }
-
-    fn validator_index(&self) -> Option<u64> {
-        self.validator_index
-    }
-}

eth2/attester/src/test_utils/local_signer.rs

@@ -1,31 +0,0 @@
-use crate::traits::Signer;
-use std::sync::RwLock;
-use types::{Keypair, Signature};
-
-/// A test-only struct used to simulate a Beacon Node.
-pub struct LocalSigner {
-    keypair: Keypair,
-    should_sign: RwLock<bool>,
-}
-
-impl LocalSigner {
-    /// Produce a new LocalSigner with signing enabled by default.
-    pub fn new(keypair: Keypair) -> Self {
-        Self {
-            keypair,
-            should_sign: RwLock::new(true),
-        }
-    }
-
-    /// If set to `false`, the service will refuse to sign all messages. Otherwise, all messages
-    /// will be signed.
-    pub fn enable_signing(&self, enabled: bool) {
-        *self.should_sign.write().unwrap() = enabled;
-    }
-}
-
-impl Signer for LocalSigner {
-    fn sign_attestation_message(&self, message: &[u8], domain: u64) -> Option<Signature> {
-        Some(Signature::new(message, domain, &self.keypair.sk))
-    }
-}

eth2/attester/src/test_utils/mod.rs

@@ -1,7 +0,0 @@
-mod epoch_map;
-mod local_signer;
-mod simulated_beacon_node;
-
-pub use self::epoch_map::EpochMap;
-pub use self::local_signer::LocalSigner;
-pub use self::simulated_beacon_node::SimulatedBeaconNode;

eth2/attester/src/test_utils/simulated_beacon_node.rs

@@ -1,44 +0,0 @@
-use crate::traits::{BeaconNode, BeaconNodeError, PublishOutcome};
-use std::sync::RwLock;
-use types::{AttestationData, FreeAttestation, Slot};
-
-type ProduceResult = Result<Option<AttestationData>, BeaconNodeError>;
-type PublishResult = Result<PublishOutcome, BeaconNodeError>;
-
-/// A test-only struct used to simulate a Beacon Node.
-#[derive(Default)]
-pub struct SimulatedBeaconNode {
-    pub produce_input: RwLock<Option<(Slot, u64)>>,
-    pub produce_result: RwLock<Option<ProduceResult>>,
-
-    pub publish_input: RwLock<Option<FreeAttestation>>,
-    pub publish_result: RwLock<Option<PublishResult>>,
-}
-
-impl SimulatedBeaconNode {
-    pub fn set_next_produce_result(&self, result: ProduceResult) {
-        *self.produce_result.write().unwrap() = Some(result);
-    }
-
-    pub fn set_next_publish_result(&self, result: PublishResult) {
-        *self.publish_result.write().unwrap() = Some(result);
-    }
-}
-
-impl BeaconNode for SimulatedBeaconNode {
-    fn produce_attestation_data(&self, slot: Slot, shard: u64) -> ProduceResult {
-        *self.produce_input.write().unwrap() = Some((slot, shard));
-        match *self.produce_result.read().unwrap() {
-            Some(ref r) => r.clone(),
-            None => panic!("TestBeaconNode: produce_result == None"),
-        }
-    }
-
-    fn publish_attestation(&self, free_attestation: FreeAttestation) -> PublishResult {
-        *self.publish_input.write().unwrap() = Some(free_attestation.clone());
-        match *self.publish_result.read().unwrap() {
-            Some(ref r) => r.clone(),
-            None => panic!("TestBeaconNode: publish_result == None"),
-        }
-    }
-}

eth2/attester/src/traits.rs

@@ -1,49 +0,0 @@
-use types::{AttestationData, FreeAttestation, Signature, Slot};
-
-#[derive(Debug, PartialEq, Clone)]
-pub enum BeaconNodeError {
-    RemoteFailure(String),
-    DecodeFailure,
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub enum PublishOutcome {
-    ValidAttestation,
-    InvalidAttestation(String),
-}
-
-/// Defines the methods required to produce and publish blocks on a Beacon Node.
-pub trait BeaconNode: Send + Sync {
-    fn produce_attestation_data(
-        &self,
-        slot: Slot,
-        shard: u64,
-    ) -> Result<Option<AttestationData>, BeaconNodeError>;
-
-    fn publish_attestation(
-        &self,
-        free_attestation: FreeAttestation,
-    ) -> Result<PublishOutcome, BeaconNodeError>;
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub enum DutiesReaderError {
-    UnknownValidator,
-    UnknownEpoch,
-    EpochLengthIsZero,
-    Poisoned,
-}
-
-/// Informs a validator of their duties (e.g., block production).
-pub trait DutiesReader: Send + Sync {
-    /// Returns `Some(shard)` if this slot is an attestation slot. Otherwise, returns `None.`
-    fn attestation_shard(&self, slot: Slot) -> Result<Option<u64>, DutiesReaderError>;
-
-    /// Returns `Some(shard)` if this slot is an attestation slot. Otherwise, returns `None.`
-    fn validator_index(&self) -> Option<u64>;
-}
-
-/// Signs message using an internally-maintained private key.
-pub trait Signer {
-    fn sign_attestation_message(&self, message: &[u8], domain: u64) -> Option<Signature>;
-}

eth2/block_proposer/Cargo.toml

@@ -1,11 +0,0 @@
-[package]
-name = "block_proposer"
-version = "0.1.0"
-authors = ["Paul Hauner <paul@paulhauner.com>"]
-edition = "2018"
-
-[dependencies]
-int_to_bytes = { path = "../utils/int_to_bytes" }
-slot_clock = { path = "../utils/slot_clock" }
-ssz = { path = "../utils/ssz" }
-types = { path = "../types" }

eth2/block_proposer/src/lib.rs

@@ -1,303 +0,0 @@
-pub mod test_utils;
-mod traits;
-
-use slot_clock::SlotClock;
-use ssz::{SignedRoot, TreeHash};
-use std::sync::Arc;
-use types::{BeaconBlock, ChainSpec, Domain, Slot};
-
-pub use self::traits::{
-    BeaconNode, BeaconNodeError, DutiesReader, DutiesReaderError, PublishOutcome, Signer,
-};
-
-#[derive(Debug, PartialEq)]
-pub enum PollOutcome {
-    /// A new block was produced.
-    BlockProduced(Slot),
-    /// A block was not produced as it would have been slashable.
-    SlashableBlockNotProduced(Slot),
-    /// The validator duties did not require a block to be produced.
-    BlockProductionNotRequired(Slot),
-    /// The duties for the present epoch were not found.
-    ProducerDutiesUnknown(Slot),
-    /// The slot has already been processed, execution was skipped.
-    SlotAlreadyProcessed(Slot),
-    /// The Beacon Node was unable to produce a block at that slot.
-    BeaconNodeUnableToProduceBlock(Slot),
-    /// The signer failed to sign the message.
-    SignerRejection(Slot),
-    /// The public key for this validator is not an active validator.
-    ValidatorIsUnknown(Slot),
-    /// Unable to determine a `Fork` for signature domain generation.
-    UnableToGetFork(Slot),
-}
-
-#[derive(Debug, PartialEq)]
-pub enum Error {
-    SlotClockError,
-    SlotUnknowable,
-    EpochMapPoisoned,
-    SlotClockPoisoned,
-    EpochLengthIsZero,
-    BeaconNodeError(BeaconNodeError),
-}
-
-/// A polling state machine which performs block production duties, based upon some epoch duties
-/// (`EpochDutiesMap`) and a concept of time (`SlotClock`).
-///
-/// Ensures that messages are not slashable.
-///
-/// Relies upon an external service to keep the `EpochDutiesMap` updated.
-pub struct BlockProducer<T: SlotClock, U: BeaconNode, V: DutiesReader, W: Signer> {
-    pub last_processed_slot: Option<Slot>,
-    spec: Arc<ChainSpec>,
-    epoch_map: Arc<V>,
-    slot_clock: Arc<T>,
-    beacon_node: Arc<U>,
-    signer: Arc<W>,
-}
-
-impl<T: SlotClock, U: BeaconNode, V: DutiesReader, W: Signer> BlockProducer<T, U, V, W> {
-    /// Returns a new instance where `last_processed_slot == 0`.
-    pub fn new(
-        spec: Arc<ChainSpec>,
-        epoch_map: Arc<V>,
-        slot_clock: Arc<T>,
-        beacon_node: Arc<U>,
-        signer: Arc<W>,
-    ) -> Self {
-        Self {
-            last_processed_slot: None,
-            spec,
-            epoch_map,
-            slot_clock,
-            beacon_node,
-            signer,
-        }
-    }
-}
-
-impl<T: SlotClock, U: BeaconNode, V: DutiesReader, W: Signer> BlockProducer<T, U, V, W> {
-    /// "Poll" to see if the validator is required to take any action.
-    ///
-    /// The slot clock will be read and any new actions undertaken.
-    pub fn poll(&mut self) -> Result<PollOutcome, Error> {
-        let slot = self
-            .slot_clock
-            .present_slot()
-            .map_err(|_| Error::SlotClockError)?
-            .ok_or(Error::SlotUnknowable)?;
-
-        // If this is a new slot.
-        if !self.is_processed_slot(slot) {
-            let is_block_production_slot = match self.epoch_map.is_block_production_slot(slot) {
-                Ok(result) => result,
-                Err(DutiesReaderError::UnknownEpoch) => {
-                    return Ok(PollOutcome::ProducerDutiesUnknown(slot));
-                }
-                Err(DutiesReaderError::UnknownValidator) => {
-                    return Ok(PollOutcome::ValidatorIsUnknown(slot));
-                }
-                Err(DutiesReaderError::EpochLengthIsZero) => return Err(Error::EpochLengthIsZero),
-                Err(DutiesReaderError::Poisoned) => return Err(Error::EpochMapPoisoned),
-            };
-
-            if is_block_production_slot {
-                self.last_processed_slot = Some(slot);
-
-                self.produce_block(slot)
-            } else {
-                Ok(PollOutcome::BlockProductionNotRequired(slot))
-            }
-        } else {
-            Ok(PollOutcome::SlotAlreadyProcessed(slot))
-        }
-    }
-
-    fn is_processed_slot(&self, slot: Slot) -> bool {
-        match self.last_processed_slot {
-            Some(processed_slot) if processed_slot >= slot => true,
-            _ => false,
-        }
-    }
-
-    /// Produce a block at some slot.
-    ///
-    /// Assumes that a block is required at this slot (does not check the duties).
-    ///
-    /// Ensures the message is not slashable.
-    ///
-    /// !!! UNSAFE !!!
-    ///
-    /// The slash-protection code is not yet implemented. There is zero protection against
-    /// slashing.
-    fn produce_block(&mut self, slot: Slot) -> Result<PollOutcome, Error> {
-        let fork = match self.epoch_map.fork() {
-            Ok(fork) => fork,
-            Err(_) => return Ok(PollOutcome::UnableToGetFork(slot)),
-        };
-
-        let randao_reveal = {
-            // TODO: add domain, etc to this message. Also ensure result matches `into_to_bytes32`.
-            let message = slot.epoch(self.spec.slots_per_epoch).hash_tree_root();
-
-            match self.signer.sign_randao_reveal(
-                &message,
-                self.spec
-                    .get_domain(slot.epoch(self.spec.slots_per_epoch), Domain::Randao, &fork),
-            ) {
-                None => return Ok(PollOutcome::SignerRejection(slot)),
-                Some(signature) => signature,
-            }
-        };
-
-        if let Some(block) = self
-            .beacon_node
-            .produce_beacon_block(slot, &randao_reveal)?
-        {
-            if self.safe_to_produce(&block) {
-                let domain = self.spec.get_domain(
-                    slot.epoch(self.spec.slots_per_epoch),
-                    Domain::BeaconBlock,
-                    &fork,
-                );
-                if let Some(block) = self.sign_block(block, domain) {
-                    self.beacon_node.publish_beacon_block(block)?;
-                    Ok(PollOutcome::BlockProduced(slot))
-                } else {
-                    Ok(PollOutcome::SignerRejection(slot))
-                }
-            } else {
-                Ok(PollOutcome::SlashableBlockNotProduced(slot))
-            }
-        } else {
-            Ok(PollOutcome::BeaconNodeUnableToProduceBlock(slot))
-        }
-    }
-
-    /// Consumes a block, returning that block signed by the validators private key.
-    ///
-    /// Important: this function will not check to ensure the block is not slashable. This must be
-    /// done upstream.
-    fn sign_block(&mut self, mut block: BeaconBlock, domain: u64) -> Option<BeaconBlock> {
-        self.store_produce(&block);
-
-        match self
-            .signer
-            .sign_block_proposal(&block.signed_root()[..], domain)
-        {
-            None => None,
-            Some(signature) => {
-                block.signature = signature;
-                Some(block)
-            }
-        }
-    }
-
-    /// Returns `true` if signing a block is safe (non-slashable).
-    ///
-    /// !!! UNSAFE !!!
-    ///
-    /// Important: this function is presently stubbed-out. It provides ZERO SAFETY.
-    fn safe_to_produce(&self, _block: &BeaconBlock) -> bool {
-        // TODO: ensure the producer doesn't produce slashable blocks.
-        // https://github.com/sigp/lighthouse/issues/160
-        true
-    }
-
-    /// Record that a block was produced so that slashable votes may not be made in the future.
-    ///
-    /// !!! UNSAFE !!!
-    ///
-    /// Important: this function is presently stubbed-out. It provides ZERO SAFETY.
-    fn store_produce(&mut self, _block: &BeaconBlock) {
-        // TODO: record this block production to prevent future slashings.
-        // https://github.com/sigp/lighthouse/issues/160
-    }
-}
-
-impl From<BeaconNodeError> for Error {
-    fn from(e: BeaconNodeError) -> Error {
-        Error::BeaconNodeError(e)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::test_utils::{EpochMap, LocalSigner, SimulatedBeaconNode};
-    use super::*;
-    use slot_clock::TestingSlotClock;
-    use types::{
-        test_utils::{SeedableRng, TestRandom, XorShiftRng},
-        Keypair,
-    };
-
-    // TODO: implement more thorough testing.
-    // https://github.com/sigp/lighthouse/issues/160
-    //
-    // These tests should serve as a good example for future tests.
-
-    #[test]
-    pub fn polling() {
-        let mut rng = XorShiftRng::from_seed([42; 16]);
-
-        let spec = Arc::new(ChainSpec::foundation());
-        let slot_clock = Arc::new(TestingSlotClock::new(0));
-        let beacon_node = Arc::new(SimulatedBeaconNode::default());
-        let signer = Arc::new(LocalSigner::new(Keypair::random()));
-
-        let mut epoch_map = EpochMap::new(spec.slots_per_epoch);
-        let produce_slot = Slot::new(100);
-        let produce_epoch = produce_slot.epoch(spec.slots_per_epoch);
-        epoch_map.map.insert(produce_epoch, produce_slot);
-        let epoch_map = Arc::new(epoch_map);
-
-        let mut block_proposer = BlockProducer::new(
-            spec.clone(),
-            epoch_map.clone(),
-            slot_clock.clone(),
-            beacon_node.clone(),
-            signer.clone(),
-        );
-
-        // Configure responses from the BeaconNode.
-        beacon_node.set_next_produce_result(Ok(Some(BeaconBlock::random_for_test(&mut rng))));
-        beacon_node.set_next_publish_result(Ok(PublishOutcome::ValidBlock));
-
-        // One slot before production slot...
-        slot_clock.set_slot(produce_slot.as_u64() - 1);
-        assert_eq!(
-            block_proposer.poll(),
-            Ok(PollOutcome::BlockProductionNotRequired(produce_slot - 1))
-        );
-
-        // On the produce slot...
-        slot_clock.set_slot(produce_slot.as_u64());
-        assert_eq!(
-            block_proposer.poll(),
-            Ok(PollOutcome::BlockProduced(produce_slot.into()))
-        );
-
-        // Trying the same produce slot again...
-        slot_clock.set_slot(produce_slot.as_u64());
-        assert_eq!(
-            block_proposer.poll(),
-            Ok(PollOutcome::SlotAlreadyProcessed(produce_slot))
-        );
-
-        // One slot after the produce slot...
-        slot_clock.set_slot(produce_slot.as_u64() + 1);
-        assert_eq!(
-            block_proposer.poll(),
-            Ok(PollOutcome::BlockProductionNotRequired(produce_slot + 1))
-        );
-
-        // In an epoch without known duties...
-        let slot = (produce_epoch.as_u64() + 1) * spec.slots_per_epoch;
-        slot_clock.set_slot(slot);
-        assert_eq!(
-            block_proposer.poll(),
-            Ok(PollOutcome::ProducerDutiesUnknown(Slot::new(slot)))
-        );
-    }
-}

eth2/block_proposer/src/test_utils/epoch_map.rs

@@ -1,36 +0,0 @@
-use crate::{DutiesReader, DutiesReaderError};
-use std::collections::HashMap;
-use types::{Epoch, Fork, Slot};
-
-pub struct EpochMap {
-    slots_per_epoch: u64,
-    pub map: HashMap<Epoch, Slot>,
-}
-
-impl EpochMap {
-    pub fn new(slots_per_epoch: u64) -> Self {
-        Self {
-            slots_per_epoch,
-            map: HashMap::new(),
-        }
-    }
-}
-
-impl DutiesReader for EpochMap {
-    fn is_block_production_slot(&self, slot: Slot) -> Result<bool, DutiesReaderError> {
-        let epoch = slot.epoch(self.slots_per_epoch);
-        match self.map.get(&epoch) {
-            Some(s) if *s == slot => Ok(true),
-            Some(s) if *s != slot => Ok(false),
-            _ => Err(DutiesReaderError::UnknownEpoch),
-        }
-    }
-
-    fn fork(&self) -> Result<Fork, DutiesReaderError> {
-        Ok(Fork {
-            previous_version: [0; 4],
-            current_version: [0; 4],
-            epoch: Epoch::new(0),
-        })
-    }
-}

eth2/block_proposer/src/test_utils/local_signer.rs

@@ -1,35 +0,0 @@
-use crate::traits::Signer;
-use std::sync::RwLock;
-use types::{Keypair, Signature};
-
-/// A test-only struct used to simulate a Beacon Node.
-pub struct LocalSigner {
-    keypair: Keypair,
-    should_sign: RwLock<bool>,
-}
-
-impl LocalSigner {
-    /// Produce a new LocalSigner with signing enabled by default.
-    pub fn new(keypair: Keypair) -> Self {
-        Self {
-            keypair,
-            should_sign: RwLock::new(true),
-        }
-    }
-
-    /// If set to `false`, the service will refuse to sign all messages. Otherwise, all messages
-    /// will be signed.
-    pub fn enable_signing(&self, enabled: bool) {
-        *self.should_sign.write().unwrap() = enabled;
-    }
-}
-
-impl Signer for LocalSigner {
-    fn sign_block_proposal(&self, message: &[u8], domain: u64) -> Option<Signature> {
-        Some(Signature::new(message, domain, &self.keypair.sk))
-    }
-
-    fn sign_randao_reveal(&self, message: &[u8], domain: u64) -> Option<Signature> {
-        Some(Signature::new(message, domain, &self.keypair.sk))
-    }
-}

eth2/block_proposer/src/test_utils/mod.rs

@@ -1,7 +0,0 @@
-mod epoch_map;
-mod local_signer;
-mod simulated_beacon_node;
-
-pub use self::epoch_map::EpochMap;
-pub use self::local_signer::LocalSigner;
-pub use self::simulated_beacon_node::SimulatedBeaconNode;

eth2/block_proposer/src/test_utils/simulated_beacon_node.rs

@@ -1,48 +0,0 @@
-use crate::traits::{BeaconNode, BeaconNodeError, PublishOutcome};
-use std::sync::RwLock;
-use types::{BeaconBlock, Signature, Slot};
-
-type ProduceResult = Result<Option<BeaconBlock>, BeaconNodeError>;
-type PublishResult = Result<PublishOutcome, BeaconNodeError>;
-
-/// A test-only struct used to simulate a Beacon Node.
-#[derive(Default)]
-pub struct SimulatedBeaconNode {
-    pub produce_input: RwLock<Option<(Slot, Signature)>>,
-    pub produce_result: RwLock<Option<ProduceResult>>,
-
-    pub publish_input: RwLock<Option<BeaconBlock>>,
-    pub publish_result: RwLock<Option<PublishResult>>,
-}
-
-impl SimulatedBeaconNode {
-    /// Set the result to be returned when `produce_beacon_block` is called.
-    pub fn set_next_produce_result(&self, result: ProduceResult) {
-        *self.produce_result.write().unwrap() = Some(result);
-    }
-
-    /// Set the result to be returned when `publish_beacon_block` is called.
-    pub fn set_next_publish_result(&self, result: PublishResult) {
-        *self.publish_result.write().unwrap() = Some(result);
-    }
-}
-
-impl BeaconNode for SimulatedBeaconNode {
-    /// Returns the value specified by the `set_next_produce_result`.
-    fn produce_beacon_block(&self, slot: Slot, randao_reveal: &Signature) -> ProduceResult {
-        *self.produce_input.write().unwrap() = Some((slot, randao_reveal.clone()));
-        match *self.produce_result.read().unwrap() {
-            Some(ref r) => r.clone(),
-            None => panic!("SimulatedBeaconNode: produce_result == None"),
-        }
-    }
-
-    /// Returns the value specified by the `set_next_publish_result`.
-    fn publish_beacon_block(&self, block: BeaconBlock) -> PublishResult {
-        *self.publish_input.write().unwrap() = Some(block);
-        match *self.publish_result.read().unwrap() {
-            Some(ref r) => r.clone(),
-            None => panic!("SimulatedBeaconNode: publish_result == None"),
-        }
-    }
-}

eth2/block_proposer/src/traits.rs

@@ -1,50 +0,0 @@
-use types::{BeaconBlock, Fork, Signature, Slot};
-
-#[derive(Debug, PartialEq, Clone)]
-pub enum BeaconNodeError {
-    RemoteFailure(String),
-    DecodeFailure,
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub enum PublishOutcome {
-    ValidBlock,
-    InvalidBlock(String),
-}
-
-/// Defines the methods required to produce and publish blocks on a Beacon Node.
-pub trait BeaconNode: Send + Sync {
-    /// Request that the node produces a block.
-    ///
-    /// Returns Ok(None) if the Beacon Node is unable to produce at the given slot.
-    fn produce_beacon_block(
-        &self,
-        slot: Slot,
-        randao_reveal: &Signature,
-    ) -> Result<Option<BeaconBlock>, BeaconNodeError>;
-
-    /// Request that the node publishes a block.
-    ///
-    /// Returns `true` if the publish was sucessful.
-    fn publish_beacon_block(&self, block: BeaconBlock) -> Result<PublishOutcome, BeaconNodeError>;
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub enum DutiesReaderError {
-    UnknownValidator,
-    UnknownEpoch,
-    EpochLengthIsZero,
-    Poisoned,
-}
-
-/// Informs a validator of their duties (e.g., block production).
-pub trait DutiesReader: Send + Sync {
-    fn is_block_production_slot(&self, slot: Slot) -> Result<bool, DutiesReaderError>;
-    fn fork(&self) -> Result<Fork, DutiesReaderError>;
-}
-
-/// Signs message using an internally-maintained private key.
-pub trait Signer {
-    fn sign_block_proposal(&self, message: &[u8], domain: u64) -> Option<Signature>;
-    fn sign_randao_reveal(&self, message: &[u8], domain: u64) -> Option<Signature>;
-}

eth2/fork_choice/src/bitwise_lmd_ghost.rs

@@ -9,8 +9,9 @@ use db::{
 };
 use log::{debug, trace};
 use std::collections::HashMap;
+use std::marker::PhantomData;
 use std::sync::Arc;
-use types::{BeaconBlock, ChainSpec, Hash256, Slot, SlotHeight};
+use types::{BeaconBlock, BeaconState, ChainSpec, EthSpec, Hash256, Slot, SlotHeight};
 
 //TODO: Pruning - Children
 //TODO: Handle Syncing
@@ -33,7 +34,7 @@ fn power_of_2_below(x: u64) -> u64 {
 }
 
 /// Stores the necessary data structures to run the optimised bitwise lmd ghost algorithm.
-pub struct BitwiseLMDGhost<T: ClientDB + Sized> {
+pub struct BitwiseLMDGhost<T: ClientDB + Sized, E> {
     /// A cache of known ancestors at given heights for a specific block.
     //TODO: Consider FnvHashMap
     cache: HashMap<CacheKey<u64>, Hash256>,
@@ -50,9 +51,10 @@ pub struct BitwiseLMDGhost<T: ClientDB + Sized> {
     /// State storage access.
     state_store: Arc<BeaconStateStore<T>>,
     max_known_height: SlotHeight,
+    _phantom: PhantomData<E>,
 }
 
-impl<T> BitwiseLMDGhost<T>
+impl<T, E: EthSpec> BitwiseLMDGhost<T, E>
 where
     T: ClientDB + Sized,
 {
@@ -68,6 +70,7 @@ where
             max_known_height: SlotHeight::new(0),
             block_store,
             state_store,
+            _phantom: PhantomData,
         }
     }
 
@@ -85,7 +88,7 @@ where
         // build a hashmap of block_hash to weighted votes
         let mut latest_votes: HashMap<Hash256, u64> = HashMap::new();
         // gets the current weighted votes
-        let current_state = self
+        let current_state: BeaconState<E> = self
             .state_store
             .get_deserialized(&state_root)?
             .ok_or_else(|| ForkChoiceError::MissingBeaconState(*state_root))?;
@@ -240,7 +243,7 @@ where
     }
 }
 
-impl<T: ClientDB + Sized> ForkChoice for BitwiseLMDGhost<T> {
+impl<T: ClientDB + Sized, E: EthSpec> ForkChoice for BitwiseLMDGhost<T, E> {
     fn add_block(
         &mut self,
         block: &BeaconBlock,

eth2/fork_choice/src/optimized_lmd_ghost.rs

@@ -9,8 +9,9 @@ use db::{
 use log::{debug, trace};
 use std::cmp::Ordering;
 use std::collections::HashMap;
+use std::marker::PhantomData;
 use std::sync::Arc;
-use types::{BeaconBlock, ChainSpec, Hash256, Slot, SlotHeight};
+use types::{BeaconBlock, BeaconState, ChainSpec, EthSpec, Hash256, Slot, SlotHeight};
 
 //TODO: Pruning - Children
 //TODO: Handle Syncing
@@ -33,7 +34,7 @@ fn power_of_2_below(x: u64) -> u64 {
 }
 
 /// Stores the necessary data structures to run the optimised lmd ghost algorithm.
-pub struct OptimizedLMDGhost<T: ClientDB + Sized> {
+pub struct OptimizedLMDGhost<T: ClientDB + Sized, E> {
     /// A cache of known ancestors at given heights for a specific block.
     //TODO: Consider FnvHashMap
     cache: HashMap<CacheKey<u64>, Hash256>,
@@ -50,9 +51,10 @@ pub struct OptimizedLMDGhost<T: ClientDB + Sized> {
     /// State storage access.
     state_store: Arc<BeaconStateStore<T>>,
     max_known_height: SlotHeight,
+    _phantom: PhantomData<E>,
 }
 
-impl<T> OptimizedLMDGhost<T>
+impl<T, E: EthSpec> OptimizedLMDGhost<T, E>
 where
     T: ClientDB + Sized,
 {
@@ -68,6 +70,7 @@ where
             max_known_height: SlotHeight::new(0),
             block_store,
             state_store,
+            _phantom: PhantomData,
         }
     }
 
@@ -85,7 +88,7 @@ where
         // build a hashmap of block_hash to weighted votes
         let mut latest_votes: HashMap<Hash256, u64> = HashMap::new();
         // gets the current weighted votes
-        let current_state = self
+        let current_state: BeaconState<E> = self
             .state_store
             .get_deserialized(&state_root)?
             .ok_or_else(|| ForkChoiceError::MissingBeaconState(*state_root))?;
@@ -211,7 +214,7 @@ where
     }
 }
 
-impl<T: ClientDB + Sized> ForkChoice for OptimizedLMDGhost<T> {
+impl<T: ClientDB + Sized, E: EthSpec> ForkChoice for OptimizedLMDGhost<T, E> {
     fn add_block(
         &mut self,
         block: &BeaconBlock,

eth2/fork_choice/src/slow_lmd_ghost.rs

@@ -7,12 +7,13 @@ use db::{
 };
 use log::{debug, trace};
 use std::collections::HashMap;
+use std::marker::PhantomData;
 use std::sync::Arc;
-use types::{BeaconBlock, ChainSpec, Hash256, Slot};
+use types::{BeaconBlock, BeaconState, ChainSpec, EthSpec, Hash256, Slot};
 
 //TODO: Pruning and syncing
 
-pub struct SlowLMDGhost<T: ClientDB + Sized> {
+pub struct SlowLMDGhost<T: ClientDB + Sized, E> {
     /// The latest attestation targets as a map of validator index to block hash.
     //TODO: Could this be a fixed size vec
     latest_attestation_targets: HashMap<u64, Hash256>,
@@ -22,9 +23,10 @@ pub struct SlowLMDGhost<T: ClientDB + Sized> {
     block_store: Arc<BeaconBlockStore<T>>,
     /// State storage access.
     state_store: Arc<BeaconStateStore<T>>,
+    _phantom: PhantomData<E>,
 }
 
-impl<T> SlowLMDGhost<T>
+impl<T, E: EthSpec> SlowLMDGhost<T, E>
 where
     T: ClientDB + Sized,
 {
@@ -37,6 +39,7 @@ where
             children: HashMap::new(),
             block_store,
             state_store,
+            _phantom: PhantomData,
         }
     }
 
@@ -54,7 +57,7 @@ where
         // build a hashmap of block_hash to weighted votes
         let mut latest_votes: HashMap<Hash256, u64> = HashMap::new();
         // gets the current weighted votes
-        let current_state = self
+        let current_state: BeaconState<E> = self
             .state_store
             .get_deserialized(&state_root)?
             .ok_or_else(|| ForkChoiceError::MissingBeaconState(*state_root))?;
@@ -105,7 +108,7 @@ where
     }
 }
 
-impl<T: ClientDB + Sized> ForkChoice for SlowLMDGhost<T> {
+impl<T: ClientDB + Sized, E: EthSpec> ForkChoice for SlowLMDGhost<T, E> {
     /// Process when a block is added
     fn add_block(
         &mut self,

eth2/fork_choice/tests/tests.rs

@@ -25,7 +25,9 @@ use std::collections::HashMap;
 use std::sync::Arc;
 use std::{fs::File, io::prelude::*, path::PathBuf};
 use types::test_utils::TestingBeaconStateBuilder;
-use types::{BeaconBlock, BeaconBlockBody, ChainSpec, Eth1Data, Hash256, Keypair, Slot};
+use types::{
+    BeaconBlock, BeaconBlockBody, Eth1Data, EthSpec, FoundationEthSpec, Hash256, Keypair, Slot,
+};
 use yaml_rust::yaml;
 
 // Note: We Assume the block Id's are hex-encoded.
@@ -82,7 +84,7 @@ fn test_yaml_vectors(
     let test_cases = load_test_cases_from_yaml(yaml_file_path);
 
     // default vars
-    let spec = ChainSpec::foundation();
+    let spec = FoundationEthSpec::spec();
     let zero_hash = Hash256::zero();
     let eth1_data = Eth1Data {
         deposit_root: zero_hash.clone(),
@@ -227,23 +229,27 @@ fn setup_inital_state(
 
     // the fork choice instantiation
     let fork_choice: Box<ForkChoice> = match fork_choice_algo {
-        ForkChoiceAlgorithm::OptimizedLMDGhost => Box::new(OptimizedLMDGhost::new(
+        ForkChoiceAlgorithm::OptimizedLMDGhost => {
-            block_store.clone(),
+            let f: OptimizedLMDGhost<MemoryDB, FoundationEthSpec> =
-            state_store.clone(),
+                OptimizedLMDGhost::new(block_store.clone(), state_store.clone());
-        )),
+            Box::new(f)
-        ForkChoiceAlgorithm::BitwiseLMDGhost => Box::new(BitwiseLMDGhost::new(
+        }
-            block_store.clone(),
+        ForkChoiceAlgorithm::BitwiseLMDGhost => {
-            state_store.clone(),
+            let f: BitwiseLMDGhost<MemoryDB, FoundationEthSpec> =
-        )),
+                BitwiseLMDGhost::new(block_store.clone(), state_store.clone());
+            Box::new(f)
+        }
         ForkChoiceAlgorithm::SlowLMDGhost => {
-            Box::new(SlowLMDGhost::new(block_store.clone(), state_store.clone()))
+            let f: SlowLMDGhost<MemoryDB, FoundationEthSpec> =
+                SlowLMDGhost::new(block_store.clone(), state_store.clone());
+            Box::new(f)
         }
         ForkChoiceAlgorithm::LongestChain => Box::new(LongestChain::new(block_store.clone())),
     };
 
-    let spec = ChainSpec::foundation();
+    let spec = FoundationEthSpec::spec();
 
-    let mut state_builder =
+    let mut state_builder: TestingBeaconStateBuilder<FoundationEthSpec> =
         TestingBeaconStateBuilder::from_single_keypair(num_validators, &Keypair::random(), &spec);
     state_builder.build_caches(&spec).unwrap();
     let (state, _keypairs) = state_builder.build();

eth2/operation_pool/src/lib.rs

@@ -13,10 +13,11 @@ use state_processing::per_block_processing::{
     verify_transfer_time_independent_only,
 };
 use std::collections::{btree_map::Entry, hash_map, BTreeMap, HashMap, HashSet};
+use std::marker::PhantomData;
 use types::chain_spec::Domain;
 use types::{
     Attestation, AttestationData, AttesterSlashing, BeaconState, ChainSpec, Deposit, Epoch,
-    ProposerSlashing, Transfer, Validator, VoluntaryExit,
+    EthSpec, ProposerSlashing, Transfer, Validator, VoluntaryExit,
 };
 
 #[cfg(test)]
@@ -25,7 +26,7 @@ const VERIFY_DEPOSIT_PROOFS: bool = false;
 const VERIFY_DEPOSIT_PROOFS: bool = false; // TODO: enable this
 
 #[derive(Default)]
-pub struct OperationPool {
+pub struct OperationPool<T: EthSpec + Default> {
     /// Map from attestation ID (see below) to vectors of attestations.
     attestations: RwLock<HashMap<AttestationId, Vec<Attestation>>>,
     /// Map from deposit index to deposit data.
@@ -42,6 +43,7 @@ pub struct OperationPool {
     voluntary_exits: RwLock<HashMap<u64, VoluntaryExit>>,
     /// Set of transfers.
     transfers: RwLock<HashSet<Transfer>>,
+    _phantom: PhantomData<T>,
 }
 
 /// Serialized `AttestationData` augmented with a domain to encode the fork info.
@@ -52,14 +54,22 @@ struct AttestationId(Vec<u8>);
 const DOMAIN_BYTES_LEN: usize = 8;
 
 impl AttestationId {
-    fn from_data(attestation: &AttestationData, state: &BeaconState, spec: &ChainSpec) -> Self {
+    fn from_data<T: EthSpec>(
+        attestation: &AttestationData,
+        state: &BeaconState<T>,
+        spec: &ChainSpec,
+    ) -> Self {
         let mut bytes = ssz_encode(attestation);
         let epoch = attestation.slot.epoch(spec.slots_per_epoch);
         bytes.extend_from_slice(&AttestationId::compute_domain_bytes(epoch, state, spec));
         AttestationId(bytes)
     }
 
-    fn compute_domain_bytes(epoch: Epoch, state: &BeaconState, spec: &ChainSpec) -> Vec<u8> {
+    fn compute_domain_bytes<T: EthSpec>(
+        epoch: Epoch,
+        state: &BeaconState<T>,
+        spec: &ChainSpec,
+    ) -> Vec<u8> {
         int_to_bytes8(spec.get_domain(epoch, Domain::Attestation, &state.fork))
     }
 
@@ -75,7 +85,11 @@ impl AttestationId {
 /// receive for including it in a block.
 // TODO: this could be optimised with a map from validator index to whether that validator has
 // attested in each of the current and previous epochs. Currently quadractic in number of validators.
-fn attestation_score(attestation: &Attestation, state: &BeaconState, spec: &ChainSpec) -> usize {
+fn attestation_score<T: EthSpec>(
+    attestation: &Attestation,
+    state: &BeaconState<T>,
+    spec: &ChainSpec,
+) -> usize {
     // Bitfield of validators whose attestations are new/fresh.
     let mut new_validators = attestation.aggregation_bitfield.clone();
 
@@ -113,7 +127,7 @@ pub enum DepositInsertStatus {
     Replaced(Box<Deposit>),
 }
 
-impl OperationPool {
+impl<T: EthSpec> OperationPool<T> {
     /// Create a new operation pool.
     pub fn new() -> Self {
         Self::default()
@@ -123,7 +137,7 @@ impl OperationPool {
     pub fn insert_attestation(
         &self,
         attestation: Attestation,
-        state: &BeaconState,
+        state: &BeaconState<T>,
         spec: &ChainSpec,
     ) -> Result<(), AttestationValidationError> {
         // Check that attestation signatures are valid.
@@ -161,15 +175,11 @@ impl OperationPool {
 
     /// Total number of attestations in the pool, including attestations for the same data.
     pub fn num_attestations(&self) -> usize {
-        self.attestations
+        self.attestations.read().values().map(Vec::len).sum()
-            .read()
-            .values()
-            .map(|atts| atts.len())
-            .sum()
     }
 
     /// Get a list of attestations for inclusion in a block.
-    pub fn get_attestations(&self, state: &BeaconState, spec: &ChainSpec) -> Vec<Attestation> {
+    pub fn get_attestations(&self, state: &BeaconState<T>, spec: &ChainSpec) -> Vec<Attestation> {
         // Attestations for the current fork, which may be from the current or previous epoch.
         let prev_epoch = state.previous_epoch(spec);
         let current_epoch = state.current_epoch(spec);
@@ -204,7 +214,7 @@ impl OperationPool {
     // TODO: we could probably prune other attestations here:
     // - ones that are completely covered by attestations included in the state
     // - maybe ones invalidated by the confirmation of one fork over another
-    pub fn prune_attestations(&self, finalized_state: &BeaconState, spec: &ChainSpec) {
+    pub fn prune_attestations(&self, finalized_state: &BeaconState<T>, spec: &ChainSpec) {
         self.attestations.write().retain(|_, attestations| {
             // All the attestations in this bucket have the same data, so we only need to
             // check the first one.
@@ -220,7 +230,7 @@ impl OperationPool {
     pub fn insert_deposit(
         &self,
         deposit: Deposit,
-        state: &BeaconState,
+        state: &BeaconState<T>,
         spec: &ChainSpec,
     ) -> Result<DepositInsertStatus, DepositValidationError> {
         use DepositInsertStatus::*;
@@ -245,7 +255,7 @@ impl OperationPool {
     /// Get an ordered list of deposits for inclusion in a block.
     ///
     /// Take at most the maximum number of deposits, beginning from the current deposit index.
-    pub fn get_deposits(&self, state: &BeaconState, spec: &ChainSpec) -> Vec<Deposit> {
+    pub fn get_deposits(&self, state: &BeaconState<T>, spec: &ChainSpec) -> Vec<Deposit> {
         let start_idx = state.deposit_index;
         (start_idx..start_idx + spec.max_deposits)
             .map(|idx| self.deposits.read().get(&idx).cloned())
@@ -255,7 +265,7 @@ impl OperationPool {
     }
 
     /// Remove all deposits with index less than the deposit index of the latest finalised block.
-    pub fn prune_deposits(&self, state: &BeaconState) -> BTreeMap<u64, Deposit> {
+    pub fn prune_deposits(&self, state: &BeaconState<T>) -> BTreeMap<u64, Deposit> {
         let deposits_keep = self.deposits.write().split_off(&state.deposit_index);
         std::mem::replace(&mut self.deposits.write(), deposits_keep)
     }
@@ -269,7 +279,7 @@ impl OperationPool {
     pub fn insert_proposer_slashing(
         &self,
         slashing: ProposerSlashing,
-        state: &BeaconState,
+        state: &BeaconState<T>,
         spec: &ChainSpec,
     ) -> Result<(), ProposerSlashingValidationError> {
         // TODO: should maybe insert anyway if the proposer is unknown in the validator index,
@@ -286,7 +296,7 @@ impl OperationPool {
     /// Depends on the fork field of the state, but not on the state's epoch.
     fn attester_slashing_id(
         slashing: &AttesterSlashing,
-        state: &BeaconState,
+        state: &BeaconState<T>,
         spec: &ChainSpec,
     ) -> (AttestationId, AttestationId) {
         (
@@ -299,7 +309,7 @@ impl OperationPool {
     pub fn insert_attester_slashing(
         &self,
         slashing: AttesterSlashing,
-        state: &BeaconState,
+        state: &BeaconState<T>,
         spec: &ChainSpec,
     ) -> Result<(), AttesterSlashingValidationError> {
         verify_attester_slashing(state, &slashing, true, spec)?;
@@ -315,7 +325,7 @@ impl OperationPool {
     /// earlier in the block.
     pub fn get_slashings(
         &self,
-        state: &BeaconState,
+        state: &BeaconState<T>,
         spec: &ChainSpec,
     ) -> (Vec<ProposerSlashing>, Vec<AttesterSlashing>) {
         let proposer_slashings = filter_limit_operations(
@@ -370,7 +380,7 @@ impl OperationPool {
     }
 
     /// Prune proposer slashings for all slashed or withdrawn validators.
-    pub fn prune_proposer_slashings(&self, finalized_state: &BeaconState, spec: &ChainSpec) {
+    pub fn prune_proposer_slashings(&self, finalized_state: &BeaconState<T>, spec: &ChainSpec) {
         prune_validator_hash_map(
             &mut self.proposer_slashings.write(),
             |validator| {
@@ -383,7 +393,7 @@ impl OperationPool {
 
     /// Prune attester slashings for all slashed or withdrawn validators, or attestations on another
     /// fork.
-    pub fn prune_attester_slashings(&self, finalized_state: &BeaconState, spec: &ChainSpec) {
+    pub fn prune_attester_slashings(&self, finalized_state: &BeaconState<T>, spec: &ChainSpec) {
         self.attester_slashings.write().retain(|id, slashing| {
             let fork_ok = &Self::attester_slashing_id(slashing, finalized_state, spec) == id;
             let curr_epoch = finalized_state.current_epoch(spec);
@@ -402,7 +412,7 @@ impl OperationPool {
     pub fn insert_voluntary_exit(
         &self,
         exit: VoluntaryExit,
-        state: &BeaconState,
+        state: &BeaconState<T>,
         spec: &ChainSpec,
     ) -> Result<(), ExitValidationError> {
         verify_exit_time_independent_only(state, &exit, spec)?;
@@ -413,7 +423,11 @@ impl OperationPool {
     }
 
     /// Get a list of voluntary exits for inclusion in a block.
-    pub fn get_voluntary_exits(&self, state: &BeaconState, spec: &ChainSpec) -> Vec<VoluntaryExit> {
+    pub fn get_voluntary_exits(
+        &self,
+        state: &BeaconState<T>,
+        spec: &ChainSpec,
+    ) -> Vec<VoluntaryExit> {
         filter_limit_operations(
             self.voluntary_exits.read().values(),
             |exit| verify_exit(state, exit, spec).is_ok(),
@@ -422,7 +436,7 @@ impl OperationPool {
     }
 
     /// Prune if validator has already exited at the last finalized state.
-    pub fn prune_voluntary_exits(&self, finalized_state: &BeaconState, spec: &ChainSpec) {
+    pub fn prune_voluntary_exits(&self, finalized_state: &BeaconState<T>, spec: &ChainSpec) {
         prune_validator_hash_map(
             &mut self.voluntary_exits.write(),
             |validator| validator.is_exited_at(finalized_state.current_epoch(spec)),
@@ -434,7 +448,7 @@ impl OperationPool {
     pub fn insert_transfer(
         &self,
         transfer: Transfer,
-        state: &BeaconState,
+        state: &BeaconState<T>,
         spec: &ChainSpec,
     ) -> Result<(), TransferValidationError> {
         // The signature of the transfer isn't hashed, but because we check
@@ -448,7 +462,7 @@ impl OperationPool {
     /// Get a list of transfers for inclusion in a block.
     // TODO: improve the economic optimality of this function by accounting for
     // dependencies between transfers in the same block e.g. A pays B, B pays C
-    pub fn get_transfers(&self, state: &BeaconState, spec: &ChainSpec) -> Vec<Transfer> {
+    pub fn get_transfers(&self, state: &BeaconState<T>, spec: &ChainSpec) -> Vec<Transfer> {
         self.transfers
             .read()
             .iter()
@@ -460,14 +474,14 @@ impl OperationPool {
     }
 
     /// Prune the set of transfers by removing all those whose slot has already passed.
-    pub fn prune_transfers(&self, finalized_state: &BeaconState) {
+    pub fn prune_transfers(&self, finalized_state: &BeaconState<T>) {
         self.transfers
             .write()
             .retain(|transfer| transfer.slot > finalized_state.slot)
     }
 
     /// Prune all types of transactions given the latest finalized state.
-    pub fn prune_all(&self, finalized_state: &BeaconState, spec: &ChainSpec) {
+    pub fn prune_all(&self, finalized_state: &BeaconState<T>, spec: &ChainSpec) {
         self.prune_attestations(finalized_state, spec);
         self.prune_deposits(finalized_state);
         self.prune_proposer_slashings(finalized_state, spec);
@@ -487,7 +501,10 @@ impl OperationPool {
 ///
 /// - Their `AttestationData` is equal.
 /// - `attestation` does not contain any signatures that `PendingAttestation` does not have.
-fn superior_attestation_exists_in_state(state: &BeaconState, attestation: &Attestation) -> bool {
+fn superior_attestation_exists_in_state<T: EthSpec>(
+    state: &BeaconState<T>,
+    attestation: &Attestation,
+) -> bool {
     state
         .current_epoch_attestations
         .iter()
@@ -522,10 +539,10 @@ where
 /// The keys in the map should be validator indices, which will be looked up
 /// in the state's validator registry and then passed to `prune_if`.
 /// Entries for unknown validators will be kept.
-fn prune_validator_hash_map<T, F>(
+fn prune_validator_hash_map<T, F, E: EthSpec>(
     map: &mut HashMap<u64, T>,
     prune_if: F,
-    finalized_state: &BeaconState,
+    finalized_state: &BeaconState<E>,
 ) where
     F: Fn(&Validator) -> bool,
 {
@@ -649,7 +666,11 @@ mod tests {
     }
 
     // Create a random deposit (with a valid proof of posession)
-    fn make_deposit(rng: &mut XorShiftRng, state: &BeaconState, spec: &ChainSpec) -> Deposit {
+    fn make_deposit<T: EthSpec>(
+        rng: &mut XorShiftRng,
+        state: &BeaconState<T>,
+        spec: &ChainSpec,
+    ) -> Deposit {
         let keypair = Keypair::random();
         let mut deposit = Deposit::random_for_test(rng);
         let mut deposit_input = DepositInput {
@@ -668,9 +689,9 @@ mod tests {
     }
 
     // Create `count` dummy deposits with sequential deposit IDs beginning from `start`.
-    fn dummy_deposits(
+    fn dummy_deposits<T: EthSpec>(
         rng: &mut XorShiftRng,
-        state: &BeaconState,
+        state: &BeaconState<T>,
         spec: &ChainSpec,
         start: u64,
         count: u64,
@@ -685,301 +706,315 @@ mod tests {
             .collect()
     }
 
-    fn test_state(rng: &mut XorShiftRng) -> (ChainSpec, BeaconState) {
+    fn test_state(rng: &mut XorShiftRng) -> (ChainSpec, BeaconState<FoundationEthSpec>) {
-        let spec = ChainSpec::foundation();
+        let spec = FoundationEthSpec::spec();
+
         let mut state = BeaconState::random_for_test(rng);
+
         state.fork = Fork::genesis(&spec);
 
         (spec, state)
     }
 
-    /// Create a signed attestation for use in tests.
-    /// Signed by all validators in `committee[signing_range]` and `committee[extra_signer]`.
     #[cfg(not(debug_assertions))]
-    fn signed_attestation<R: std::slice::SliceIndex<[usize], Output = [usize]>>(
+    mod release_tests {
-        committee: &CrosslinkCommittee,
+        use super::*;
-        keypairs: &[Keypair],
+
-        signing_range: R,
+        /// Create a signed attestation for use in tests.
-        slot: Slot,
+        /// Signed by all validators in `committee[signing_range]` and `committee[extra_signer]`.
-        state: &BeaconState,
+        fn signed_attestation<R: std::slice::SliceIndex<[usize], Output = [usize]>, E: EthSpec>(
-        spec: &ChainSpec,
+            committee: &CrosslinkCommittee,
-        extra_signer: Option<usize>,
+            keypairs: &[Keypair],
-    ) -> Attestation {
+            signing_range: R,
-        let mut builder = TestingAttestationBuilder::new(
+            slot: Slot,
-            state,
+            state: &BeaconState<E>,
-            &committee.committee,
+            spec: &ChainSpec,
-            slot,
+            extra_signer: Option<usize>,
-            committee.shard,
+        ) -> Attestation {
-            spec,
+            let mut builder = TestingAttestationBuilder::new(
-        );
+                state,
-        let signers = &committee.committee[signing_range];
+                &committee.committee,
-        let committee_keys = signers.iter().map(|&i| &keypairs[i].sk).collect::<Vec<_>>();
+                slot,
-        builder.sign(signers, &committee_keys, &state.fork, spec);
+                committee.shard,
-        extra_signer.map(|c_idx| {
-            let validator_index = committee.committee[c_idx];
-            builder.sign(
-                &[validator_index],
-                &[&keypairs[validator_index].sk],
-                &state.fork,
                 spec,
-            )
-        });
-        builder.build()
-    }
-
-    /// Test state for attestation-related tests.
-    #[cfg(not(debug_assertions))]
-    fn attestation_test_state(
-        spec: &ChainSpec,
-        num_committees: usize,
-    ) -> (BeaconState, Vec<Keypair>) {
-        let num_validators =
-            num_committees * (spec.slots_per_epoch * spec.target_committee_size) as usize;
-        let mut state_builder =
-            TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(num_validators, spec);
-        let slot_offset = 1000 * spec.slots_per_epoch + spec.slots_per_epoch / 2;
-        let slot = spec.genesis_slot + slot_offset;
-        state_builder.teleport_to_slot(slot, spec);
-        state_builder.build_caches(spec).unwrap();
-        state_builder.build()
-    }
-
-    /// Set the latest crosslink in the state to match the attestation.
-    #[cfg(not(debug_assertions))]
-    fn fake_latest_crosslink(att: &Attestation, state: &mut BeaconState, spec: &ChainSpec) {
-        state.latest_crosslinks[att.data.shard as usize] = Crosslink {
-            crosslink_data_root: att.data.crosslink_data_root,
-            epoch: att.data.slot.epoch(spec.slots_per_epoch),
-        };
-    }
-
-    #[test]
-    #[cfg(not(debug_assertions))]
-    fn test_attestation_score() {
-        let spec = &ChainSpec::foundation();
-        let (ref mut state, ref keypairs) = attestation_test_state(spec, 1);
-        let slot = state.slot - 1;
-        let committees = state
-            .get_crosslink_committees_at_slot(slot, spec)
-            .unwrap()
-            .clone();
-
-        for committee in committees {
-            let att1 = signed_attestation(&committee, keypairs, ..2, slot, state, spec, None);
-            let att2 = signed_attestation(&committee, keypairs, .., slot, state, spec, None);
-
-            assert_eq!(
-                att1.aggregation_bitfield.num_set_bits(),
-                attestation_score(&att1, state, spec)
-            );
-
-            state
-                .current_epoch_attestations
-                .push(PendingAttestation::from_attestation(&att1, state.slot));
-
-            assert_eq!(
-                committee.committee.len() - 2,
-                attestation_score(&att2, state, spec)
             );
+            let signers = &committee.committee[signing_range];
+            let committee_keys = signers.iter().map(|&i| &keypairs[i].sk).collect::<Vec<_>>();
+            builder.sign(signers, &committee_keys, &state.fork, spec);
+            extra_signer.map(|c_idx| {
+                let validator_index = committee.committee[c_idx];
+                builder.sign(
+                    &[validator_index],
+                    &[&keypairs[validator_index].sk],
+                    &state.fork,
+                    spec,
+                )
+            });
+            builder.build()
         }
-    }
 
-    /// End-to-end test of basic attestation handling.
+        /// Test state for attestation-related tests.
-    #[test]
+        fn attestation_test_state<E: EthSpec>(
-    #[cfg(not(debug_assertions))]
+            num_committees: usize,
-    fn attestation_aggregation_insert_get_prune() {
+        ) -> (BeaconState<E>, Vec<Keypair>, ChainSpec) {
-        let spec = &ChainSpec::foundation();
+            let spec = E::spec();
-        let (ref mut state, ref keypairs) = attestation_test_state(spec, 1);
-        let op_pool = OperationPool::new();
 
-        let slot = state.slot - 1;
+            let num_validators =
-        let committees = state
+                num_committees * (spec.slots_per_epoch * spec.target_committee_size) as usize;
-            .get_crosslink_committees_at_slot(slot, spec)
+            let mut state_builder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(
-            .unwrap()
+                num_validators,
-            .clone();
+                &spec,
+            );
+            let slot_offset = 1000 * spec.slots_per_epoch + spec.slots_per_epoch / 2;
+            let slot = spec.genesis_slot + slot_offset;
+            state_builder.teleport_to_slot(slot, &spec);
+            state_builder.build_caches(&spec).unwrap();
+            let (state, keypairs) = state_builder.build();
 
-        assert_eq!(
+            (state, keypairs, FoundationEthSpec::spec())
-            committees.len(),
+        }
-            1,
+
-            "we expect just one committee with this many validators"
+        /// Set the latest crosslink in the state to match the attestation.
-        );
+        fn fake_latest_crosslink<E: EthSpec>(
+            att: &Attestation,
+            state: &mut BeaconState<E>,
+            spec: &ChainSpec,
+        ) {
+            state.latest_crosslinks[att.data.shard as usize] = Crosslink {
+                crosslink_data_root: att.data.crosslink_data_root,
+                epoch: att.data.slot.epoch(spec.slots_per_epoch),
+            };
+        }
+
+        #[test]
+        fn test_attestation_score() {
+            let (ref mut state, ref keypairs, ref spec) =
+                attestation_test_state::<FoundationEthSpec>(1);
+
+            let slot = state.slot - 1;
+            let committees = state
+                .get_crosslink_committees_at_slot(slot, spec)
+                .unwrap()
+                .clone();
+
+            for committee in committees {
+                let att1 = signed_attestation(&committee, keypairs, ..2, slot, state, spec, None);
+                let att2 = signed_attestation(&committee, keypairs, .., slot, state, spec, None);
+
+                assert_eq!(
+                    att1.aggregation_bitfield.num_set_bits(),
+                    attestation_score(&att1, state, spec)
+                );
+
+                state
+                    .current_epoch_attestations
+                    .push(PendingAttestation::from_attestation(&att1, state.slot));
+
+                assert_eq!(
+                    committee.committee.len() - 2,
+                    attestation_score(&att2, state, spec)
+                );
+            }
+        }
+
+        /// End-to-end test of basic attestation handling.
+        #[test]
+        fn attestation_aggregation_insert_get_prune() {
+            let (ref mut state, ref keypairs, ref spec) =
+                attestation_test_state::<FoundationEthSpec>(1);
+
+            let op_pool = OperationPool::new();
+
+            let slot = state.slot - 1;
+            let committees = state
+                .get_crosslink_committees_at_slot(slot, spec)
+                .unwrap()
+                .clone();
+
+            assert_eq!(
+                committees.len(),
+                1,
+                "we expect just one committee with this many validators"
+            );
+
+            for committee in &committees {
+                let step_size = 2;
+                for i in (0..committee.committee.len()).step_by(step_size) {
+                    let att = signed_attestation(
+                        committee,
+                        keypairs,
+                        i..i + step_size,
+                        slot,
+                        state,
+                        spec,
+                        None,
+                    );
+                    fake_latest_crosslink(&att, state, spec);
+                    op_pool.insert_attestation(att, state, spec).unwrap();
+                }
+            }
+
+            assert_eq!(op_pool.attestations.read().len(), committees.len());
+            assert_eq!(op_pool.num_attestations(), committees.len());
+
+            // Before the min attestation inclusion delay, get_attestations shouldn't return anything.
+            assert_eq!(op_pool.get_attestations(state, spec).len(), 0);
+
+            // Then once the delay has elapsed, we should get a single aggregated attestation.
+            state.slot += spec.min_attestation_inclusion_delay;
+
+            let block_attestations = op_pool.get_attestations(state, spec);
+            assert_eq!(block_attestations.len(), committees.len());
+
+            let agg_att = &block_attestations[0];
+            assert_eq!(
+                agg_att.aggregation_bitfield.num_set_bits(),
+                spec.target_committee_size as usize
+            );
+
+            // Prune attestations shouldn't do anything at this point.
+            op_pool.prune_attestations(state, spec);
+            assert_eq!(op_pool.num_attestations(), committees.len());
+
+            // But once we advance to an epoch after the attestation, it should prune it out of
+            // existence.
+            state.slot = slot + spec.slots_per_epoch;
+            op_pool.prune_attestations(state, spec);
+            assert_eq!(op_pool.num_attestations(), 0);
+        }
+
+        /// Adding an attestation already in the pool should not increase the size of the pool.
+        #[test]
+        fn attestation_duplicate() {
+            let (ref mut state, ref keypairs, ref spec) =
+                attestation_test_state::<FoundationEthSpec>(1);
+
+            let op_pool = OperationPool::new();
+
+            let slot = state.slot - 1;
+            let committees = state
+                .get_crosslink_committees_at_slot(slot, spec)
+                .unwrap()
+                .clone();
+
+            for committee in &committees {
+                let att = signed_attestation(committee, keypairs, .., slot, state, spec, None);
+                fake_latest_crosslink(&att, state, spec);
+                op_pool
+                    .insert_attestation(att.clone(), state, spec)
+                    .unwrap();
+                op_pool.insert_attestation(att, state, spec).unwrap();
+            }
+
+            assert_eq!(op_pool.num_attestations(), committees.len());
+        }
+
+        /// Adding lots of attestations that only intersect pairwise should lead to two aggregate
+        /// attestations.
+        #[test]
+        fn attestation_pairwise_overlapping() {
+            let (ref mut state, ref keypairs, ref spec) =
+                attestation_test_state::<FoundationEthSpec>(1);
+
+            let op_pool = OperationPool::new();
+
+            let slot = state.slot - 1;
+            let committees = state
+                .get_crosslink_committees_at_slot(slot, spec)
+                .unwrap()
+                .clone();
 
-        for committee in &committees {
             let step_size = 2;
-            for i in (0..committee.committee.len()).step_by(step_size) {
+            for committee in &committees {
-                let att = signed_attestation(
+                // Create attestations that overlap on `step_size` validators, like:
-                    committee,
+                // {0,1,2,3}, {2,3,4,5}, {4,5,6,7}, ...
-                    keypairs,
+                for i in (0..committee.committee.len() - step_size).step_by(step_size) {
-                    i..i + step_size,
+                    let att = signed_attestation(
-                    slot,
+                        committee,
-                    state,
+                        keypairs,
-                    spec,
+                        i..i + 2 * step_size,
-                    None,
+                        slot,
-                );
+                        state,
-                fake_latest_crosslink(&att, state, spec);
+                        spec,
-                op_pool.insert_attestation(att, state, spec).unwrap();
+                        None,
+                    );
+                    fake_latest_crosslink(&att, state, spec);
+                    op_pool.insert_attestation(att, state, spec).unwrap();
+                }
             }
+
+            // The attestations should get aggregated into two attestations that comprise all
+            // validators.
+            assert_eq!(op_pool.attestations.read().len(), committees.len());
+            assert_eq!(op_pool.num_attestations(), 2 * committees.len());
         }
 
-        assert_eq!(op_pool.attestations.read().len(), committees.len());
+        /// Create a bunch of attestations signed by a small number of validators, and another
-        assert_eq!(op_pool.num_attestations(), committees.len());
+        /// bunch signed by a larger number, such that there are at least `max_attestations`
+        /// signed by the larger number. Then, check that `get_attestations` only returns the
+        /// high-quality attestations. To ensure that no aggregation occurs, ALL attestations
+        /// are also signed by the 0th member of the committee.
+        #[test]
+        fn attestation_get_max() {
+            let small_step_size = 2;
+            let big_step_size = 4;
 
-        // Before the min attestation inclusion delay, get_attestations shouldn't return anything.
+            let (ref mut state, ref keypairs, ref spec) =
-        assert_eq!(op_pool.get_attestations(state, spec).len(), 0);
+                attestation_test_state::<FoundationEthSpec>(big_step_size);
 
-        // Then once the delay has elapsed, we should get a single aggregated attestation.
+            let op_pool = OperationPool::new();
-        state.slot += spec.min_attestation_inclusion_delay;
 
-        let block_attestations = op_pool.get_attestations(state, spec);
+            let slot = state.slot - 1;
-        assert_eq!(block_attestations.len(), committees.len());
+            let committees = state
+                .get_crosslink_committees_at_slot(slot, spec)
+                .unwrap()
+                .clone();
 
-        let agg_att = &block_attestations[0];
+            let max_attestations = spec.max_attestations as usize;
-        assert_eq!(
+            let target_committee_size = spec.target_committee_size as usize;
-            agg_att.aggregation_bitfield.num_set_bits(),
-            spec.target_committee_size as usize
-        );
 
-        // Prune attestations shouldn't do anything at this point.
+            let mut insert_attestations = |committee, step_size| {
-        op_pool.prune_attestations(state, spec);
+                for i in (0..target_committee_size).step_by(step_size) {
-        assert_eq!(op_pool.num_attestations(), committees.len());
+                    let att = signed_attestation(
+                        committee,
+                        keypairs,
+                        i..i + step_size,
+                        slot,
+                        state,
+                        spec,
+                        if i == 0 { None } else { Some(0) },
+                    );
+                    fake_latest_crosslink(&att, state, spec);
+                    op_pool.insert_attestation(att, state, spec).unwrap();
+                }
+            };
 
-        // But once we advance to an epoch after the attestation, it should prune it out of
+            for committee in &committees {
-        // existence.
+                assert_eq!(committee.committee.len(), target_committee_size);
-        state.slot = slot + spec.slots_per_epoch;
+                // Attestations signed by only 2-3 validators
-        op_pool.prune_attestations(state, spec);
+                insert_attestations(committee, small_step_size);
-        assert_eq!(op_pool.num_attestations(), 0);
+                // Attestations signed by 4+ validators
-    }
+                insert_attestations(committee, big_step_size);
-
-    /// Adding an attestation already in the pool should not increase the size of the pool.
-    #[test]
-    #[cfg(not(debug_assertions))]
-    fn attestation_duplicate() {
-        let spec = &ChainSpec::foundation();
-        let (ref mut state, ref keypairs) = attestation_test_state(spec, 1);
-        let op_pool = OperationPool::new();
-
-        let slot = state.slot - 1;
-        let committees = state
-            .get_crosslink_committees_at_slot(slot, spec)
-            .unwrap()
-            .clone();
-
-        for committee in &committees {
-            let att = signed_attestation(committee, keypairs, .., slot, state, spec, None);
-            fake_latest_crosslink(&att, state, spec);
-            op_pool
-                .insert_attestation(att.clone(), state, spec)
-                .unwrap();
-            op_pool.insert_attestation(att, state, spec).unwrap();
-        }
-
-        assert_eq!(op_pool.num_attestations(), committees.len());
-    }
-
-    /// Adding lots of attestations that only intersect pairwise should lead to two aggregate
-    /// attestations.
-    #[test]
-    #[cfg(not(debug_assertions))]
-    fn attestation_pairwise_overlapping() {
-        let spec = &ChainSpec::foundation();
-        let (ref mut state, ref keypairs) = attestation_test_state(spec, 1);
-        let op_pool = OperationPool::new();
-
-        let slot = state.slot - 1;
-        let committees = state
-            .get_crosslink_committees_at_slot(slot, spec)
-            .unwrap()
-            .clone();
-
-        let step_size = 2;
-        for committee in &committees {
-            // Create attestations that overlap on `step_size` validators, like:
-            // {0,1,2,3}, {2,3,4,5}, {4,5,6,7}, ...
-            for i in (0..committee.committee.len() - step_size).step_by(step_size) {
-                let att = signed_attestation(
-                    committee,
-                    keypairs,
-                    i..i + 2 * step_size,
-                    slot,
-                    state,
-                    spec,
-                    None,
-                );
-                fake_latest_crosslink(&att, state, spec);
-                op_pool.insert_attestation(att, state, spec).unwrap();
             }
-        }
 
-        // The attestations should get aggregated into two attestations that comprise all
+            let num_small = target_committee_size / small_step_size;
-        // validators.
+            let num_big = target_committee_size / big_step_size;
-        assert_eq!(op_pool.attestations.read().len(), committees.len());
-        assert_eq!(op_pool.num_attestations(), 2 * committees.len());
-    }
 
-    /// Create a bunch of attestations signed by a small number of validators, and another
+            assert_eq!(op_pool.attestations.read().len(), committees.len());
-    /// bunch signed by a larger number, such that there are at least `max_attestations`
+            assert_eq!(
-    /// signed by the larger number. Then, check that `get_attestations` only returns the
+                op_pool.num_attestations(),
-    /// high-quality attestations. To ensure that no aggregation occurs, ALL attestations
+                (num_small + num_big) * committees.len()
-    /// are also signed by the 0th member of the committee.
+            );
-    #[test]
+            assert!(op_pool.num_attestations() > max_attestations);
-    #[cfg(not(debug_assertions))]
-    fn attestation_get_max() {
-        let spec = &ChainSpec::foundation();
-        let small_step_size = 2;
-        let big_step_size = 4;
-        let (ref mut state, ref keypairs) = attestation_test_state(spec, big_step_size);
-        let op_pool = OperationPool::new();
 
-        let slot = state.slot - 1;
+            state.slot += spec.min_attestation_inclusion_delay;
-        let committees = state
+            let best_attestations = op_pool.get_attestations(state, spec);
-            .get_crosslink_committees_at_slot(slot, spec)
+            assert_eq!(best_attestations.len(), max_attestations);
-            .unwrap()
-            .clone();
 
-        let max_attestations = spec.max_attestations as usize;
+            // All the best attestations should be signed by at least `big_step_size` (4) validators.
-        let target_committee_size = spec.target_committee_size as usize;
+            for att in &best_attestations {
-
+                assert!(att.aggregation_bitfield.num_set_bits() >= big_step_size);
-        let mut insert_attestations = |committee, step_size| {
-            for i in (0..target_committee_size).step_by(step_size) {
-                let att = signed_attestation(
-                    committee,
-                    keypairs,
-                    i..i + step_size,
-                    slot,
-                    state,
-                    spec,
-                    if i == 0 { None } else { Some(0) },
-                );
-                fake_latest_crosslink(&att, state, spec);
-                op_pool.insert_attestation(att, state, spec).unwrap();
             }
-        };
-
-        for committee in &committees {
-            assert_eq!(committee.committee.len(), target_committee_size);
-            // Attestations signed by only 2-3 validators
-            insert_attestations(committee, small_step_size);
-            // Attestations signed by 4+ validators
-            insert_attestations(committee, big_step_size);
-        }
-
-        let num_small = target_committee_size / small_step_size;
-        let num_big = target_committee_size / big_step_size;
-
-        assert_eq!(op_pool.attestations.read().len(), committees.len());
-        assert_eq!(
-            op_pool.num_attestations(),
-            (num_small + num_big) * committees.len()
-        );
-        assert!(op_pool.num_attestations() > max_attestations);
-
-        state.slot += spec.min_attestation_inclusion_delay;
-        let best_attestations = op_pool.get_attestations(state, spec);
-        assert_eq!(best_attestations.len(), max_attestations);
-
-        // All the best attestations should be signed by at least `big_step_size` (4) validators.
-        for att in &best_attestations {
-            assert!(att.aggregation_bitfield.num_set_bits() >= big_step_size);
         }
     }
 

eth2/state_processing/Cargo.toml

@@ -14,7 +14,6 @@ env_logger = "0.6.0"
 serde = "1.0"
 serde_derive = "1.0"
 serde_yaml = "0.8"
-yaml-utils = { path = "yaml_utils" }
 
 [dependencies]
 bls = { path = "../utils/bls" }
@@ -26,5 +25,7 @@ log = "0.4"
 merkle_proof = { path = "../utils/merkle_proof" }
 ssz = { path = "../utils/ssz" }
 ssz_derive = { path = "../utils/ssz_derive" }
+tree_hash = { path = "../utils/tree_hash" }
+tree_hash_derive = { path = "../utils/tree_hash_derive" }
 types = { path = "../types" }
 rayon = "1.0"

eth2/state_processing/benches/bench_block_processing.rs

@@ -1,6 +1,5 @@
 use criterion::Criterion;
 use criterion::{black_box, Benchmark};
-use ssz::TreeHash;
 use state_processing::{
     per_block_processing,
     per_block_processing::{
@@ -9,6 +8,7 @@ use state_processing::{
         verify_block_signature,
     },
 };
+use tree_hash::TreeHash;
 use types::*;
 
 /// Run the detailed benchmarking suite on the given `BeaconState`.
@@ -263,7 +263,7 @@ pub fn bench_block_processing(
     c.bench(
         &format!("{}/block_processing", desc),
         Benchmark::new("tree_hash_block", move |b| {
-            b.iter(|| black_box(block.hash_tree_root()))
+            b.iter(|| black_box(block.tree_hash_root()))
         })
         .sample_size(10),
     );

eth2/state_processing/benches/bench_epoch_processing.rs

@@ -1,6 +1,5 @@
 use criterion::Criterion;
 use criterion::{black_box, Benchmark};
-use ssz::TreeHash;
 use state_processing::{
     per_epoch_processing,
     per_epoch_processing::{
@@ -9,6 +8,7 @@ use state_processing::{
         update_active_tree_index_roots, update_latest_slashed_balances,
     },
 };
+use tree_hash::TreeHash;
 use types::test_utils::TestingBeaconStateBuilder;
 use types::*;
 
@@ -256,7 +256,7 @@ fn bench_epoch_processing(c: &mut Criterion, state: &BeaconState, spec: &ChainSp
     c.bench(
         &format!("{}/epoch_processing", desc),
         Benchmark::new("tree_hash_state", move |b| {
-            b.iter(|| black_box(state_clone.hash_tree_root()))
+            b.iter(|| black_box(state_clone.tree_hash_root()))
         })
         .sample_size(SMALL_BENCHING_SAMPLE_SIZE),
     );

eth2/state_processing/src/common/exit_validator.rs

@@ -2,9 +2,9 @@ use types::{BeaconStateError as Error, *};
 
 /// Exit the validator of the given `index`.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn exit_validator(
+pub fn exit_validator<T: EthSpec>(
-    state: &mut BeaconState,
+    state: &mut BeaconState<T>,
     validator_index: usize,
     spec: &ChainSpec,
 ) -> Result<(), Error> {

eth2/state_processing/src/common/slash_validator.rs

@@ -3,9 +3,9 @@ use types::{BeaconStateError as Error, *};
 
 /// Slash the validator with index ``index``.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn slash_validator(
+pub fn slash_validator<T: EthSpec>(
-    state: &mut BeaconState,
+    state: &mut BeaconState<T>,
     validator_index: usize,
     spec: &ChainSpec,
 ) -> Result<(), Error> {
@@ -36,8 +36,7 @@ pub fn slash_validator(
 
     state.set_slashed_balance(
         current_epoch,
-        state.get_slashed_balance(current_epoch, spec)? + effective_balance,
+        state.get_slashed_balance(current_epoch)? + effective_balance,
-        spec,
     )?;
 
     let whistleblower_index =
@@ -56,7 +55,7 @@ pub fn slash_validator(
     state.validator_registry[validator_index].slashed = true;
 
     state.validator_registry[validator_index].withdrawable_epoch =
-        current_epoch + Epoch::from(spec.latest_slashed_exit_length);
+        current_epoch + Epoch::from(T::LatestSlashedExitLength::to_usize());
 
     Ok(())
 }

eth2/state_processing/src/common/verify_bitfield.rs

@@ -4,7 +4,7 @@ use types::*;
 ///
 /// Is title `verify_bitfield` in spec.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 pub fn verify_bitfield_length(bitfield: &Bitfield, committee_size: usize) -> bool {
     if bitfield.num_bytes() != ((committee_size + 7) / 8) {
         return false;
@@ -18,3 +18,62 @@ pub fn verify_bitfield_length(bitfield: &Bitfield, committee_size: usize) -> boo
 
     true
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn bitfield_length() {
+        assert_eq!(
+            verify_bitfield_length(&Bitfield::from_bytes(&[0b0000_0001]), 4),
+            true
+        );
+
+        assert_eq!(
+            verify_bitfield_length(&Bitfield::from_bytes(&[0b0001_0001]), 4),
+            false
+        );
+
+        assert_eq!(
+            verify_bitfield_length(&Bitfield::from_bytes(&[0b0000_0000]), 4),
+            true
+        );
+
+        assert_eq!(
+            verify_bitfield_length(&Bitfield::from_bytes(&[0b1000_0000]), 8),
+            true
+        );
+
+        assert_eq!(
+            verify_bitfield_length(&Bitfield::from_bytes(&[0b1000_0000, 0b0000_0000]), 16),
+            true
+        );
+
+        assert_eq!(
+            verify_bitfield_length(&Bitfield::from_bytes(&[0b1000_0000, 0b0000_0000]), 15),
+            false
+        );
+
+        assert_eq!(
+            verify_bitfield_length(&Bitfield::from_bytes(&[0b0000_0000, 0b0000_0000]), 8),
+            false
+        );
+
+        assert_eq!(
+            verify_bitfield_length(
+                &Bitfield::from_bytes(&[0b0000_0000, 0b0000_0000, 0b0000_0000]),
+                8
+            ),
+            false
+        );
+
+        assert_eq!(
+            verify_bitfield_length(
+                &Bitfield::from_bytes(&[0b0000_0000, 0b0000_0000, 0b0000_0000]),
+                24
+            ),
+            true
+        );
+    }
+}

eth2/state_processing/src/get_genesis_state.rs

@@ -1,5 +1,5 @@
 use super::per_block_processing::{errors::BlockProcessingError, process_deposits};
-use ssz::TreeHash;
+use tree_hash::TreeHash;
 use types::*;
 
 pub enum GenesisError {
@@ -9,13 +9,13 @@ pub enum GenesisError {
 
 /// Returns the genesis `BeaconState`
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn get_genesis_state(
+pub fn get_genesis_state<T: EthSpec>(
     genesis_validator_deposits: &[Deposit],
     genesis_time: u64,
     genesis_eth1_data: Eth1Data,
     spec: &ChainSpec,
-) -> Result<(), BlockProcessingError> {
+) -> Result<BeaconState<T>, BlockProcessingError> {
     // Get the genesis `BeaconState`
     let mut state = BeaconState::genesis(genesis_time, genesis_eth1_data, spec);
 
@@ -36,13 +36,13 @@ pub fn get_genesis_state(
     let active_validator_indices = state
         .get_cached_active_validator_indices(RelativeEpoch::Current, spec)?
         .to_vec();
-    let genesis_active_index_root = Hash256::from_slice(&active_validator_indices.hash_tree_root());
+    let genesis_active_index_root = Hash256::from_slice(&active_validator_indices.tree_hash_root());
-    state.fill_active_index_roots_with(genesis_active_index_root, spec);
+    state.fill_active_index_roots_with(genesis_active_index_root);
 
     // Generate the current shuffling seed.
     state.current_shuffling_seed = state.generate_seed(spec.genesis_epoch, spec)?;
 
-    Ok(())
+    Ok(state)
 }
 
 impl From<BlockProcessingError> for GenesisError {

eth2/state_processing/src/per_block_processing.rs

@@ -1,7 +1,7 @@
 use crate::common::slash_validator;
 use errors::{BlockInvalid as Invalid, BlockProcessingError as Error, IntoWithIndex};
 use rayon::prelude::*;
-use ssz::{SignedRoot, TreeHash};
+use tree_hash::{SignedRoot, TreeHash};
 use types::*;
 
 pub use self::verify_attester_slashing::{
@@ -41,9 +41,9 @@ const VERIFY_DEPOSIT_MERKLE_PROOFS: bool = false;
 /// Returns `Ok(())` if the block is valid and the state was successfully updated. Otherwise
 /// returns an error describing why the block was invalid or how the function failed to execute.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn per_block_processing(
+pub fn per_block_processing<T: EthSpec>(
-    state: &mut BeaconState,
+    state: &mut BeaconState<T>,
     block: &BeaconBlock,
     spec: &ChainSpec,
 ) -> Result<(), Error> {
@@ -56,9 +56,9 @@ pub fn per_block_processing(
 /// Returns `Ok(())` if the block is valid and the state was successfully updated. Otherwise
 /// returns an error describing why the block was invalid or how the function failed to execute.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn per_block_processing_without_verifying_block_signature(
+pub fn per_block_processing_without_verifying_block_signature<T: EthSpec>(
-    state: &mut BeaconState,
+    state: &mut BeaconState<T>,
     block: &BeaconBlock,
     spec: &ChainSpec,
 ) -> Result<(), Error> {
@@ -71,9 +71,9 @@ pub fn per_block_processing_without_verifying_block_signature(
 /// Returns `Ok(())` if the block is valid and the state was successfully updated. Otherwise
 /// returns an error describing why the block was invalid or how the function failed to execute.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-fn per_block_processing_signature_optional(
+fn per_block_processing_signature_optional<T: EthSpec>(
-    mut state: &mut BeaconState,
+    mut state: &mut BeaconState<T>,
     block: &BeaconBlock,
     should_verify_block_signature: bool,
     spec: &ChainSpec,
@@ -101,20 +101,22 @@ fn per_block_processing_signature_optional(
 
 /// Processes the block header.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn process_block_header(
+pub fn process_block_header<T: EthSpec>(
-    state: &mut BeaconState,
+    state: &mut BeaconState<T>,
     block: &BeaconBlock,
     spec: &ChainSpec,
 ) -> Result<(), Error> {
     verify!(block.slot == state.slot, Invalid::StateSlotMismatch);
 
-    // NOTE: this is not to spec. I think spec is broken. See:
+    let expected_previous_block_root =
-    //
+        Hash256::from_slice(&state.latest_block_header.signed_root());
-    // https://github.com/ethereum/eth2.0-specs/issues/797
     verify!(
-        block.previous_block_root == *state.get_block_root(state.slot - 1, spec)?,
+        block.previous_block_root == expected_previous_block_root,
-        Invalid::ParentBlockRootMismatch
+        Invalid::ParentBlockRootMismatch {
+            state: expected_previous_block_root,
+            block: block.previous_block_root,
+        }
     );
 
     state.latest_block_header = block.temporary_block_header(spec);
@@ -124,9 +126,9 @@ pub fn process_block_header(
 
 /// Verifies the signature of a block.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn verify_block_signature(
+pub fn verify_block_signature<T: EthSpec>(
-    state: &BeaconState,
+    state: &BeaconState<T>,
     block: &BeaconBlock,
     spec: &ChainSpec,
 ) -> Result<(), Error> {
@@ -152,9 +154,9 @@ pub fn verify_block_signature(
 /// Verifies the `randao_reveal` against the block's proposer pubkey and updates
 /// `state.latest_randao_mixes`.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn process_randao(
+pub fn process_randao<T: EthSpec>(
-    state: &mut BeaconState,
+    state: &mut BeaconState<T>,
     block: &BeaconBlock,
     spec: &ChainSpec,
 ) -> Result<(), Error> {
@@ -164,7 +166,7 @@ pub fn process_randao(
     // Verify the RANDAO is a valid signature of the proposer.
     verify!(
         block.body.randao_reveal.verify(
-            &state.current_epoch(spec).hash_tree_root()[..],
+            &state.current_epoch(spec).tree_hash_root()[..],
             spec.get_domain(
                 block.slot.epoch(spec.slots_per_epoch),
                 Domain::Randao,
@@ -183,8 +185,11 @@ pub fn process_randao(
 
 /// Update the `state.eth1_data_votes` based upon the `eth1_data` provided.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn process_eth1_data(state: &mut BeaconState, eth1_data: &Eth1Data) -> Result<(), Error> {
+pub fn process_eth1_data<T: EthSpec>(
+    state: &mut BeaconState<T>,
+    eth1_data: &Eth1Data,
+) -> Result<(), Error> {
     // Attempt to find a `Eth1DataVote` with matching `Eth1Data`.
     let matching_eth1_vote_index = state
         .eth1_data_votes
@@ -209,9 +214,9 @@ pub fn process_eth1_data(state: &mut BeaconState, eth1_data: &Eth1Data) -> Resul
 /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns
 /// an `Err` describing the invalid object or cause of failure.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn process_proposer_slashings(
+pub fn process_proposer_slashings<T: EthSpec>(
-    state: &mut BeaconState,
+    state: &mut BeaconState<T>,
     proposer_slashings: &[ProposerSlashing],
     spec: &ChainSpec,
 ) -> Result<(), Error> {
@@ -242,9 +247,9 @@ pub fn process_proposer_slashings(
 /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns
 /// an `Err` describing the invalid object or cause of failure.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn process_attester_slashings(
+pub fn process_attester_slashings<T: EthSpec>(
-    state: &mut BeaconState,
+    state: &mut BeaconState<T>,
     attester_slashings: &[AttesterSlashing],
     spec: &ChainSpec,
 ) -> Result<(), Error> {
@@ -300,9 +305,9 @@ pub fn process_attester_slashings(
 /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns
 /// an `Err` describing the invalid object or cause of failure.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn process_attestations(
+pub fn process_attestations<T: EthSpec>(
-    state: &mut BeaconState,
+    state: &mut BeaconState<T>,
     attestations: &[Attestation],
     spec: &ChainSpec,
 ) -> Result<(), Error> {
@@ -342,9 +347,9 @@ pub fn process_attestations(
 /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns
 /// an `Err` describing the invalid object or cause of failure.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn process_deposits(
+pub fn process_deposits<T: EthSpec>(
-    state: &mut BeaconState,
+    state: &mut BeaconState<T>,
     deposits: &[Deposit],
     spec: &ChainSpec,
 ) -> Result<(), Error> {
@@ -412,9 +417,9 @@ pub fn process_deposits(
 /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns
 /// an `Err` describing the invalid object or cause of failure.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn process_exits(
+pub fn process_exits<T: EthSpec>(
-    state: &mut BeaconState,
+    state: &mut BeaconState<T>,
     voluntary_exits: &[VoluntaryExit],
     spec: &ChainSpec,
 ) -> Result<(), Error> {
@@ -444,9 +449,9 @@ pub fn process_exits(
 /// Returns `Ok(())` if the validation and state updates completed successfully, otherwise returns
 /// an `Err` describing the invalid object or cause of failure.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn process_transfers(
+pub fn process_transfers<T: EthSpec>(
-    state: &mut BeaconState,
+    state: &mut BeaconState<T>,
     transfers: &[Transfer],
     spec: &ChainSpec,
 ) -> Result<(), Error> {

eth2/state_processing/src/per_block_processing/errors.rs

@@ -67,7 +67,10 @@ impl_from_beacon_state_error!(BlockProcessingError);
 #[derive(Debug, PartialEq)]
 pub enum BlockInvalid {
     StateSlotMismatch,
-    ParentBlockRootMismatch,
+    ParentBlockRootMismatch {
+        state: Hash256,
+        block: Hash256,
+    },
     BadSignature,
     BadRandaoSignature,
     MaxAttestationsExceeded,
@@ -271,10 +274,10 @@ pub enum ProposerSlashingValidationError {
 pub enum ProposerSlashingInvalid {
     /// The proposer index is not a known validator.
     ProposerUnknown(u64),
-    /// The two proposal have different slots.
+    /// The two proposal have different epochs.
     ///
     /// (proposal_1_slot, proposal_2_slot)
-    ProposalSlotMismatch(Slot, Slot),
+    ProposalEpochMismatch(Slot, Slot),
     /// The proposals are identical and therefore not slashable.
     ProposalsIdentical,
     /// The specified proposer has already been slashed.

eth2/state_processing/src/per_block_processing/validate_attestation.rs

@@ -1,6 +1,6 @@
 use super::errors::{AttestationInvalid as Invalid, AttestationValidationError as Error};
 use crate::common::verify_bitfield_length;
-use ssz::TreeHash;
+use tree_hash::TreeHash;
 use types::*;
 
 /// Indicates if an `Attestation` is valid to be included in a block in the current epoch of the
@@ -8,9 +8,9 @@ use types::*;
 ///
 /// Returns `Ok(())` if the `Attestation` is valid, otherwise indicates the reason for invalidity.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn validate_attestation(
+pub fn validate_attestation<T: EthSpec>(
-    state: &BeaconState,
+    state: &BeaconState<T>,
     attestation: &Attestation,
     spec: &ChainSpec,
 ) -> Result<(), Error> {
@@ -18,8 +18,8 @@ pub fn validate_attestation(
 }
 
 /// Like `validate_attestation` but doesn't run checks which may become true in future states.
-pub fn validate_attestation_time_independent_only(
+pub fn validate_attestation_time_independent_only<T: EthSpec>(
-    state: &BeaconState,
+    state: &BeaconState<T>,
     attestation: &Attestation,
     spec: &ChainSpec,
 ) -> Result<(), Error> {
@@ -31,9 +31,9 @@ pub fn validate_attestation_time_independent_only(
 ///
 /// Returns `Ok(())` if the `Attestation` is valid, otherwise indicates the reason for invalidity.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn validate_attestation_without_signature(
+pub fn validate_attestation_without_signature<T: EthSpec>(
-    state: &BeaconState,
+    state: &BeaconState<T>,
     attestation: &Attestation,
     spec: &ChainSpec,
 ) -> Result<(), Error> {
@@ -44,9 +44,9 @@ pub fn validate_attestation_without_signature(
 /// given state, optionally validating the aggregate signature.
 ///
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-fn validate_attestation_parametric(
+fn validate_attestation_parametric<T: EthSpec>(
-    state: &BeaconState,
+    state: &BeaconState<T>,
     attestation: &Attestation,
     spec: &ChainSpec,
     verify_signature: bool,
@@ -167,10 +167,10 @@ fn validate_attestation_parametric(
 /// Verify that the `source_epoch` and `source_root` of an `Attestation` correctly
 /// match the current (or previous) justified epoch and root from the state.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-fn verify_justified_epoch_and_root(
+fn verify_justified_epoch_and_root<T: EthSpec>(
     attestation: &Attestation,
-    state: &BeaconState,
+    state: &BeaconState<T>,
     spec: &ChainSpec,
 ) -> Result<(), Error> {
     let state_epoch = state.slot.epoch(spec.slots_per_epoch);
@@ -222,9 +222,9 @@ fn verify_justified_epoch_and_root(
 ///  - `custody_bitfield` does not have a bit for each index of `committee`.
 ///  - A `validator_index` in `committee` is not in `state.validator_registry`.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-fn verify_attestation_signature(
+fn verify_attestation_signature<T: EthSpec>(
-    state: &BeaconState,
+    state: &BeaconState<T>,
     committee: &[usize],
     a: &Attestation,
     spec: &ChainSpec,
@@ -270,14 +270,14 @@ fn verify_attestation_signature(
         data: a.data.clone(),
         custody_bit: false,
     }
-    .hash_tree_root();
+    .tree_hash_root();
 
     // Message when custody bitfield is `true`
     let message_1 = AttestationDataAndCustodyBit {
         data: a.data.clone(),
         custody_bit: true,
     }
-    .hash_tree_root();
+    .tree_hash_root();
 
     let mut messages = vec![];
     let mut keys = vec![];

eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs

@@ -7,9 +7,9 @@ use types::*;
 ///
 /// Returns `Ok(())` if the `AttesterSlashing` is valid, otherwise indicates the reason for invalidity.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn verify_attester_slashing(
+pub fn verify_attester_slashing<T: EthSpec>(
-    state: &BeaconState,
+    state: &BeaconState<T>,
     attester_slashing: &AttesterSlashing,
     should_verify_slashable_attestations: bool,
     spec: &ChainSpec,
@@ -41,9 +41,9 @@ pub fn verify_attester_slashing(
 ///
 /// Returns Ok(indices) if `indices.len() > 0`.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn gather_attester_slashing_indices(
+pub fn gather_attester_slashing_indices<T: EthSpec>(
-    state: &BeaconState,
+    state: &BeaconState<T>,
     attester_slashing: &AttesterSlashing,
     spec: &ChainSpec,
 ) -> Result<Vec<u64>, Error> {
@@ -57,8 +57,8 @@ pub fn gather_attester_slashing_indices(
 
 /// Same as `gather_attester_slashing_indices` but allows the caller to specify the criteria
 /// for determining whether a given validator should be considered slashed.
-pub fn gather_attester_slashing_indices_modular<F>(
+pub fn gather_attester_slashing_indices_modular<F, T: EthSpec>(
-    state: &BeaconState,
+    state: &BeaconState<T>,
     attester_slashing: &AttesterSlashing,
     is_slashed: F,
     spec: &ChainSpec,

eth2/state_processing/src/per_block_processing/verify_deposit.rs

@@ -15,9 +15,9 @@ use types::*;
 ///
 /// Note: this function is incomplete.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn verify_deposit(
+pub fn verify_deposit<T: EthSpec>(
-    state: &BeaconState,
+    state: &BeaconState<T>,
     deposit: &Deposit,
     verify_merkle_branch: bool,
     spec: &ChainSpec,
@@ -46,8 +46,11 @@ pub fn verify_deposit(
 
 /// Verify that the `Deposit` index is correct.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn verify_deposit_index(state: &BeaconState, deposit: &Deposit) -> Result<(), Error> {
+pub fn verify_deposit_index<T: EthSpec>(
+    state: &BeaconState<T>,
+    deposit: &Deposit,
+) -> Result<(), Error> {
     verify!(
         deposit.index == state.deposit_index,
         Invalid::BadIndex {
@@ -65,8 +68,8 @@ pub fn verify_deposit_index(state: &BeaconState, deposit: &Deposit) -> Result<()
 /// ## Errors
 ///
 /// Errors if the state's `pubkey_cache` is not current.
-pub fn get_existing_validator_index(
+pub fn get_existing_validator_index<T: EthSpec>(
-    state: &BeaconState,
+    state: &BeaconState<T>,
     deposit: &Deposit,
 ) -> Result<Option<u64>, Error> {
     let deposit_input = &deposit.deposit_data.deposit_input;
@@ -88,12 +91,16 @@ pub fn get_existing_validator_index(
 
 /// Verify that a deposit is included in the state's eth1 deposit root.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-fn verify_deposit_merkle_proof(state: &BeaconState, deposit: &Deposit, spec: &ChainSpec) -> bool {
+fn verify_deposit_merkle_proof<T: EthSpec>(
+    state: &BeaconState<T>,
+    deposit: &Deposit,
+    spec: &ChainSpec,
+) -> bool {
     let leaf = hash(&get_serialized_deposit_data(deposit));
     verify_merkle_proof(
         Hash256::from_slice(&leaf),
-        &deposit.proof,
+        &deposit.proof[..],
         spec.deposit_contract_tree_depth as usize,
         deposit.index as usize,
         state.latest_eth1_data.deposit_root,
@@ -102,7 +109,7 @@ fn verify_deposit_merkle_proof(state: &BeaconState, deposit: &Deposit, spec: &Ch
 
 /// Helper struct for easily getting the serialized data generated by the deposit contract.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 #[derive(Encode)]
 struct SerializedDepositData {
     amount: u64,
@@ -113,7 +120,7 @@ struct SerializedDepositData {
 /// Return the serialized data generated by the deposit contract that is used to generate the
 /// merkle proof.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn get_serialized_deposit_data(deposit: &Deposit) -> Vec<u8> {
     let serialized_deposit_data = SerializedDepositData {
         amount: deposit.deposit_data.amount,

eth2/state_processing/src/per_block_processing/verify_exit.rs

@@ -1,5 +1,5 @@
 use super::errors::{ExitInvalid as Invalid, ExitValidationError as Error};
-use ssz::SignedRoot;
+use tree_hash::SignedRoot;
 use types::*;
 
 /// Indicates if an `Exit` is valid to be included in a block in the current epoch of the given
@@ -7,9 +7,9 @@ use types::*;
 ///
 /// Returns `Ok(())` if the `Exit` is valid, otherwise indicates the reason for invalidity.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn verify_exit(
+pub fn verify_exit<T: EthSpec>(
-    state: &BeaconState,
+    state: &BeaconState<T>,
     exit: &VoluntaryExit,
     spec: &ChainSpec,
 ) -> Result<(), Error> {
@@ -17,8 +17,8 @@ pub fn verify_exit(
 }
 
 /// Like `verify_exit` but doesn't run checks which may become true in future states.
-pub fn verify_exit_time_independent_only(
+pub fn verify_exit_time_independent_only<T: EthSpec>(
-    state: &BeaconState,
+    state: &BeaconState<T>,
     exit: &VoluntaryExit,
     spec: &ChainSpec,
 ) -> Result<(), Error> {
@@ -26,8 +26,8 @@ pub fn verify_exit_time_independent_only(
 }
 
 /// Parametric version of `verify_exit` that skips some checks if `time_independent_only` is true.
-fn verify_exit_parametric(
+fn verify_exit_parametric<T: EthSpec>(
-    state: &BeaconState,
+    state: &BeaconState<T>,
     exit: &VoluntaryExit,
     spec: &ChainSpec,
     time_independent_only: bool,

eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs

@@ -1,5 +1,5 @@
 use super::errors::{ProposerSlashingInvalid as Invalid, ProposerSlashingValidationError as Error};
-use ssz::SignedRoot;
+use tree_hash::SignedRoot;
 use types::*;
 
 /// Indicates if a `ProposerSlashing` is valid to be included in a block in the current epoch of the given
@@ -7,10 +7,10 @@ use types::*;
 ///
 /// Returns `Ok(())` if the `ProposerSlashing` is valid, otherwise indicates the reason for invalidity.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn verify_proposer_slashing(
+pub fn verify_proposer_slashing<T: EthSpec>(
     proposer_slashing: &ProposerSlashing,
-    state: &BeaconState,
+    state: &BeaconState<T>,
     spec: &ChainSpec,
 ) -> Result<(), Error> {
     let proposer = state
@@ -21,8 +21,9 @@ pub fn verify_proposer_slashing(
         })?;
 
     verify!(
-        proposer_slashing.header_1.slot == proposer_slashing.header_2.slot,
+        proposer_slashing.header_1.slot.epoch(spec.slots_per_epoch)
-        Invalid::ProposalSlotMismatch(
+            == proposer_slashing.header_2.slot.epoch(spec.slots_per_epoch),
+        Invalid::ProposalEpochMismatch(
             proposer_slashing.header_1.slot,
             proposer_slashing.header_2.slot
         )
@@ -66,7 +67,7 @@ pub fn verify_proposer_slashing(
 ///
 /// Returns `true` if the signature is valid.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn verify_header_signature(
     header: &BeaconBlockHeader,
     pubkey: &PublicKey,

eth2/state_processing/src/per_block_processing/verify_slashable_attestation.rs

@@ -2,7 +2,7 @@ use super::errors::{
     SlashableAttestationInvalid as Invalid, SlashableAttestationValidationError as Error,
 };
 use crate::common::verify_bitfield_length;
-use ssz::TreeHash;
+use tree_hash::TreeHash;
 use types::*;
 
 /// Indicates if a `SlashableAttestation` is valid to be included in a block in the current epoch of the given
@@ -10,9 +10,9 @@ use types::*;
 ///
 /// Returns `Ok(())` if the `SlashableAttestation` is valid, otherwise indicates the reason for invalidity.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn verify_slashable_attestation(
+pub fn verify_slashable_attestation<T: EthSpec>(
-    state: &BeaconState,
+    state: &BeaconState<T>,
     slashable_attestation: &SlashableAttestation,
     spec: &ChainSpec,
 ) -> Result<(), Error> {
@@ -77,12 +77,12 @@ pub fn verify_slashable_attestation(
         data: slashable_attestation.data.clone(),
         custody_bit: false,
     }
-    .hash_tree_root();
+    .tree_hash_root();
     let message_1 = AttestationDataAndCustodyBit {
         data: slashable_attestation.data.clone(),
         custody_bit: true,
     }
-    .hash_tree_root();
+    .tree_hash_root();
 
     let mut messages = vec![];
     let mut keys = vec![];

eth2/state_processing/src/per_block_processing/verify_transfer.rs

@@ -1,6 +1,6 @@
 use super::errors::{TransferInvalid as Invalid, TransferValidationError as Error};
 use bls::get_withdrawal_credentials;
-use ssz::SignedRoot;
+use tree_hash::SignedRoot;
 use types::*;
 
 /// Indicates if a `Transfer` is valid to be included in a block in the current epoch of the given
@@ -10,9 +10,9 @@ use types::*;
 ///
 /// Note: this function is incomplete.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn verify_transfer(
+pub fn verify_transfer<T: EthSpec>(
-    state: &BeaconState,
+    state: &BeaconState<T>,
     transfer: &Transfer,
     spec: &ChainSpec,
 ) -> Result<(), Error> {
@@ -20,8 +20,8 @@ pub fn verify_transfer(
 }
 
 /// Like `verify_transfer` but doesn't run checks which may become true in future states.
-pub fn verify_transfer_time_independent_only(
+pub fn verify_transfer_time_independent_only<T: EthSpec>(
-    state: &BeaconState,
+    state: &BeaconState<T>,
     transfer: &Transfer,
     spec: &ChainSpec,
 ) -> Result<(), Error> {
@@ -29,8 +29,8 @@ pub fn verify_transfer_time_independent_only(
 }
 
 /// Parametric version of `verify_transfer` that allows some checks to be skipped.
-fn verify_transfer_parametric(
+fn verify_transfer_parametric<T: EthSpec>(
-    state: &BeaconState,
+    state: &BeaconState<T>,
     transfer: &Transfer,
     spec: &ChainSpec,
     time_independent_only: bool,
@@ -122,9 +122,9 @@ fn verify_transfer_parametric(
 ///
 /// Does not check that the transfer is valid, however checks for overflow in all actions.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn execute_transfer(
+pub fn execute_transfer<T: EthSpec>(
-    state: &mut BeaconState,
+    state: &mut BeaconState<T>,
     transfer: &Transfer,
     spec: &ChainSpec,
 ) -> Result<(), Error> {

eth2/state_processing/src/per_epoch_processing.rs

@@ -3,8 +3,8 @@ use errors::EpochProcessingError as Error;
 use process_ejections::process_ejections;
 use process_exit_queue::process_exit_queue;
 use process_slashings::process_slashings;
-use ssz::TreeHash;
 use std::collections::HashMap;
+use tree_hash::TreeHash;
 use types::*;
 use update_registry_and_shuffling_data::update_registry_and_shuffling_data;
 use validator_statuses::{TotalBalances, ValidatorStatuses};
@@ -32,8 +32,11 @@ pub type WinningRootHashSet = HashMap<u64, WinningRoot>;
 /// Mutates the given `BeaconState`, returning early if an error is encountered. If an error is
 /// returned, a state might be "half-processed" and therefore in an invalid state.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn per_epoch_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> {
+pub fn per_epoch_processing<T: EthSpec>(
+    state: &mut BeaconState<T>,
+    spec: &ChainSpec,
+) -> Result<(), Error> {
     // Ensure the previous and next epoch caches are built.
     state.build_epoch_cache(RelativeEpoch::Previous, spec)?;
     state.build_epoch_cache(RelativeEpoch::Current, spec)?;
@@ -86,8 +89,8 @@ pub fn per_epoch_processing(state: &mut BeaconState, spec: &ChainSpec) -> Result
 
 /// Maybe resets the eth1 period.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn maybe_reset_eth1_period(state: &mut BeaconState, spec: &ChainSpec) {
+pub fn maybe_reset_eth1_period<T: EthSpec>(state: &mut BeaconState<T>, spec: &ChainSpec) {
     let next_epoch = state.next_epoch(spec);
     let voting_period = spec.epochs_per_eth1_voting_period;
 
@@ -108,9 +111,9 @@ pub fn maybe_reset_eth1_period(state: &mut BeaconState, spec: &ChainSpec) {
 /// - `justified_epoch`
 /// - `previous_justified_epoch`
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn update_justification_and_finalization(
+pub fn update_justification_and_finalization<T: EthSpec>(
-    state: &mut BeaconState,
+    state: &mut BeaconState<T>,
     total_balances: &TotalBalances,
     spec: &ChainSpec,
 ) -> Result<(), Error> {
@@ -160,13 +163,13 @@ pub fn update_justification_and_finalization(
     if new_justified_epoch != state.current_justified_epoch {
         state.current_justified_epoch = new_justified_epoch;
         state.current_justified_root =
-            *state.get_block_root(new_justified_epoch.start_slot(spec.slots_per_epoch), spec)?;
+            *state.get_block_root(new_justified_epoch.start_slot(spec.slots_per_epoch))?;
     }
 
     if new_finalized_epoch != state.finalized_epoch {
         state.finalized_epoch = new_finalized_epoch;
         state.finalized_root =
-            *state.get_block_root(new_finalized_epoch.start_slot(spec.slots_per_epoch), spec)?;
+            *state.get_block_root(new_finalized_epoch.start_slot(spec.slots_per_epoch))?;
     }
 
     Ok(())
@@ -178,9 +181,9 @@ pub fn update_justification_and_finalization(
 ///
 /// Also returns a `WinningRootHashSet` for later use during epoch processing.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn process_crosslinks(
+pub fn process_crosslinks<T: EthSpec>(
-    state: &mut BeaconState,
+    state: &mut BeaconState<T>,
     spec: &ChainSpec,
 ) -> Result<WinningRootHashSet, Error> {
     let mut winning_root_for_shards: WinningRootHashSet = HashMap::new();
@@ -221,8 +224,11 @@ pub fn process_crosslinks(
 
 /// Finish up an epoch update.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn finish_epoch_update(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> {
+pub fn finish_epoch_update<T: EthSpec>(
+    state: &mut BeaconState<T>,
+    spec: &ChainSpec,
+) -> Result<(), Error> {
     let current_epoch = state.current_epoch(spec);
     let next_epoch = state.next_epoch(spec);
 
@@ -236,16 +242,12 @@ pub fn finish_epoch_update(state: &mut BeaconState, spec: &ChainSpec) -> Result<
         let active_index_root = Hash256::from_slice(
             &state
                 .get_active_validator_indices(next_epoch + spec.activation_exit_delay)
-                .hash_tree_root()[..],
+                .tree_hash_root()[..],
         );
         state.set_active_index_root(next_epoch, active_index_root, spec)?;
 
         // Set total slashed balances
-        state.set_slashed_balance(
+        state.set_slashed_balance(next_epoch, state.get_slashed_balance(current_epoch)?)?;
-            next_epoch,
-            state.get_slashed_balance(current_epoch, spec)?,
-            spec,
-        )?;
 
         // Set randao mix
         state.set_randao_mix(
@@ -257,11 +259,11 @@ pub fn finish_epoch_update(state: &mut BeaconState, spec: &ChainSpec) -> Result<
         state.slot -= 1;
     }
 
-    if next_epoch.as_u64() % (spec.slots_per_historical_root as u64 / spec.slots_per_epoch) == 0 {
+    if next_epoch.as_u64() % (T::SlotsPerHistoricalRoot::to_u64() / spec.slots_per_epoch) == 0 {
-        let historical_batch: HistoricalBatch = state.historical_batch();
+        let historical_batch = state.historical_batch();
         state
             .historical_roots
-            .push(Hash256::from_slice(&historical_batch.hash_tree_root()[..]));
+            .push(Hash256::from_slice(&historical_batch.tree_hash_root()[..]));
     }
 
     state.previous_epoch_attestations = state.current_epoch_attestations.clone();

eth2/state_processing/src/per_epoch_processing/apply_rewards.rs

@@ -32,9 +32,9 @@ impl std::ops::AddAssign for Delta {
 
 /// Apply attester and proposer rewards.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn apply_rewards(
+pub fn apply_rewards<T: EthSpec>(
-    state: &mut BeaconState,
+    state: &mut BeaconState<T>,
     validator_statuses: &mut ValidatorStatuses,
     winning_root_for_shards: &WinningRootHashSet,
     spec: &ChainSpec,
@@ -79,10 +79,10 @@ pub fn apply_rewards(
 /// Applies the attestation inclusion reward to each proposer for every validator who included an
 /// attestation in the previous epoch.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-fn get_proposer_deltas(
+fn get_proposer_deltas<T: EthSpec>(
     deltas: &mut Vec<Delta>,
-    state: &mut BeaconState,
+    state: &mut BeaconState<T>,
     validator_statuses: &mut ValidatorStatuses,
     winning_root_for_shards: &WinningRootHashSet,
     spec: &ChainSpec,
@@ -120,10 +120,10 @@ fn get_proposer_deltas(
 
 /// Apply rewards for participation in attestations during the previous epoch.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-fn get_justification_and_finalization_deltas(
+fn get_justification_and_finalization_deltas<T: EthSpec>(
     deltas: &mut Vec<Delta>,
-    state: &BeaconState,
+    state: &BeaconState<T>,
     validator_statuses: &ValidatorStatuses,
     spec: &ChainSpec,
 ) -> Result<(), Error> {
@@ -163,7 +163,7 @@ fn get_justification_and_finalization_deltas(
 
 /// Determine the delta for a single validator, if the chain is finalizing normally.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn compute_normal_justification_and_finalization_delta(
     validator: &ValidatorStatus,
     total_balances: &TotalBalances,
@@ -215,7 +215,7 @@ fn compute_normal_justification_and_finalization_delta(
 
 /// Determine the delta for a single delta, assuming the chain is _not_ finalizing normally.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
 fn compute_inactivity_leak_delta(
     validator: &ValidatorStatus,
     base_reward: u64,
@@ -261,10 +261,10 @@ fn compute_inactivity_leak_delta(
 
 /// Calculate the deltas based upon the winning roots for attestations during the previous epoch.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-fn get_crosslink_deltas(
+fn get_crosslink_deltas<T: EthSpec>(
     deltas: &mut Vec<Delta>,
-    state: &BeaconState,
+    state: &BeaconState<T>,
     validator_statuses: &ValidatorStatuses,
     spec: &ChainSpec,
 ) -> Result<(), Error> {
@@ -295,9 +295,9 @@ fn get_crosslink_deltas(
 
 /// Returns the base reward for some validator.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-fn get_base_reward(
+fn get_base_reward<T: EthSpec>(
-    state: &BeaconState,
+    state: &BeaconState<T>,
     index: usize,
     previous_total_balance: u64,
     spec: &ChainSpec,
@@ -312,9 +312,9 @@ fn get_base_reward(
 
 /// Returns the inactivity penalty for some validator.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-fn get_inactivity_penalty(
+fn get_inactivity_penalty<T: EthSpec>(
-    state: &BeaconState,
+    state: &BeaconState<T>,
     index: usize,
     epochs_since_finality: u64,
     previous_total_balance: u64,
@@ -328,7 +328,7 @@ fn get_inactivity_penalty(
 
 /// Returns the epochs since the last finalized epoch.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-fn epochs_since_finality(state: &BeaconState, spec: &ChainSpec) -> Epoch {
+fn epochs_since_finality<T: EthSpec>(state: &BeaconState<T>, spec: &ChainSpec) -> Epoch {
     state.current_epoch(spec) + 1 - state.finalized_epoch
 }

eth2/state_processing/src/per_epoch_processing/get_attestation_participants.rs

@@ -3,9 +3,9 @@ use types::*;
 
 /// Returns validator indices which participated in the attestation.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn get_attestation_participants(
+pub fn get_attestation_participants<T: EthSpec>(
-    state: &BeaconState,
+    state: &BeaconState<T>,
     attestation_data: &AttestationData,
     bitfield: &Bitfield,
     spec: &ChainSpec,

eth2/state_processing/src/per_epoch_processing/inclusion_distance.rs

@@ -5,9 +5,9 @@ use types::*;
 /// Returns the distance between the first included attestation for some validator and this
 /// slot.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn inclusion_distance(
+pub fn inclusion_distance<T: EthSpec>(
-    state: &BeaconState,
+    state: &BeaconState<T>,
     attestations: &[&PendingAttestation],
     validator_index: usize,
     spec: &ChainSpec,
@@ -18,9 +18,9 @@ pub fn inclusion_distance(
 
 /// Returns the slot of the earliest included attestation for some validator.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn inclusion_slot(
+pub fn inclusion_slot<T: EthSpec>(
-    state: &BeaconState,
+    state: &BeaconState<T>,
     attestations: &[&PendingAttestation],
     validator_index: usize,
     spec: &ChainSpec,
@@ -31,9 +31,9 @@ pub fn inclusion_slot(
 
 /// Finds the earliest included attestation for some validator.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-fn earliest_included_attestation(
+fn earliest_included_attestation<T: EthSpec>(
-    state: &BeaconState,
+    state: &BeaconState<T>,
     attestations: &[&PendingAttestation],
     validator_index: usize,
     spec: &ChainSpec,

eth2/state_processing/src/per_epoch_processing/process_ejections.rs

@@ -4,8 +4,11 @@ use types::{BeaconStateError as Error, *};
 /// Iterate through the validator registry and eject active validators with balance below
 /// ``EJECTION_BALANCE``.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn process_ejections(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), Error> {
+pub fn process_ejections<T: EthSpec>(
+    state: &mut BeaconState<T>,
+    spec: &ChainSpec,
+) -> Result<(), Error> {
     // There is an awkward double (triple?) loop here because we can't loop across the borrowed
     // active validator indices and mutate state in the one loop.
     let exitable: Vec<usize> = state

eth2/state_processing/src/per_epoch_processing/process_exit_queue.rs

@@ -2,8 +2,8 @@ use types::*;
 
 /// Process the exit queue.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn process_exit_queue(state: &mut BeaconState, spec: &ChainSpec) {
+pub fn process_exit_queue<T: EthSpec>(state: &mut BeaconState<T>, spec: &ChainSpec) {
     let current_epoch = state.current_epoch(spec);
 
     let eligible = |index: usize| {
@@ -31,9 +31,9 @@ pub fn process_exit_queue(state: &mut BeaconState, spec: &ChainSpec) {
 
 /// Initiate an exit for the validator of the given `index`.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-fn prepare_validator_for_withdrawal(
+fn prepare_validator_for_withdrawal<T: EthSpec>(
-    state: &mut BeaconState,
+    state: &mut BeaconState<T>,
     validator_index: usize,
     spec: &ChainSpec,
 ) {

eth2/state_processing/src/per_epoch_processing/process_slashings.rs

@@ -2,21 +2,21 @@ use types::{BeaconStateError as Error, *};
 
 /// Process slashings.
 ///
-/// Spec v0.5.0
+/// Spec v0.5.1
-pub fn process_slashings(
+pub fn process_slashings<T: EthSpec>(
-    state: &mut BeaconState,
+    state: &mut BeaconState<T>,
     current_total_balance: u64,
     spec: &ChainSpec,
 ) -> Result<(), Error> {
     let current_epoch = state.current_epoch(spec);
 
-    let total_at_start = state.get_slashed_balance(current_epoch + 1, spec)?;
+    let total_at_start = state.get_slashed_balance(current_epoch + 1)?;
-    let total_at_end = state.get_slashed_balance(current_epoch, spec)?;
+    let total_at_end = state.get_slashed_balance(current_epoch)?;
     let total_penalities = total_at_end - total_at_start;
 
     for (index, validator) in state.validator_registry.iter().enumerate() {
         let should_penalize = current_epoch.as_usize()
-            == validator.withdrawable_epoch.as_usize() - spec.latest_slashed_exit_length / 2;
+            == validator.withdrawable_epoch.as_usize() - T::LatestSlashedExitLength::to_usize() / 2;
 
         if validator.slashed && should_penalize {
             let effective_balance = state.get_effective_balance(index, spec)?;