Merge branch 'master' into v0.6.1

Cargo.toml

@@ -24,8 +24,9 @@ members = [
 	"eth2/utils/fisher_yates_shuffle",
     "eth2/utils/test_random_derive",
 	"beacon_node",
-	"beacon_node/db",
+	"beacon_node/store",
 	"beacon_node/client",
+	"beacon_node/http_server",
 	"beacon_node/network",
 	"beacon_node/eth2-libp2p",
     "beacon_node/rpc",

README.md

@@ -111,7 +111,7 @@ tests and benchmarks which may be of interest.
 
 A few basic steps are needed to get set up:
 
-   1. Install [rustup](https://rustup.rs/).  It's a toolchain manager for Rust (Linux | macos | Windows). For installation run the below command in your terminal `$ curl https://sh.rustup.rs -sSf | sh`
+   1. Install [rustup](https://rustup.rs/).  It's a toolchain manager for Rust (Linux | macOS | Windows). For installation, download the script with `$ curl -f https://sh.rustup.rs > rustup.sh`, review its content (e.g. `$ less ./rustup.sh`) and run the script `$ ./rustup.sh` (you may need to change the permissions to allow execution, i.e. `$ chmod +x rustup.sh`) 
    2. (Linux & MacOS) To configure your current shell run: `$ source $HOME/.cargo/env`
    3. Use the command `rustup show` to get information about the Rust installation. You should see that the
    active toolchain is the stable version.
@@ -176,4 +176,4 @@ Ping @paulhauner or @AgeManning to get the quickest response.
 
 If you support the cause, we could certainly use donations to help fund development:
 
-`0x25c4a76E7d118705e7Ea2e9b7d8C59930d8aCD3b`
+`0x25c4a76E7d118705e7Ea2e9b7d8C59930d8aCD3b` (donation.sigmaprime.eth)

beacon_node/Cargo.toml

@@ -6,6 +6,7 @@ edition = "2018"
 
 [dependencies]
 types = { path = "../eth2/types" }
+store = { path = "./store" }
 client = { path = "client" }
 version = { path = "version" }
 clap = "2.32.0"

beacon_node/beacon_chain/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2018"
 [dependencies]
 bls = { path = "../../eth2/utils/bls" }
 boolean-bitfield = { path = "../../eth2/utils/boolean-bitfield" }
-db = { path = "../db" }
+store = { path = "../store" }
 failure = "0.1"
 failure_derive = "0.1"
 hashing = { path = "../../eth2/utils/hashing" }

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -1,16 +1,11 @@
 use crate::checkpoint::CheckPoint;
 use crate::errors::{BeaconChainError as Error, BlockProductionError};
-use db::{
-    stores::{BeaconBlockStore, BeaconStateStore},
-    ClientDB, DBError,
-};
 use fork_choice::{ForkChoice, ForkChoiceError};
 use log::{debug, trace};
 use operation_pool::DepositInsertStatus;
 use operation_pool::OperationPool;
 use parking_lot::{RwLock, RwLockReadGuard};
 use slot_clock::SlotClock;
-use ssz::ssz_encode;
 use state_processing::per_block_processing::errors::{
     AttestationValidationError, AttesterSlashingValidationError, DepositValidationError,
     ExitValidationError, ProposerSlashingValidationError, TransferValidationError,
@@ -20,6 +15,7 @@ use state_processing::{
     per_slot_processing, BlockProcessingError, SlotProcessingError,
 };
 use std::sync::Arc;
+use store::{Error as DBError, Store};
 use types::*;
 
 #[derive(Debug, PartialEq)]
@@ -83,40 +79,39 @@ impl BlockProcessingOutcome {
     }
 }
 
-pub struct BeaconChain<T: ClientDB + Sized, U: SlotClock, F: ForkChoice, E: EthSpec> {
+pub trait BeaconChainTypes {
-    pub block_store: Arc<BeaconBlockStore<T>>,
+    type Store: store::Store;
-    pub state_store: Arc<BeaconStateStore<T>>,
+    type SlotClock: slot_clock::SlotClock;
-    pub slot_clock: U,
+    type ForkChoice: fork_choice::ForkChoice;
-    pub op_pool: OperationPool<E>,
+    type EthSpec: types::EthSpec;
-    canonical_head: RwLock<CheckPoint<E>>,
-    finalized_head: RwLock<CheckPoint<E>>,
-    pub state: RwLock<BeaconState<E>>,
-    pub spec: ChainSpec,
-    pub fork_choice: RwLock<F>,
 }
 
-impl<T, U, F, E> BeaconChain<T, U, F, E>
+pub struct BeaconChain<T: BeaconChainTypes> {
-where
+    pub store: Arc<T::Store>,
-    T: ClientDB,
+    pub slot_clock: T::SlotClock,
-    U: SlotClock,
+    pub op_pool: OperationPool<T::EthSpec>,
-    F: ForkChoice,
+    canonical_head: RwLock<CheckPoint<T::EthSpec>>,
-    E: EthSpec,
+    finalized_head: RwLock<CheckPoint<T::EthSpec>>,
-{
+    pub state: RwLock<BeaconState<T::EthSpec>>,
+    pub spec: ChainSpec,
+    pub fork_choice: RwLock<T::ForkChoice>,
+}
+
+impl<T: BeaconChainTypes> BeaconChain<T> {
     /// Instantiate a new Beacon Chain, from genesis.
     pub fn from_genesis(
-        state_store: Arc<BeaconStateStore<T>>,
+        store: Arc<T::Store>,
-        block_store: Arc<BeaconBlockStore<T>>,
+        slot_clock: T::SlotClock,
-        slot_clock: U,
+        mut genesis_state: BeaconState<T::EthSpec>,
-        mut genesis_state: BeaconState<E>,
         genesis_block: BeaconBlock,
         spec: ChainSpec,
-        fork_choice: F,
+        fork_choice: T::ForkChoice,
     ) -> Result<Self, Error> {
         let state_root = genesis_state.canonical_root();
-        state_store.put(&state_root, &ssz_encode(&genesis_state)[..])?;
+        store.put(&state_root, &genesis_state)?;
 
         let block_root = genesis_block.block_header().canonical_root();
-        block_store.put(&block_root, &ssz_encode(&genesis_block)[..])?;
+        store.put(&block_root, &genesis_block)?;
 
         let finalized_head = RwLock::new(CheckPoint::new(
             genesis_block.clone(),
@@ -134,8 +129,7 @@ where
         genesis_state.build_all_caches(&spec)?;
 
         Ok(Self {
-            block_store,
+            store,
-            state_store,
             slot_clock,
             op_pool: OperationPool::new(),
             state: RwLock::new(genesis_state),
@@ -230,12 +224,12 @@ where
                 Err(BeaconStateError::SlotOutOfBounds) => {
                     // Read the earliest historic state in the current slot.
                     let earliest_historic_slot =
-                        state.slot - Slot::from(E::SlotsPerHistoricalRoot::to_usize());
+                        state.slot - Slot::from(T::EthSpec::slots_per_historical_root());
                     // Load the earlier state from disk.
                     let new_state_root = state.get_state_root(earliest_historic_slot)?;
 
                     // Break if the DB is unable to load the state.
-                    state = match self.state_store.get_deserialized(&new_state_root) {
+                    state = match self.store.get(&new_state_root) {
                         Ok(Some(state)) => state,
                         _ => break,
                     }
@@ -262,7 +256,7 @@ where
     ///
     /// May return a database error.
     pub fn get_block(&self, block_root: &Hash256) -> Result<Option<BeaconBlock>, Error> {
-        Ok(self.block_store.get_deserialized(block_root)?)
+        Ok(self.store.get(block_root)?)
     }
 
     /// Update the canonical head to some new values.
@@ -270,7 +264,7 @@ where
         &self,
         new_beacon_block: BeaconBlock,
         new_beacon_block_root: Hash256,
-        new_beacon_state: BeaconState<E>,
+        new_beacon_state: BeaconState<T::EthSpec>,
         new_beacon_state_root: Hash256,
     ) {
         debug!(
@@ -292,7 +286,7 @@ where
     /// It is important to note that the `beacon_state` returned may not match the present slot. It
     /// is the state as it was when the head block was received, which could be some slots prior to
     /// now.
-    pub fn head(&self) -> RwLockReadGuard<CheckPoint<E>> {
+    pub fn head(&self) -> RwLockReadGuard<CheckPoint<T::EthSpec>> {
         self.canonical_head.read()
     }
 
@@ -302,7 +296,7 @@ where
     /// state and calling `catchup_state` as it will not result in an old state being installed and
     /// then having it iteratively updated -- in such a case it's possible for another thread to
     /// find the state at an old slot.
-    pub fn update_state(&self, mut state: BeaconState<E>) -> Result<(), Error> {
+    pub fn update_state(&self, mut state: BeaconState<T::EthSpec>) -> Result<(), Error> {
         let present_slot = match self.slot_clock.present_slot() {
             Ok(Some(slot)) => slot,
             _ => return Err(Error::UnableToReadSlot),
@@ -357,7 +351,7 @@ where
         &self,
         new_beacon_block: BeaconBlock,
         new_beacon_block_root: Hash256,
-        new_beacon_state: BeaconState<E>,
+        new_beacon_state: BeaconState<T::EthSpec>,
         new_beacon_state_root: Hash256,
     ) {
         let mut finalized_head = self.finalized_head.write();
@@ -371,7 +365,7 @@ where
 
     /// Returns a read-lock guarded `CheckPoint` struct for reading the justified head (as chosen,
     /// indirectly,  by the fork-choice rule).
-    pub fn finalized_head(&self) -> RwLockReadGuard<CheckPoint<E>> {
+    pub fn finalized_head(&self) -> RwLockReadGuard<CheckPoint<T::EthSpec>> {
         self.finalized_head.read()
     }
 
@@ -588,7 +582,7 @@ where
         // Load the blocks parent block from the database, returning invalid if that block is not
         // found.
         let parent_block_root = block.previous_block_root;
-        let parent_block = match self.block_store.get_deserialized(&parent_block_root)? {
+        let parent_block: BeaconBlock = match self.store.get(&parent_block_root)? {
             Some(previous_block_root) => previous_block_root,
             None => {
                 return Ok(BlockProcessingOutcome::InvalidBlock(
@@ -601,15 +595,15 @@ where
         // It is an error because if know the parent block we should also know the parent state.
         let parent_state_root = parent_block.state_root;
         let parent_state = self
-            .state_store
+            .store
-            .get_deserialized(&parent_state_root)?
+            .get(&parent_state_root)?
             .ok_or_else(|| Error::DBInconsistent(format!("Missing state {}", parent_state_root)))?;
 
         // TODO: check the block proposer signature BEFORE doing a state transition. This will
         // significantly lower exposure surface to DoS attacks.
 
         // Transition the parent state to the block slot.
-        let mut state = parent_state;
+        let mut state: BeaconState<T::EthSpec> = parent_state;
         for _ in state.slot.as_u64()..block.slot.as_u64() {
             if let Err(e) = per_slot_processing(&mut state, &self.spec) {
                 return Ok(BlockProcessingOutcome::InvalidBlock(
@@ -635,8 +629,8 @@ where
         }
 
         // Store the block and state.
-        self.block_store.put(&block_root, &ssz_encode(&block)[..])?;
+        self.store.put(&block_root, &block)?;
-        self.state_store.put(&state_root, &ssz_encode(&state)[..])?;
+        self.store.put(&state_root, &state)?;
 
         // run the fork_choice add_block logic
         self.fork_choice
@@ -664,7 +658,7 @@ where
     pub fn produce_block(
         &self,
         randao_reveal: Signature,
-    ) -> Result<(BeaconBlock, BeaconState<E>), BlockProductionError> {
+    ) -> Result<(BeaconBlock, BeaconState<T::EthSpec>), BlockProductionError> {
         debug!("Producing block at slot {}...", self.state.read().slot);
 
         let mut state = self.state.read().clone();
@@ -729,15 +723,15 @@ where
             .find_head(&present_head, &self.spec)?;
 
         if new_head != present_head {
-            let block = self
+            let block: BeaconBlock = self
-                .block_store
+                .store
-                .get_deserialized(&new_head)?
+                .get(&new_head)?
                 .ok_or_else(|| Error::MissingBeaconBlock(new_head))?;
             let block_root = block.canonical_root();
 
-            let state = self
+            let state: BeaconState<T::EthSpec> = self
-                .state_store
+                .store
-                .get_deserialized(&block.state_root)?
+                .get(&block.state_root)?
                 .ok_or_else(|| Error::MissingBeaconState(block.state_root))?;
             let state_root = state.canonical_root();
 
@@ -752,14 +746,14 @@ where
 
     /// Returns `true` if the given block root has not been processed.
     pub fn is_new_block_root(&self, beacon_block_root: &Hash256) -> Result<bool, Error> {
-        Ok(!self.block_store.exists(beacon_block_root)?)
+        Ok(!self.store.exists::<BeaconBlock>(beacon_block_root)?)
     }
 
     /// Dumps the entire canonical chain, from the head to genesis to a vector for analysis.
     ///
     /// This could be a very expensive operation and should only be done in testing/analysis
     /// activities.
-    pub fn chain_dump(&self) -> Result<Vec<CheckPoint<E>>, Error> {
+    pub fn chain_dump(&self) -> Result<Vec<CheckPoint<T::EthSpec>>, Error> {
         let mut dump = vec![];
 
         let mut last_slot = CheckPoint {
@@ -778,17 +772,12 @@ where
                 break; // Genesis has been reached.
             }
 
-            let beacon_block = self
+            let beacon_block: BeaconBlock =
-                .block_store
+                self.store.get(&beacon_block_root)?.ok_or_else(|| {
-                .get_deserialized(&beacon_block_root)?
-                .ok_or_else(|| {
                     Error::DBInconsistent(format!("Missing block {}", beacon_block_root))
                 })?;
             let beacon_state_root = beacon_block.state_root;
-            let beacon_state = self
+            let beacon_state = self.store.get(&beacon_state_root)?.ok_or_else(|| {
-                .state_store
-                .get_deserialized(&beacon_state_root)?
-                .ok_or_else(|| {
                 Error::DBInconsistent(format!("Missing state {}", beacon_state_root))
             })?;
 
@@ -811,7 +800,7 @@ where
 
 impl From<DBError> for Error {
     fn from(e: DBError) -> Error {
-        Error::DBError(e.message)
+        Error::DBError(e)
     }
 }
 

beacon_node/beacon_chain/src/errors.rs

@@ -20,7 +20,7 @@ pub enum BeaconChainError {
     UnableToReadSlot,
     BeaconStateError(BeaconStateError),
     DBInconsistent(String),
-    DBError(String),
+    DBError(store::Error),
     ForkChoiceError(ForkChoiceError),
     MissingBeaconBlock(Hash256),
     MissingBeaconState(Hash256),

beacon_node/beacon_chain/src/initialise.rs

@@ -1,116 +0,0 @@
-// Initialisation functions to generate a new BeaconChain.
-// Note: A new version of ClientTypes may need to be implemented for the lighthouse
-// testnet. These are examples. Also. there is code duplication which can/should be cleaned up.
-
-use crate::BeaconChain;
-use db::stores::{BeaconBlockStore, BeaconStateStore};
-use db::{DiskDB, MemoryDB};
-use fork_choice::BitwiseLMDGhost;
-use slot_clock::SystemTimeSlotClock;
-use std::path::PathBuf;
-use std::sync::Arc;
-use tree_hash::TreeHash;
-use types::test_utils::TestingBeaconStateBuilder;
-use types::{BeaconBlock, ChainSpec, FewValidatorsEthSpec, FoundationEthSpec, Hash256};
-
-//TODO: Correct this for prod
-//TODO: Account for historical db
-pub fn initialise_beacon_chain(
-    spec: &ChainSpec,
-    db_name: Option<&PathBuf>,
-) -> Arc<
-    BeaconChain<
-        DiskDB,
-        SystemTimeSlotClock,
-        BitwiseLMDGhost<DiskDB, FoundationEthSpec>,
-        FoundationEthSpec,
-    >,
-> {
-    // set up the db
-    let db = Arc::new(DiskDB::open(
-        db_name.expect("Database directory must be included"),
-        None,
-    ));
-
-    let block_store = Arc::new(BeaconBlockStore::new(db.clone()));
-    let state_store = Arc::new(BeaconStateStore::new(db.clone()));
-
-    let state_builder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(8, &spec);
-    let (genesis_state, _keypairs) = state_builder.build();
-
-    let mut genesis_block = BeaconBlock::empty(&spec);
-    genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root());
-
-    // Slot clock
-    let slot_clock = SystemTimeSlotClock::new(
-        spec.genesis_slot,
-        genesis_state.genesis_time,
-        spec.seconds_per_slot,
-    )
-    .expect("Unable to load SystemTimeSlotClock");
-    // Choose the fork choice
-    let fork_choice = BitwiseLMDGhost::new(block_store.clone(), state_store.clone());
-
-    // Genesis chain
-    //TODO: Handle error correctly
-    Arc::new(
-        BeaconChain::from_genesis(
-            state_store.clone(),
-            block_store.clone(),
-            slot_clock,
-            genesis_state,
-            genesis_block,
-            spec.clone(),
-            fork_choice,
-        )
-        .expect("Terminate if beacon chain generation fails"),
-    )
-}
-
-/// Initialisation of a test beacon chain, uses an in memory db with fixed genesis time.
-pub fn initialise_test_beacon_chain(
-    spec: &ChainSpec,
-    _db_name: Option<&PathBuf>,
-) -> Arc<
-    BeaconChain<
-        MemoryDB,
-        SystemTimeSlotClock,
-        BitwiseLMDGhost<MemoryDB, FewValidatorsEthSpec>,
-        FewValidatorsEthSpec,
-    >,
-> {
-    let db = Arc::new(MemoryDB::open());
-    let block_store = Arc::new(BeaconBlockStore::new(db.clone()));
-    let state_store = Arc::new(BeaconStateStore::new(db.clone()));
-
-    let state_builder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(8, spec);
-    let (genesis_state, _keypairs) = state_builder.build();
-
-    let mut genesis_block = BeaconBlock::empty(spec);
-    genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root());
-
-    // Slot clock
-    let slot_clock = SystemTimeSlotClock::new(
-        spec.genesis_slot,
-        genesis_state.genesis_time,
-        spec.seconds_per_slot,
-    )
-    .expect("Unable to load SystemTimeSlotClock");
-    // Choose the fork choice
-    let fork_choice = BitwiseLMDGhost::new(block_store.clone(), state_store.clone());
-
-    // Genesis chain
-    //TODO: Handle error correctly
-    Arc::new(
-        BeaconChain::from_genesis(
-            state_store.clone(),
-            block_store.clone(),
-            slot_clock,
-            genesis_state,
-            genesis_block,
-            spec.clone(),
-            fork_choice,
-        )
-        .expect("Terminate if beacon chain generation fails"),
-    )
-}

beacon_node/beacon_chain/src/lib.rs

@@ -1,13 +1,12 @@
 mod beacon_chain;
 mod checkpoint;
 mod errors;
-pub mod initialise;
-pub mod test_utils;
 
-pub use self::beacon_chain::{BeaconChain, BlockProcessingOutcome, InvalidBlock, ValidBlock};
+pub use self::beacon_chain::{
+    BeaconChain, BeaconChainTypes, BlockProcessingOutcome, InvalidBlock, ValidBlock,
+};
 pub use self::checkpoint::CheckPoint;
 pub use self::errors::{BeaconChainError, BlockProductionError};
-pub use db;
 pub use fork_choice;
 pub use parking_lot;
 pub use slot_clock;
@@ -15,4 +14,5 @@ pub use state_processing::per_block_processing::errors::{
     AttestationValidationError, AttesterSlashingValidationError, DepositValidationError,
     ExitValidationError, ProposerSlashingValidationError, TransferValidationError,
 };
+pub use store;
 pub use types;

beacon_node/beacon_chain/src/test_utils/mod.rs

@@ -1,3 +0,0 @@
-mod testing_beacon_chain_builder;
-
-pub use testing_beacon_chain_builder::TestingBeaconChainBuilder;

beacon_node/beacon_chain/src/test_utils/testing_beacon_chain_builder.rs

@@ -1,51 +0,0 @@
-pub use crate::{BeaconChain, BeaconChainError, CheckPoint};
-use db::{
-    stores::{BeaconBlockStore, BeaconStateStore},
-    MemoryDB,
-};
-use fork_choice::BitwiseLMDGhost;
-use slot_clock::TestingSlotClock;
-use std::sync::Arc;
-use tree_hash::TreeHash;
-use types::*;
-use types::{test_utils::TestingBeaconStateBuilder, EthSpec, FewValidatorsEthSpec};
-
-type TestingBeaconChain<E> =
-    BeaconChain<MemoryDB, TestingSlotClock, BitwiseLMDGhost<MemoryDB, FewValidatorsEthSpec>, E>;
-
-pub struct TestingBeaconChainBuilder<E: EthSpec> {
-    state_builder: TestingBeaconStateBuilder<E>,
-}
-
-impl<E: EthSpec> TestingBeaconChainBuilder<E> {
-    pub fn build(self, spec: &ChainSpec) -> TestingBeaconChain<E> {
-        let db = Arc::new(MemoryDB::open());
-        let block_store = Arc::new(BeaconBlockStore::new(db.clone()));
-        let state_store = Arc::new(BeaconStateStore::new(db.clone()));
-        let slot_clock = TestingSlotClock::new(spec.genesis_slot.as_u64());
-        let fork_choice = BitwiseLMDGhost::new(block_store.clone(), state_store.clone());
-
-        let (genesis_state, _keypairs) = self.state_builder.build();
-
-        let mut genesis_block = BeaconBlock::empty(&spec);
-        genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root());
-
-        // Create the Beacon Chain
-        BeaconChain::from_genesis(
-            state_store.clone(),
-            block_store.clone(),
-            slot_clock,
-            genesis_state,
-            genesis_block,
-            spec.clone(),
-            fork_choice,
-        )
-        .unwrap()
-    }
-}
-
-impl<E: EthSpec> From<TestingBeaconStateBuilder<E>> for TestingBeaconChainBuilder<E> {
-    fn from(state_builder: TestingBeaconStateBuilder<E>) -> TestingBeaconChainBuilder<E> {
-        TestingBeaconChainBuilder { state_builder }
-    }
-}

beacon_node/client/Cargo.toml

@@ -7,10 +7,12 @@ edition = "2018"
 [dependencies]
 beacon_chain = { path = "../beacon_chain" }
 network = { path = "../network" }
-db = { path = "../db" }
+store = { path = "../store" }
+http_server = { path = "../http_server" }
 rpc = { path = "../rpc" }
 fork_choice = { path = "../../eth2/fork_choice" }
 types = { path = "../../eth2/types" }
+tree_hash = { path = "../../eth2/utils/tree_hash" }
 slot_clock = { path = "../../eth2/utils/slot_clock" }
 error-chain = "0.12.0"
 slog = "^2.2.3"

beacon_node/client/src/beacon_chain_types.rs

@@ -0,0 +1,109 @@
+use crate::ClientConfig;
+use beacon_chain::{
+    fork_choice::BitwiseLMDGhost,
+    slot_clock::SystemTimeSlotClock,
+    store::{DiskStore, MemoryStore, Store},
+    BeaconChain, BeaconChainTypes,
+};
+use std::sync::Arc;
+use tree_hash::TreeHash;
+use types::{
+    test_utils::TestingBeaconStateBuilder, BeaconBlock, EthSpec, FewValidatorsEthSpec, Hash256,
+};
+
+/// Provides a new, initialized `BeaconChain`
+pub trait InitialiseBeaconChain<T: BeaconChainTypes> {
+    fn initialise_beacon_chain(config: &ClientConfig) -> BeaconChain<T>;
+}
+
+/// A testnet-suitable BeaconChainType, using `MemoryStore`.
+#[derive(Clone)]
+pub struct TestnetMemoryBeaconChainTypes;
+
+impl BeaconChainTypes for TestnetMemoryBeaconChainTypes {
+    type Store = MemoryStore;
+    type SlotClock = SystemTimeSlotClock;
+    type ForkChoice = BitwiseLMDGhost<Self::Store, Self::EthSpec>;
+    type EthSpec = FewValidatorsEthSpec;
+}
+
+impl<T> InitialiseBeaconChain<T> for TestnetMemoryBeaconChainTypes
+where
+    T: BeaconChainTypes<
+        Store = MemoryStore,
+        SlotClock = SystemTimeSlotClock,
+        ForkChoice = BitwiseLMDGhost<MemoryStore, FewValidatorsEthSpec>,
+    >,
+{
+    fn initialise_beacon_chain(_config: &ClientConfig) -> BeaconChain<T> {
+        initialize_chain(MemoryStore::open())
+    }
+}
+
+/// A testnet-suitable BeaconChainType, using `DiskStore`.
+#[derive(Clone)]
+pub struct TestnetDiskBeaconChainTypes;
+
+impl BeaconChainTypes for TestnetDiskBeaconChainTypes {
+    type Store = DiskStore;
+    type SlotClock = SystemTimeSlotClock;
+    type ForkChoice = BitwiseLMDGhost<Self::Store, Self::EthSpec>;
+    type EthSpec = FewValidatorsEthSpec;
+}
+
+impl<T> InitialiseBeaconChain<T> for TestnetDiskBeaconChainTypes
+where
+    T: BeaconChainTypes<
+        Store = DiskStore,
+        SlotClock = SystemTimeSlotClock,
+        ForkChoice = BitwiseLMDGhost<DiskStore, FewValidatorsEthSpec>,
+    >,
+{
+    fn initialise_beacon_chain(config: &ClientConfig) -> BeaconChain<T> {
+        let store = DiskStore::open(&config.db_name).expect("Unable to open DB.");
+
+        initialize_chain(store)
+    }
+}
+
+/// Produces a `BeaconChain` given some pre-initialized `Store`.
+fn initialize_chain<T, U: Store, V: EthSpec>(store: U) -> BeaconChain<T>
+where
+    T: BeaconChainTypes<
+        Store = U,
+        SlotClock = SystemTimeSlotClock,
+        ForkChoice = BitwiseLMDGhost<U, V>,
+    >,
+{
+    let spec = T::EthSpec::spec();
+
+    let store = Arc::new(store);
+
+    let state_builder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(8, &spec);
+    let (genesis_state, _keypairs) = state_builder.build();
+
+    let mut genesis_block = BeaconBlock::empty(&spec);
+    genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root());
+
+    // Slot clock
+    let slot_clock = SystemTimeSlotClock::new(
+        spec.genesis_slot,
+        genesis_state.genesis_time,
+        spec.seconds_per_slot,
+    )
+    .expect("Unable to load SystemTimeSlotClock");
+    // Choose the fork choice
+    let fork_choice = BitwiseLMDGhost::new(store.clone());
+
+    // Genesis chain
+    //TODO: Handle error correctly
+    BeaconChain::from_genesis(
+        store,
+        slot_clock,
+        genesis_state,
+        genesis_block,
+        spec.clone(),
+        fork_choice,
+    )
+    .expect("Terminate if beacon chain generation fails")
+}

beacon_node/client/src/client_config.rs

@@ -1,6 +1,6 @@
 use clap::ArgMatches;
-use db::DBType;
 use fork_choice::ForkChoiceAlgorithm;
+use http_server::HttpServerConfig;
 use network::NetworkConfig;
 use slog::error;
 use std::fs;
@@ -12,6 +12,12 @@ use types::multiaddr::ToMultiaddr;
 use types::Multiaddr;
 use types::{ChainSpec, EthSpec, LighthouseTestnetEthSpec};
 
+#[derive(Debug, Clone)]
+pub enum DBType {
+    Memory,
+    Disk,
+}
+
 /// Stores the client configuration for this Lighthouse instance.
 #[derive(Debug, Clone)]
 pub struct ClientConfig {
@@ -22,7 +28,7 @@ pub struct ClientConfig {
     pub db_type: DBType,
     pub db_name: PathBuf,
     pub rpc_conf: rpc::RPCConfig,
-    //pub ipc_conf:
+    pub http_conf: HttpServerConfig, //pub ipc_conf:
 }
 
 impl Default for ClientConfig {
@@ -48,8 +54,9 @@ impl Default for ClientConfig {
             // default to memory db for now
             db_type: DBType::Memory,
             // default db name for disk-based dbs
-            db_name: data_dir.join("chain.db"),
+            db_name: data_dir.join("chain_db"),
             rpc_conf: rpc::RPCConfig::default(),
+            http_conf: HttpServerConfig::default(),
         }
     }
 }
@@ -90,15 +97,17 @@ impl ClientConfig {
         }
 
         // Custom bootnodes
-        // TODO: Handle list of addresses
         if let Some(boot_addresses_str) = args.value_of("boot-nodes") {
-            if let Ok(boot_address) = boot_addresses_str.parse::<Multiaddr>() {
+            let mut boot_addresses_split = boot_addresses_str.split(",");
+            for boot_address in boot_addresses_split {
+                if let Ok(boot_address) = boot_address.parse::<Multiaddr>() {
                     config.net_conf.boot_nodes.append(&mut vec![boot_address]);
                 } else {
                     error!(log, "Invalid Bootnode multiaddress"; "Multiaddr" => boot_addresses_str);
                     return Err("Invalid IP Address");
                 }
             }
+        }
 
         /* Filesystem related arguments */
 
@@ -131,6 +140,12 @@ impl ClientConfig {
             }
         }
 
+        match args.value_of("db") {
+            Some("disk") => config.db_type = DBType::Disk,
+            Some("memory") => config.db_type = DBType::Memory,
+            _ => unreachable!(), // clap prevents this.
+        };
+
         Ok(config)
     }
 }

beacon_node/client/src/client_types.rs

@@ -1,50 +0,0 @@
-use crate::{ArcBeaconChain, ClientConfig};
-use beacon_chain::{
-    db::{ClientDB, DiskDB, MemoryDB},
-    fork_choice::BitwiseLMDGhost,
-    initialise,
-    slot_clock::{SlotClock, SystemTimeSlotClock},
-};
-use fork_choice::ForkChoice;
-use types::{EthSpec, FewValidatorsEthSpec, FoundationEthSpec};
-
-pub trait ClientTypes {
-    type DB: ClientDB + 'static;
-    type SlotClock: SlotClock + 'static;
-    type ForkChoice: ForkChoice + 'static;
-    type EthSpec: EthSpec + 'static;
-
-    fn initialise_beacon_chain(
-        config: &ClientConfig,
-    ) -> ArcBeaconChain<Self::DB, Self::SlotClock, Self::ForkChoice, Self::EthSpec>;
-}
-
-pub struct StandardClientType;
-
-impl ClientTypes for StandardClientType {
-    type DB = DiskDB;
-    type SlotClock = SystemTimeSlotClock;
-    type ForkChoice = BitwiseLMDGhost<DiskDB, Self::EthSpec>;
-    type EthSpec = FoundationEthSpec;
-
-    fn initialise_beacon_chain(
-        config: &ClientConfig,
-    ) -> ArcBeaconChain<Self::DB, Self::SlotClock, Self::ForkChoice, Self::EthSpec> {
-        initialise::initialise_beacon_chain(&config.spec, Some(&config.db_name))
-    }
-}
-
-pub struct TestingClientType;
-
-impl ClientTypes for TestingClientType {
-    type DB = MemoryDB;
-    type SlotClock = SystemTimeSlotClock;
-    type ForkChoice = BitwiseLMDGhost<MemoryDB, Self::EthSpec>;
-    type EthSpec = FewValidatorsEthSpec;
-
-    fn initialise_beacon_chain(
-        config: &ClientConfig,
-    ) -> ArcBeaconChain<Self::DB, Self::SlotClock, Self::ForkChoice, Self::EthSpec> {
-        initialise::initialise_test_beacon_chain(&config.spec, None)
-    }
-}

beacon_node/client/src/lib.rs

@@ -1,16 +1,13 @@
 extern crate slog;
 
+mod beacon_chain_types;
 mod client_config;
-pub mod client_types;
 pub mod error;
 pub mod notifier;
 
 use beacon_chain::BeaconChain;
-pub use client_config::ClientConfig;
+use beacon_chain_types::InitialiseBeaconChain;
-pub use client_types::ClientTypes;
-use db::ClientDB;
 use exit_future::Signal;
-use fork_choice::ForkChoice;
 use futures::{future::Future, Stream};
 use network::Service as NetworkService;
 use slog::{error, info, o};
@@ -20,21 +17,24 @@ use std::sync::Arc;
 use std::time::{Duration, Instant};
 use tokio::runtime::TaskExecutor;
 use tokio::timer::Interval;
-use types::EthSpec;
 
-type ArcBeaconChain<D, S, F, B> = Arc<BeaconChain<D, S, F, B>>;
+pub use beacon_chain::BeaconChainTypes;
+pub use beacon_chain_types::{TestnetDiskBeaconChainTypes, TestnetMemoryBeaconChainTypes};
+pub use client_config::{ClientConfig, DBType};
 
 /// Main beacon node client service. This provides the connection and initialisation of the clients
 /// sub-services in multiple threads.
-pub struct Client<T: ClientTypes> {
+pub struct Client<T: BeaconChainTypes> {
     /// Configuration for the lighthouse client.
     _config: ClientConfig,
     /// The beacon chain for the running client.
-    _beacon_chain: ArcBeaconChain<T::DB, T::SlotClock, T::ForkChoice, T::EthSpec>,
+    _beacon_chain: Arc<BeaconChain<T>>,
     /// Reference to the network service.
-    pub network: Arc<NetworkService<T::EthSpec>>,
+    pub network: Arc<NetworkService<T>>,
     /// Signal to terminate the RPC server.
     pub rpc_exit_signal: Option<Signal>,
+    /// Signal to terminate the HTTP server.
+    pub http_exit_signal: Option<Signal>,
     /// Signal to terminate the slot timer.
     pub slot_timer_exit_signal: Option<Signal>,
     /// The clients logger.
@@ -43,7 +43,10 @@ pub struct Client<T: ClientTypes> {
     phantom: PhantomData<T>,
 }
 
-impl<TClientType: ClientTypes> Client<TClientType> {
+impl<T> Client<T>
+where
+    T: BeaconChainTypes + InitialiseBeaconChain<T> + Clone + 'static,
+{
     /// Generate an instance of the client. Spawn and link all internal sub-processes.
     pub fn new(
         config: ClientConfig,
@@ -51,7 +54,7 @@ impl<TClientType: ClientTypes> Client<TClientType> {
         executor: &TaskExecutor,
     ) -> error::Result<Self> {
         // generate a beacon chain
-        let beacon_chain = TClientType::initialise_beacon_chain(&config);
+        let beacon_chain = Arc::new(T::initialise_beacon_chain(&config));
 
         if beacon_chain.read_slot_clock().is_none() {
             panic!("Cannot start client before genesis!")
@@ -98,7 +101,7 @@ impl<TClientType: ClientTypes> Client<TClientType> {
             Some(rpc::start_server(
                 &config.rpc_conf,
                 executor,
-                network_send,
+                network_send.clone(),
                 beacon_chain.clone(),
                 &log,
             ))
@@ -106,6 +109,17 @@ impl<TClientType: ClientTypes> Client<TClientType> {
             None
         };
 
+        // Start the `http_server` service.
+        //
+        // Note: presently we are ignoring the config and _always_ starting a HTTP server.
+        let http_exit_signal = Some(http_server::start_service(
+            &config.http_conf,
+            executor,
+            network_send,
+            beacon_chain.clone(),
+            &log,
+        ));
+
         let (slot_timer_exit_signal, exit) = exit_future::signal();
         if let Ok(Some(duration_to_next_slot)) = beacon_chain.slot_clock.duration_to_next_slot() {
             // set up the validator work interval - start at next slot and proceed every slot
@@ -135,6 +149,7 @@ impl<TClientType: ClientTypes> Client<TClientType> {
         Ok(Client {
             _config: config,
             _beacon_chain: beacon_chain,
+            http_exit_signal,
             rpc_exit_signal,
             slot_timer_exit_signal: Some(slot_timer_exit_signal),
             log,
@@ -144,13 +159,7 @@ impl<TClientType: ClientTypes> Client<TClientType> {
     }
 }
 
-fn do_state_catchup<T, U, F, E>(chain: &Arc<BeaconChain<T, U, F, E>>, log: &slog::Logger)
+fn do_state_catchup<T: BeaconChainTypes>(chain: &Arc<BeaconChain<T>>, log: &slog::Logger) {
-where
-    T: ClientDB,
-    U: SlotClock,
-    F: ForkChoice,
-    E: EthSpec,
-{
     if let Some(genesis_height) = chain.slots_since_genesis() {
         let result = chain.catchup_state();
 

beacon_node/client/src/notifier.rs

@@ -1,5 +1,5 @@
 use crate::Client;
-use crate::ClientTypes;
+use beacon_chain::BeaconChainTypes;
 use exit_future::Exit;
 use futures::{Future, Stream};
 use slog::{debug, o};
@@ -10,7 +10,11 @@ use tokio::timer::Interval;
 
 /// Thread that monitors the client and reports useful statistics to the user.
 
-pub fn run<T: ClientTypes>(client: &Client<T>, executor: TaskExecutor, exit: Exit) {
+pub fn run<T: BeaconChainTypes + Send + Sync + 'static>(
+    client: &Client<T>,
+    executor: TaskExecutor,
+    exit: Exit,
+) {
     // notification heartbeat
     let interval = Interval::new(Instant::now(), Duration::from_secs(5));
 

beacon_node/db/Cargo.toml

@@ -1,13 +0,0 @@
-[package]
-name = "db"
-version = "0.1.0"
-authors = ["Paul Hauner <paul@paulhauner.com>"]
-edition = "2018"
-
-[dependencies]
-blake2-rfc = "0.2.18"
-bls = { path = "../../eth2/utils/bls" }
-bytes = "0.4.10"
-rocksdb = "0.10.1"
-ssz = { path = "../../eth2/utils/ssz" }
-types = { path = "../../eth2/types" }

beacon_node/db/src/lib.rs

@@ -1,21 +0,0 @@
-extern crate blake2_rfc as blake2;
-extern crate bls;
-extern crate rocksdb;
-
-mod disk_db;
-mod memory_db;
-pub mod stores;
-mod traits;
-
-use self::stores::COLUMNS;
-
-pub use self::disk_db::DiskDB;
-pub use self::memory_db::MemoryDB;
-pub use self::traits::{ClientDB, DBError, DBValue};
-
-/// Currently available database options
-#[derive(Debug, Clone)]
-pub enum DBType {
-    Memory,
-    RocksDB,
-}

beacon_node/db/src/memory_db.rs

@@ -1,236 +0,0 @@
-use super::blake2::blake2b::blake2b;
-use super::COLUMNS;
-use super::{ClientDB, DBError, DBValue};
-use std::collections::{HashMap, HashSet};
-use std::sync::RwLock;
-
-type DBHashMap = HashMap<Vec<u8>, Vec<u8>>;
-type ColumnHashSet = HashSet<String>;
-
-/// An in-memory database implementing the ClientDB trait.
-///
-/// It is not particularily optimized, it exists for ease and speed of testing. It's not expected
-/// this DB would be used outside of tests.
-pub struct MemoryDB {
-    db: RwLock<DBHashMap>,
-    known_columns: RwLock<ColumnHashSet>,
-}
-
-impl MemoryDB {
-    /// Open the in-memory database.
-    ///
-    /// All columns must be supplied initially, you will get an error if you try to access a column
-    /// that was not declared here. This condition is enforced artificially to simulate RocksDB.
-    pub fn open() -> Self {
-        let db: DBHashMap = HashMap::new();
-        let mut known_columns: ColumnHashSet = HashSet::new();
-        for col in &COLUMNS {
-            known_columns.insert(col.to_string());
-        }
-        Self {
-            db: RwLock::new(db),
-            known_columns: RwLock::new(known_columns),
-        }
-    }
-
-    /// Hashes a key and a column name in order to get a unique key for the supplied column.
-    fn get_key_for_col(col: &str, key: &[u8]) -> Vec<u8> {
-        blake2b(32, col.as_bytes(), key).as_bytes().to_vec()
-    }
-}
-
-impl ClientDB for MemoryDB {
-    /// Get the value of some key from the database. Returns `None` if the key does not exist.
-    fn get(&self, col: &str, key: &[u8]) -> Result<Option<DBValue>, DBError> {
-        // Panic if the DB locks are poisoned.
-        let db = self.db.read().unwrap();
-        let known_columns = self.known_columns.read().unwrap();
-
-        if known_columns.contains(&col.to_string()) {
-            let column_key = MemoryDB::get_key_for_col(col, key);
-            Ok(db.get(&column_key).and_then(|val| Some(val.clone())))
-        } else {
-            Err(DBError {
-                message: "Unknown column".to_string(),
-            })
-        }
-    }
-
-    /// Puts a key in the database.
-    fn put(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), DBError> {
-        // Panic if the DB locks are poisoned.
-        let mut db = self.db.write().unwrap();
-        let known_columns = self.known_columns.read().unwrap();
-
-        if known_columns.contains(&col.to_string()) {
-            let column_key = MemoryDB::get_key_for_col(col, key);
-            db.insert(column_key, val.to_vec());
-            Ok(())
-        } else {
-            Err(DBError {
-                message: "Unknown column".to_string(),
-            })
-        }
-    }
-
-    /// Return true if some key exists in some column.
-    fn exists(&self, col: &str, key: &[u8]) -> Result<bool, DBError> {
-        // Panic if the DB locks are poisoned.
-        let db = self.db.read().unwrap();
-        let known_columns = self.known_columns.read().unwrap();
-
-        if known_columns.contains(&col.to_string()) {
-            let column_key = MemoryDB::get_key_for_col(col, key);
-            Ok(db.contains_key(&column_key))
-        } else {
-            Err(DBError {
-                message: "Unknown column".to_string(),
-            })
-        }
-    }
-
-    /// Delete some key from the database.
-    fn delete(&self, col: &str, key: &[u8]) -> Result<(), DBError> {
-        // Panic if the DB locks are poisoned.
-        let mut db = self.db.write().unwrap();
-        let known_columns = self.known_columns.read().unwrap();
-
-        if known_columns.contains(&col.to_string()) {
-            let column_key = MemoryDB::get_key_for_col(col, key);
-            db.remove(&column_key);
-            Ok(())
-        } else {
-            Err(DBError {
-                message: "Unknown column".to_string(),
-            })
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::super::stores::{BLOCKS_DB_COLUMN, VALIDATOR_DB_COLUMN};
-    use super::super::ClientDB;
-    use super::*;
-    use std::sync::Arc;
-    use std::thread;
-
-    #[test]
-    fn test_memorydb_can_delete() {
-        let col_a: &str = BLOCKS_DB_COLUMN;
-
-        let db = MemoryDB::open();
-
-        db.put(col_a, "dogs".as_bytes(), "lol".as_bytes()).unwrap();
-
-        assert_eq!(
-            db.get(col_a, "dogs".as_bytes()).unwrap().unwrap(),
-            "lol".as_bytes()
-        );
-
-        db.delete(col_a, "dogs".as_bytes()).unwrap();
-
-        assert_eq!(db.get(col_a, "dogs".as_bytes()).unwrap(), None);
-    }
-
-    #[test]
-    fn test_memorydb_column_access() {
-        let col_a: &str = BLOCKS_DB_COLUMN;
-        let col_b: &str = VALIDATOR_DB_COLUMN;
-
-        let db = MemoryDB::open();
-
-        /*
-         * Testing that if we write to the same key in different columns that
-         * there is not an overlap.
-         */
-        db.put(col_a, "same".as_bytes(), "cat".as_bytes()).unwrap();
-        db.put(col_b, "same".as_bytes(), "dog".as_bytes()).unwrap();
-
-        assert_eq!(
-            db.get(col_a, "same".as_bytes()).unwrap().unwrap(),
-            "cat".as_bytes()
-        );
-        assert_eq!(
-            db.get(col_b, "same".as_bytes()).unwrap().unwrap(),
-            "dog".as_bytes()
-        );
-    }
-
-    #[test]
-    fn test_memorydb_unknown_column_access() {
-        let col_a: &str = BLOCKS_DB_COLUMN;
-        let col_x: &str = "ColumnX";
-
-        let db = MemoryDB::open();
-
-        /*
-         * Test that we get errors when using undeclared columns
-         */
-        assert!(db.put(col_a, "cats".as_bytes(), "lol".as_bytes()).is_ok());
-        assert!(db.put(col_x, "cats".as_bytes(), "lol".as_bytes()).is_err());
-
-        assert!(db.get(col_a, "cats".as_bytes()).is_ok());
-        assert!(db.get(col_x, "cats".as_bytes()).is_err());
-    }
-
-    #[test]
-    fn test_memorydb_exists() {
-        let col_a: &str = BLOCKS_DB_COLUMN;
-        let col_b: &str = VALIDATOR_DB_COLUMN;
-
-        let db = MemoryDB::open();
-
-        /*
-         * Testing that if we write to the same key in different columns that
-         * there is not an overlap.
-         */
-        db.put(col_a, "cats".as_bytes(), "lol".as_bytes()).unwrap();
-
-        assert_eq!(true, db.exists(col_a, "cats".as_bytes()).unwrap());
-        assert_eq!(false, db.exists(col_b, "cats".as_bytes()).unwrap());
-
-        assert_eq!(false, db.exists(col_a, "dogs".as_bytes()).unwrap());
-        assert_eq!(false, db.exists(col_b, "dogs".as_bytes()).unwrap());
-    }
-
-    #[test]
-    fn test_memorydb_threading() {
-        let col_name: &str = BLOCKS_DB_COLUMN;
-
-        let db = Arc::new(MemoryDB::open());
-
-        let thread_count = 10;
-        let write_count = 10;
-
-        // We're execting the product of these numbers to fit in one byte.
-        assert!(thread_count * write_count <= 255);
-
-        let mut handles = vec![];
-        for t in 0..thread_count {
-            let wc = write_count;
-            let db = db.clone();
-            let col = col_name.clone();
-            let handle = thread::spawn(move || {
-                for w in 0..wc {
-                    let key = (t * w) as u8;
-                    let val = 42;
-                    db.put(&col, &vec![key], &vec![val]).unwrap();
-                }
-            });
-            handles.push(handle);
-        }
-
-        for handle in handles {
-            handle.join().unwrap();
-        }
-
-        for t in 0..thread_count {
-            for w in 0..write_count {
-                let key = (t * w) as u8;
-                let val = db.get(&col_name, &vec![key]).unwrap().unwrap();
-                assert_eq!(vec![42], val);
-            }
-        }
-    }
-}

beacon_node/db/src/stores/beacon_block_store.rs

@@ -1,246 +0,0 @@
-use super::BLOCKS_DB_COLUMN as DB_COLUMN;
-use super::{ClientDB, DBError};
-use ssz::Decode;
-use std::sync::Arc;
-use types::{BeaconBlock, Hash256, Slot};
-
-#[derive(Clone, Debug, PartialEq)]
-pub enum BeaconBlockAtSlotError {
-    UnknownBeaconBlock(Hash256),
-    InvalidBeaconBlock(Hash256),
-    DBError(String),
-}
-
-pub struct BeaconBlockStore<T>
-where
-    T: ClientDB,
-{
-    db: Arc<T>,
-}
-
-// Implements `put`, `get`, `exists` and `delete` for the store.
-impl_crud_for_store!(BeaconBlockStore, DB_COLUMN);
-
-impl<T: ClientDB> BeaconBlockStore<T> {
-    pub fn new(db: Arc<T>) -> Self {
-        Self { db }
-    }
-
-    pub fn get_deserialized(&self, hash: &Hash256) -> Result<Option<BeaconBlock>, DBError> {
-        match self.get(&hash)? {
-            None => Ok(None),
-            Some(ssz) => {
-                let block = BeaconBlock::from_ssz_bytes(&ssz).map_err(|_| DBError {
-                    message: "Bad BeaconBlock SSZ.".to_string(),
-                })?;
-                Ok(Some(block))
-            }
-        }
-    }
-
-    /// Retrieve the block at a slot given a "head_hash" and a slot.
-    ///
-    /// A "head_hash" must be a block hash with a slot number greater than or equal to the desired
-    /// slot.
-    ///
-    /// This function will read each block down the chain until it finds a block with the given
-    /// slot number. If the slot is skipped, the function will return None.
-    ///
-    /// If a block is found, a tuple of (block_hash, serialized_block) is returned.
-    ///
-    /// Note: this function uses a loop instead of recursion as the compiler is over-strict when it
-    /// comes to recursion and the `impl Trait` pattern. See:
-    /// https://stackoverflow.com/questions/54032940/using-impl-trait-in-a-recursive-function
-    pub fn block_at_slot(
-        &self,
-        head_hash: &Hash256,
-        slot: Slot,
-    ) -> Result<Option<(Hash256, BeaconBlock)>, BeaconBlockAtSlotError> {
-        let mut current_hash = *head_hash;
-
-        loop {
-            if let Some(block) = self.get_deserialized(&current_hash)? {
-                if block.slot == slot {
-                    break Ok(Some((current_hash, block)));
-                } else if block.slot < slot {
-                    break Ok(None);
-                } else {
-                    current_hash = block.previous_block_root;
-                }
-            } else {
-                break Err(BeaconBlockAtSlotError::UnknownBeaconBlock(current_hash));
-            }
-        }
-    }
-}
-
-impl From<DBError> for BeaconBlockAtSlotError {
-    fn from(e: DBError) -> Self {
-        BeaconBlockAtSlotError::DBError(e.message)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::super::super::MemoryDB;
-    use super::*;
-
-    use std::sync::Arc;
-    use std::thread;
-
-    use ssz::ssz_encode;
-    use types::test_utils::{SeedableRng, TestRandom, XorShiftRng};
-    use types::BeaconBlock;
-    use types::Hash256;
-
-    test_crud_for_store!(BeaconBlockStore, DB_COLUMN);
-
-    #[test]
-    fn head_hash_slot_too_low() {
-        let db = Arc::new(MemoryDB::open());
-        let bs = Arc::new(BeaconBlockStore::new(db.clone()));
-        let mut rng = XorShiftRng::from_seed([42; 16]);
-
-        let mut block = BeaconBlock::random_for_test(&mut rng);
-        block.slot = Slot::from(10_u64);
-
-        let block_root = block.canonical_root();
-        bs.put(&block_root, &ssz_encode(&block)).unwrap();
-
-        let result = bs.block_at_slot(&block_root, Slot::from(11_u64)).unwrap();
-        assert_eq!(result, None);
-    }
-
-    #[test]
-    fn test_invalid_block_at_slot() {
-        let db = Arc::new(MemoryDB::open());
-        let store = BeaconBlockStore::new(db.clone());
-
-        let ssz = "definitly not a valid block".as_bytes();
-        let hash = &Hash256::from([0xAA; 32]);
-
-        db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap();
-        assert_eq!(
-            store.block_at_slot(hash, Slot::from(42_u64)),
-            Err(BeaconBlockAtSlotError::DBError(
-                "Bad BeaconBlock SSZ.".into()
-            ))
-        );
-    }
-
-    #[test]
-    fn test_unknown_block_at_slot() {
-        let db = Arc::new(MemoryDB::open());
-        let store = BeaconBlockStore::new(db.clone());
-
-        let ssz = "some bytes".as_bytes();
-        let hash = &Hash256::from([0xAA; 32]);
-        let other_hash = &Hash256::from([0xBB; 32]);
-
-        db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap();
-        assert_eq!(
-            store.block_at_slot(other_hash, Slot::from(42_u64)),
-            Err(BeaconBlockAtSlotError::UnknownBeaconBlock(*other_hash))
-        );
-    }
-
-    #[test]
-    fn test_block_store_on_memory_db() {
-        let db = Arc::new(MemoryDB::open());
-        let bs = Arc::new(BeaconBlockStore::new(db.clone()));
-
-        let thread_count = 10;
-        let write_count = 10;
-
-        let mut handles = vec![];
-        for t in 0..thread_count {
-            let wc = write_count;
-            let bs = bs.clone();
-            let handle = thread::spawn(move || {
-                for w in 0..wc {
-                    let key = t * w;
-                    let val = 42;
-                    bs.put(&Hash256::from_low_u64_le(key), &vec![val]).unwrap();
-                }
-            });
-            handles.push(handle);
-        }
-
-        for handle in handles {
-            handle.join().unwrap();
-        }
-
-        for t in 0..thread_count {
-            for w in 0..write_count {
-                let key = t * w;
-                assert!(bs.exists(&Hash256::from_low_u64_le(key)).unwrap());
-                let val = bs.get(&Hash256::from_low_u64_le(key)).unwrap().unwrap();
-                assert_eq!(vec![42], val);
-            }
-        }
-    }
-
-    #[test]
-    #[ignore]
-    fn test_block_at_slot() {
-        let db = Arc::new(MemoryDB::open());
-        let bs = Arc::new(BeaconBlockStore::new(db.clone()));
-        let mut rng = XorShiftRng::from_seed([42; 16]);
-
-        // Specify test block parameters.
-        let hashes = [
-            Hash256::from([0; 32]),
-            Hash256::from([1; 32]),
-            Hash256::from([2; 32]),
-            Hash256::from([3; 32]),
-            Hash256::from([4; 32]),
-        ];
-        let parent_hashes = [
-            Hash256::from([255; 32]), // Genesis block.
-            Hash256::from([0; 32]),
-            Hash256::from([1; 32]),
-            Hash256::from([2; 32]),
-            Hash256::from([3; 32]),
-        ];
-        let unknown_hash = Hash256::from([101; 32]); // different from all above
-        let slots: Vec<Slot> = vec![0, 1, 3, 4, 5].iter().map(|x| Slot::new(*x)).collect();
-
-        // Generate a vec of random blocks and store them in the DB.
-        let block_count = 5;
-        let mut blocks: Vec<BeaconBlock> = Vec::with_capacity(5);
-        for i in 0..block_count {
-            let mut block = BeaconBlock::random_for_test(&mut rng);
-
-            block.previous_block_root = parent_hashes[i];
-            block.slot = slots[i];
-
-            let ssz = ssz_encode(&block);
-            db.put(DB_COLUMN, hashes[i].as_bytes(), &ssz).unwrap();
-
-            blocks.push(block);
-        }
-
-        // Test that certain slots can be reached from certain hashes.
-        let test_cases = vec![(4, 4), (4, 3), (4, 2), (4, 1), (4, 0)];
-        for (hashes_index, slot_index) in test_cases {
-            let (matched_block_hash, block) = bs
-                .block_at_slot(&hashes[hashes_index], slots[slot_index])
-                .unwrap()
-                .unwrap();
-            assert_eq!(matched_block_hash, hashes[slot_index]);
-            assert_eq!(block.slot, slots[slot_index]);
-        }
-
-        let ssz = bs.block_at_slot(&hashes[4], Slot::new(2)).unwrap();
-        assert_eq!(ssz, None);
-
-        let ssz = bs.block_at_slot(&hashes[4], Slot::new(6)).unwrap();
-        assert_eq!(ssz, None);
-
-        let ssz = bs.block_at_slot(&unknown_hash, Slot::new(2));
-        assert_eq!(
-            ssz,
-            Err(BeaconBlockAtSlotError::UnknownBeaconBlock(unknown_hash))
-        );
-    }
-}

beacon_node/db/src/stores/beacon_state_store.rs

@@ -1,65 +0,0 @@
-use super::STATES_DB_COLUMN as DB_COLUMN;
-use super::{ClientDB, DBError};
-use ssz::Decode;
-use std::sync::Arc;
-use types::{BeaconState, EthSpec, Hash256};
-
-pub struct BeaconStateStore<T>
-where
-    T: ClientDB,
-{
-    db: Arc<T>,
-}
-
-// Implements `put`, `get`, `exists` and `delete` for the store.
-impl_crud_for_store!(BeaconStateStore, DB_COLUMN);
-
-impl<T: ClientDB> BeaconStateStore<T> {
-    pub fn new(db: Arc<T>) -> Self {
-        Self { db }
-    }
-
-    pub fn get_deserialized<E: EthSpec>(
-        &self,
-        hash: &Hash256,
-    ) -> Result<Option<BeaconState<E>>, DBError> {
-        match self.get(&hash)? {
-            None => Ok(None),
-            Some(ssz) => {
-                let state = BeaconState::from_ssz_bytes(&ssz).map_err(|_| DBError {
-                    message: "Bad State SSZ.".to_string(),
-                })?;
-                Ok(Some(state))
-            }
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::super::super::MemoryDB;
-    use super::*;
-
-    use ssz::ssz_encode;
-    use std::sync::Arc;
-    use types::test_utils::{SeedableRng, TestRandom, XorShiftRng};
-    use types::{FoundationBeaconState, Hash256};
-
-    test_crud_for_store!(BeaconStateStore, DB_COLUMN);
-
-    #[test]
-    fn test_reader() {
-        let db = Arc::new(MemoryDB::open());
-        let store = BeaconStateStore::new(db.clone());
-
-        let mut rng = XorShiftRng::from_seed([42; 16]);
-        let state: FoundationBeaconState = BeaconState::random_for_test(&mut rng);
-        let state_root = state.canonical_root();
-
-        store.put(&state_root, &ssz_encode(&state)).unwrap();
-
-        let decoded = store.get_deserialized(&state_root).unwrap().unwrap();
-
-        assert_eq!(state, decoded);
-    }
-}

beacon_node/db/src/stores/macros.rs

@@ -1,103 +0,0 @@
-macro_rules! impl_crud_for_store {
-    ($store: ident, $db_column: expr) => {
-        impl<T: ClientDB> $store<T> {
-            pub fn put(&self, hash: &Hash256, ssz: &[u8]) -> Result<(), DBError> {
-                self.db.put($db_column, hash.as_bytes(), ssz)
-            }
-
-            pub fn get(&self, hash: &Hash256) -> Result<Option<Vec<u8>>, DBError> {
-                self.db.get($db_column, hash.as_bytes())
-            }
-
-            pub fn exists(&self, hash: &Hash256) -> Result<bool, DBError> {
-                self.db.exists($db_column, hash.as_bytes())
-            }
-
-            pub fn delete(&self, hash: &Hash256) -> Result<(), DBError> {
-                self.db.delete($db_column, hash.as_bytes())
-            }
-        }
-    };
-}
-
-#[cfg(test)]
-macro_rules! test_crud_for_store {
-    ($store: ident, $db_column: expr) => {
-        #[test]
-        fn test_put() {
-            let db = Arc::new(MemoryDB::open());
-            let store = $store::new(db.clone());
-
-            let ssz = "some bytes".as_bytes();
-            let hash = &Hash256::from([0xAA; 32]);
-
-            store.put(hash, ssz).unwrap();
-            assert_eq!(db.get(DB_COLUMN, hash.as_bytes()).unwrap().unwrap(), ssz);
-        }
-
-        #[test]
-        fn test_get() {
-            let db = Arc::new(MemoryDB::open());
-            let store = $store::new(db.clone());
-
-            let ssz = "some bytes".as_bytes();
-            let hash = &Hash256::from([0xAA; 32]);
-
-            db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap();
-            assert_eq!(store.get(hash).unwrap().unwrap(), ssz);
-        }
-
-        #[test]
-        fn test_get_unknown() {
-            let db = Arc::new(MemoryDB::open());
-            let store = $store::new(db.clone());
-
-            let ssz = "some bytes".as_bytes();
-            let hash = &Hash256::from([0xAA; 32]);
-            let other_hash = &Hash256::from([0xBB; 32]);
-
-            db.put(DB_COLUMN, other_hash.as_bytes(), ssz).unwrap();
-            assert_eq!(store.get(hash).unwrap(), None);
-        }
-
-        #[test]
-        fn test_exists() {
-            let db = Arc::new(MemoryDB::open());
-            let store = $store::new(db.clone());
-
-            let ssz = "some bytes".as_bytes();
-            let hash = &Hash256::from([0xAA; 32]);
-
-            db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap();
-            assert!(store.exists(hash).unwrap());
-        }
-
-        #[test]
-        fn test_block_does_not_exist() {
-            let db = Arc::new(MemoryDB::open());
-            let store = $store::new(db.clone());
-
-            let ssz = "some bytes".as_bytes();
-            let hash = &Hash256::from([0xAA; 32]);
-            let other_hash = &Hash256::from([0xBB; 32]);
-
-            db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap();
-            assert!(!store.exists(other_hash).unwrap());
-        }
-
-        #[test]
-        fn test_delete() {
-            let db = Arc::new(MemoryDB::open());
-            let store = $store::new(db.clone());
-
-            let ssz = "some bytes".as_bytes();
-            let hash = &Hash256::from([0xAA; 32]);
-
-            db.put(DB_COLUMN, hash.as_bytes(), ssz).unwrap();
-            assert!(db.exists(DB_COLUMN, hash.as_bytes()).unwrap());
-
-            store.delete(hash).unwrap();
-            assert!(!db.exists(DB_COLUMN, hash.as_bytes()).unwrap());
-        }
-    };
-}

beacon_node/db/src/stores/mod.rs

@@ -1,25 +0,0 @@
-use super::{ClientDB, DBError};
-
-#[macro_use]
-mod macros;
-mod beacon_block_store;
-mod beacon_state_store;
-mod pow_chain_store;
-mod validator_store;
-
-pub use self::beacon_block_store::{BeaconBlockAtSlotError, BeaconBlockStore};
-pub use self::beacon_state_store::BeaconStateStore;
-pub use self::pow_chain_store::PoWChainStore;
-pub use self::validator_store::{ValidatorStore, ValidatorStoreError};
-
-pub const BLOCKS_DB_COLUMN: &str = "blocks";
-pub const STATES_DB_COLUMN: &str = "states";
-pub const POW_CHAIN_DB_COLUMN: &str = "powchain";
-pub const VALIDATOR_DB_COLUMN: &str = "validator";
-
-pub const COLUMNS: [&str; 4] = [
-    BLOCKS_DB_COLUMN,
-    STATES_DB_COLUMN,
-    POW_CHAIN_DB_COLUMN,
-    VALIDATOR_DB_COLUMN,
-];

beacon_node/db/src/stores/pow_chain_store.rs

@@ -1,68 +0,0 @@
-use super::POW_CHAIN_DB_COLUMN as DB_COLUMN;
-use super::{ClientDB, DBError};
-use std::sync::Arc;
-
-pub struct PoWChainStore<T>
-where
-    T: ClientDB,
-{
-    db: Arc<T>,
-}
-
-impl<T: ClientDB> PoWChainStore<T> {
-    pub fn new(db: Arc<T>) -> Self {
-        Self { db }
-    }
-
-    pub fn put_block_hash(&self, hash: &[u8]) -> Result<(), DBError> {
-        self.db.put(DB_COLUMN, hash, &[0])
-    }
-
-    pub fn block_hash_exists(&self, hash: &[u8]) -> Result<bool, DBError> {
-        self.db.exists(DB_COLUMN, hash)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    extern crate types;
-
-    use super::super::super::MemoryDB;
-    use super::*;
-
-    use self::types::Hash256;
-
-    #[test]
-    fn test_put_block_hash() {
-        let db = Arc::new(MemoryDB::open());
-        let store = PoWChainStore::new(db.clone());
-
-        let hash = &Hash256::from([0xAA; 32]).as_bytes().to_vec();
-        store.put_block_hash(hash).unwrap();
-
-        assert!(db.exists(DB_COLUMN, hash).unwrap());
-    }
-
-    #[test]
-    fn test_block_hash_exists() {
-        let db = Arc::new(MemoryDB::open());
-        let store = PoWChainStore::new(db.clone());
-
-        let hash = &Hash256::from([0xAA; 32]).as_bytes().to_vec();
-        db.put(DB_COLUMN, hash, &[0]).unwrap();
-
-        assert!(store.block_hash_exists(hash).unwrap());
-    }
-
-    #[test]
-    fn test_block_hash_does_not_exist() {
-        let db = Arc::new(MemoryDB::open());
-        let store = PoWChainStore::new(db.clone());
-
-        let hash = &Hash256::from([0xAA; 32]).as_bytes().to_vec();
-        let other_hash = &Hash256::from([0xBB; 32]).as_bytes().to_vec();
-        db.put(DB_COLUMN, hash, &[0]).unwrap();
-
-        assert!(!store.block_hash_exists(other_hash).unwrap());
-    }
-}

beacon_node/db/src/stores/validator_store.rs

@@ -1,215 +0,0 @@
-extern crate bytes;
-
-use self::bytes::{BufMut, BytesMut};
-use super::VALIDATOR_DB_COLUMN as DB_COLUMN;
-use super::{ClientDB, DBError};
-use bls::PublicKey;
-use ssz::{Decode, Encode};
-use std::sync::Arc;
-
-#[derive(Debug, PartialEq)]
-pub enum ValidatorStoreError {
-    DBError(String),
-    DecodeError,
-}
-
-impl From<DBError> for ValidatorStoreError {
-    fn from(error: DBError) -> Self {
-        ValidatorStoreError::DBError(error.message)
-    }
-}
-
-#[derive(Debug, PartialEq)]
-enum KeyPrefixes {
-    PublicKey,
-}
-
-pub struct ValidatorStore<T>
-where
-    T: ClientDB,
-{
-    db: Arc<T>,
-}
-
-impl<T: ClientDB> ValidatorStore<T> {
-    pub fn new(db: Arc<T>) -> Self {
-        Self { db }
-    }
-
-    fn prefix_bytes(&self, key_prefix: &KeyPrefixes) -> Vec<u8> {
-        match key_prefix {
-            KeyPrefixes::PublicKey => b"pubkey".to_vec(),
-        }
-    }
-
-    fn get_db_key_for_index(&self, key_prefix: &KeyPrefixes, index: usize) -> Vec<u8> {
-        let mut buf = BytesMut::with_capacity(6 + 8);
-        buf.put(self.prefix_bytes(key_prefix));
-        buf.put_u64_be(index as u64);
-        buf.take().to_vec()
-    }
-
-    pub fn put_public_key_by_index(
-        &self,
-        index: usize,
-        public_key: &PublicKey,
-    ) -> Result<(), ValidatorStoreError> {
-        let key = self.get_db_key_for_index(&KeyPrefixes::PublicKey, index);
-        let val = public_key.as_ssz_bytes();
-        self.db
-            .put(DB_COLUMN, &key[..], &val[..])
-            .map_err(ValidatorStoreError::from)
-    }
-
-    pub fn get_public_key_by_index(
-        &self,
-        index: usize,
-    ) -> Result<Option<PublicKey>, ValidatorStoreError> {
-        let key = self.get_db_key_for_index(&KeyPrefixes::PublicKey, index);
-        let val = self.db.get(DB_COLUMN, &key[..])?;
-        match val {
-            None => Ok(None),
-            Some(val) => match PublicKey::from_ssz_bytes(&val) {
-                Ok(key) => Ok(Some(key)),
-                Err(_) => Err(ValidatorStoreError::DecodeError),
-            },
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::super::super::MemoryDB;
-    use super::*;
-    use bls::Keypair;
-
-    #[test]
-    fn test_prefix_bytes() {
-        let db = Arc::new(MemoryDB::open());
-        let store = ValidatorStore::new(db.clone());
-
-        assert_eq!(
-            store.prefix_bytes(&KeyPrefixes::PublicKey),
-            b"pubkey".to_vec()
-        );
-    }
-
-    #[test]
-    fn test_get_db_key_for_index() {
-        let db = Arc::new(MemoryDB::open());
-        let store = ValidatorStore::new(db.clone());
-
-        let mut buf = BytesMut::with_capacity(6 + 8);
-        buf.put(b"pubkey".to_vec());
-        buf.put_u64_be(42);
-        assert_eq!(
-            store.get_db_key_for_index(&KeyPrefixes::PublicKey, 42),
-            buf.take().to_vec()
-        )
-    }
-
-    #[test]
-    fn test_put_public_key_by_index() {
-        let db = Arc::new(MemoryDB::open());
-        let store = ValidatorStore::new(db.clone());
-
-        let index = 3;
-        let public_key = Keypair::random().pk;
-
-        store.put_public_key_by_index(index, &public_key).unwrap();
-        let public_key_at_index = db
-            .get(
-                DB_COLUMN,
-                &store.get_db_key_for_index(&KeyPrefixes::PublicKey, index)[..],
-            )
-            .unwrap()
-            .unwrap();
-
-        assert_eq!(public_key_at_index, public_key.as_ssz_bytes());
-    }
-
-    #[test]
-    fn test_get_public_key_by_index() {
-        let db = Arc::new(MemoryDB::open());
-        let store = ValidatorStore::new(db.clone());
-
-        let index = 4;
-        let public_key = Keypair::random().pk;
-
-        db.put(
-            DB_COLUMN,
-            &store.get_db_key_for_index(&KeyPrefixes::PublicKey, index)[..],
-            &public_key.as_ssz_bytes(),
-        )
-        .unwrap();
-
-        let public_key_at_index = store.get_public_key_by_index(index).unwrap().unwrap();
-        assert_eq!(public_key_at_index, public_key);
-    }
-
-    #[test]
-    fn test_get_public_key_by_unknown_index() {
-        let db = Arc::new(MemoryDB::open());
-        let store = ValidatorStore::new(db.clone());
-
-        let public_key = Keypair::random().pk;
-
-        db.put(
-            DB_COLUMN,
-            &store.get_db_key_for_index(&KeyPrefixes::PublicKey, 3)[..],
-            &public_key.as_ssz_bytes(),
-        )
-        .unwrap();
-
-        let public_key_at_index = store.get_public_key_by_index(4).unwrap();
-        assert_eq!(public_key_at_index, None);
-    }
-
-    #[test]
-    fn test_get_invalid_public_key() {
-        let db = Arc::new(MemoryDB::open());
-        let store = ValidatorStore::new(db.clone());
-
-        let key = store.get_db_key_for_index(&KeyPrefixes::PublicKey, 42);
-        db.put(DB_COLUMN, &key[..], "cats".as_bytes()).unwrap();
-
-        assert_eq!(
-            store.get_public_key_by_index(42),
-            Err(ValidatorStoreError::DecodeError)
-        );
-    }
-
-    #[test]
-    fn test_validator_store_put_get() {
-        let db = Arc::new(MemoryDB::open());
-        let store = ValidatorStore::new(db);
-
-        let keys = vec![
-            Keypair::random(),
-            Keypair::random(),
-            Keypair::random(),
-            Keypair::random(),
-            Keypair::random(),
-        ];
-
-        for i in 0..keys.len() {
-            store.put_public_key_by_index(i, &keys[i].pk).unwrap();
-        }
-
-        /*
-         * Check all keys are retrieved correctly.
-         */
-        for i in 0..keys.len() {
-            let retrieved = store.get_public_key_by_index(i).unwrap().unwrap();
-            assert_eq!(retrieved, keys[i].pk);
-        }
-
-        /*
-         * Check that an index that wasn't stored returns None.
-         */
-        assert!(store
-            .get_public_key_by_index(keys.len() + 1)
-            .unwrap()
-            .is_none());
-    }
-}

beacon_node/db/src/traits.rs

@@ -1,28 +0,0 @@
-pub type DBValue = Vec<u8>;
-
-#[derive(Debug)]
-pub struct DBError {
-    pub message: String,
-}
-
-impl DBError {
-    pub fn new(message: String) -> Self {
-        Self { message }
-    }
-}
-
-/// A generic database to be used by the "client' (i.e.,
-/// the lighthouse blockchain client).
-///
-/// The purpose of having this generic trait is to allow the
-/// program to use a persistent on-disk database during production,
-/// but use a transient database during tests.
-pub trait ClientDB: Sync + Send {
-    fn get(&self, col: &str, key: &[u8]) -> Result<Option<DBValue>, DBError>;
-
-    fn put(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), DBError>;
-
-    fn exists(&self, col: &str, key: &[u8]) -> Result<bool, DBError>;
-
-    fn delete(&self, col: &str, key: &[u8]) -> Result<(), DBError>;
-}

beacon_node/http_server/Cargo.toml

@@ -0,0 +1,36 @@
+[package]
+name = "http_server"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+[dependencies]
+bls = { path = "../../eth2/utils/bls" }
+beacon_chain = { path = "../beacon_chain" }
+iron = "^0.6"
+router = "^0.6"
+network = { path = "../network" }
+eth2-libp2p = { path = "../eth2-libp2p" }
+version = { path = "../version" }
+types = { path = "../../eth2/types" }
+ssz = { path = "../../eth2/utils/ssz" }
+slot_clock = { path = "../../eth2/utils/slot_clock" }
+protos = { path = "../../protos" }
+fork_choice = { path = "../../eth2/fork_choice" }
+grpcio = { version = "0.4", default-features = false, features = ["protobuf-codec"] }
+persistent = "^0.4"
+protobuf = "2.0.2"
+prometheus = "^0.6"
+clap = "2.32.0"
+store = { path = "../store" }
+dirs = "1.0.3"
+futures = "0.1.23"
+serde = "1.0"
+serde_derive = "1.0"
+serde_json = "1.0"
+slog = "^2.2.3"
+slog-term = "^2.4.0"
+slog-async = "^2.3.0"
+tokio = "0.1.17"
+exit-future = "0.1.4"
+crossbeam-channel = "0.3.8"

beacon_node/http_server/src/api.rs

@@ -0,0 +1,71 @@
+use crate::{key::BeaconChainKey, map_persistent_err_to_500};
+use beacon_chain::{BeaconChain, BeaconChainTypes};
+use iron::prelude::*;
+use iron::{
+    headers::{CacheControl, CacheDirective, ContentType},
+    status::Status,
+    AfterMiddleware, Handler, IronResult, Request, Response,
+};
+use persistent::Read;
+use router::Router;
+use serde_json::json;
+use std::sync::Arc;
+
+/// Yields a handler for the HTTP API.
+pub fn build_handler<T: BeaconChainTypes + 'static>(
+    beacon_chain: Arc<BeaconChain<T>>,
+) -> impl Handler {
+    let mut router = Router::new();
+
+    router.get("/node/fork", handle_fork::<T>, "fork");
+
+    let mut chain = Chain::new(router);
+
+    // Insert `BeaconChain` so it may be accessed in a request.
+    chain.link(Read::<BeaconChainKey<T>>::both(beacon_chain.clone()));
+    // Set the content-type headers.
+    chain.link_after(SetJsonContentType);
+    // Set the cache headers.
+    chain.link_after(SetCacheDirectives);
+
+    chain
+}
+
+/// Sets the `cache-control` headers on _all_ responses, unless they are already set.
+struct SetCacheDirectives;
+impl AfterMiddleware for SetCacheDirectives {
+    fn after(&self, _req: &mut Request, mut resp: Response) -> IronResult<Response> {
+        // This is run for every requests, AFTER all handlers have been executed
+        if resp.headers.get::<CacheControl>() == None {
+            resp.headers.set(CacheControl(vec![
+                CacheDirective::NoCache,
+                CacheDirective::NoStore,
+            ]));
+        }
+        Ok(resp)
+    }
+}
+
+/// Sets the `content-type` headers on _all_ responses, unless they are already set.
+struct SetJsonContentType;
+impl AfterMiddleware for SetJsonContentType {
+    fn after(&self, _req: &mut Request, mut resp: Response) -> IronResult<Response> {
+        if resp.headers.get::<ContentType>() == None {
+            resp.headers.set(ContentType::json());
+        }
+        Ok(resp)
+    }
+}
+
+fn handle_fork<T: BeaconChainTypes + 'static>(req: &mut Request) -> IronResult<Response> {
+    let beacon_chain = req
+        .get::<Read<BeaconChainKey<T>>>()
+        .map_err(map_persistent_err_to_500)?;
+
+    let response = json!({
+        "fork": beacon_chain.head().beacon_state.fork,
+        "chain_id": beacon_chain.spec.chain_id
+    });
+
+    Ok(Response::with((Status::Ok, response.to_string())))
+}

beacon_node/http_server/src/key.rs

@@ -0,0 +1,12 @@
+use beacon_chain::{BeaconChain, BeaconChainTypes};
+use iron::typemap::Key;
+use std::marker::PhantomData;
+use std::sync::Arc;
+
+pub struct BeaconChainKey<T> {
+    _phantom: PhantomData<T>,
+}
+
+impl<T: BeaconChainTypes + 'static> Key for BeaconChainKey<T> {
+    type Value = Arc<BeaconChain<T>>;
+}

beacon_node/http_server/src/lib.rs

@@ -0,0 +1,118 @@
+mod api;
+mod key;
+mod metrics;
+
+use beacon_chain::{BeaconChain, BeaconChainTypes};
+use futures::Future;
+use iron::prelude::*;
+use network::NetworkMessage;
+use router::Router;
+use slog::{info, o, warn};
+use std::sync::Arc;
+use tokio::runtime::TaskExecutor;
+
+#[derive(PartialEq, Clone, Debug)]
+pub struct HttpServerConfig {
+    pub enabled: bool,
+    pub listen_address: String,
+}
+
+impl Default for HttpServerConfig {
+    fn default() -> Self {
+        Self {
+            enabled: false,
+            listen_address: "127.0.0.1:5051".to_string(),
+        }
+    }
+}
+
+/// Build the `iron` HTTP server, defining the core routes.
+pub fn create_iron_http_server<T: BeaconChainTypes + 'static>(
+    beacon_chain: Arc<BeaconChain<T>>,
+) -> Iron<Router> {
+    let mut router = Router::new();
+
+    // A `GET` request to `/metrics` is handled by the `metrics` module.
+    router.get(
+        "/metrics",
+        metrics::build_handler(beacon_chain.clone()),
+        "metrics",
+    );
+
+    // Any request to all other endpoints is handled by the `api` module.
+    router.any("/*", api::build_handler(beacon_chain.clone()), "api");
+
+    Iron::new(router)
+}
+
+/// Start the HTTP service on the tokio `TaskExecutor`.
+pub fn start_service<T: BeaconChainTypes + 'static>(
+    config: &HttpServerConfig,
+    executor: &TaskExecutor,
+    _network_chan: crossbeam_channel::Sender<NetworkMessage>,
+    beacon_chain: Arc<BeaconChain<T>>,
+    log: &slog::Logger,
+) -> exit_future::Signal {
+    let log = log.new(o!("Service"=>"HTTP"));
+
+    // Create:
+    //  - `shutdown_trigger` a one-shot to shut down this service.
+    //  - `wait_for_shutdown` a future that will wait until someone calls shutdown.
+    let (shutdown_trigger, wait_for_shutdown) = exit_future::signal();
+
+    // Create an `iron` http, without starting it yet.
+    let iron = create_iron_http_server(beacon_chain);
+
+    // Create a HTTP server future.
+    //
+    // 1. Start the HTTP server
+    // 2. Build an exit future that will shutdown the server when requested.
+    // 3. Return the exit future, so the caller may shutdown the service when desired.
+    let http_service = {
+        // Start the HTTP server
+        let server_start_result = iron.http(config.listen_address.clone());
+
+        if server_start_result.is_ok() {
+            info!(log, "HTTP server running on {}", config.listen_address);
+        } else {
+            warn!(
+                log,
+                "HTTP server failed to start on {}", config.listen_address
+            );
+        }
+
+        // Build a future that will shutdown the HTTP server when the `shutdown_trigger` is
+        // triggered.
+        wait_for_shutdown.and_then(move |_| {
+            info!(log, "HTTP server shutting down");
+
+            if let Ok(mut server) = server_start_result {
+                // According to the documentation, `server.close()` "doesn't work" and the server
+                // keeps listening.
+                //
+                // It is being called anyway, because it seems like the right thing to do. If you
+                // know this has negative side-effects, please create an issue to discuss.
+                //
+                // See: https://docs.rs/iron/0.6.0/iron/struct.Listening.html#impl
+                match server.close() {
+                    _ => (),
+                };
+            }
+            info!(log, "HTTP server shutdown complete.");
+            Ok(())
+        })
+    };
+
+    // Attach the HTTP server to the executor.
+    executor.spawn(http_service);
+
+    shutdown_trigger
+}
+
+/// Helper function for mapping a failure to read state to a 500 server error.
+fn map_persistent_err_to_500(e: persistent::PersistentError) -> iron::error::IronError {
+    iron::error::IronError {
+        error: Box::new(e),
+        response: iron::Response::with(iron::status::Status::InternalServerError),
+    }
+}

beacon_node/http_server/src/metrics.rs

@@ -0,0 +1,60 @@
+use crate::{key::BeaconChainKey, map_persistent_err_to_500};
+use beacon_chain::{BeaconChain, BeaconChainTypes};
+use iron::prelude::*;
+use iron::{status::Status, Handler, IronResult, Request, Response};
+use persistent::Read;
+use prometheus::{Encoder, IntCounter, Opts, Registry, TextEncoder};
+use slot_clock::SlotClock;
+use std::sync::Arc;
+use types::Slot;
+
+/// Yields a handler for the metrics endpoint.
+pub fn build_handler<T: BeaconChainTypes + 'static>(
+    beacon_chain: Arc<BeaconChain<T>>,
+) -> impl Handler {
+    let mut chain = Chain::new(handle_metrics::<T>);
+
+    chain.link(Read::<BeaconChainKey<T>>::both(beacon_chain));
+
+    chain
+}
+
+/// Handle a request for Prometheus metrics.
+///
+/// Returns a text string containing all metrics.
+fn handle_metrics<T: BeaconChainTypes + 'static>(req: &mut Request) -> IronResult<Response> {
+    let beacon_chain = req
+        .get::<Read<BeaconChainKey<T>>>()
+        .map_err(map_persistent_err_to_500)?;
+
+    let r = Registry::new();
+
+    let present_slot = if let Ok(Some(slot)) = beacon_chain.slot_clock.present_slot() {
+        slot
+    } else {
+        Slot::new(0)
+    };
+    register_and_set_slot(
+        &r,
+        "present_slot",
+        "direct_slock_clock_reading",
+        present_slot,
+    );
+
+    // Gather the metrics.
+    let mut buffer = vec![];
+    let encoder = TextEncoder::new();
+    let metric_families = r.gather();
+    encoder.encode(&metric_families, &mut buffer).unwrap();
+
+    let prom_string = String::from_utf8(buffer).unwrap();
+
+    Ok(Response::with((Status::Ok, prom_string)))
+}
+
+fn register_and_set_slot(registry: &Registry, name: &str, help: &str, slot: Slot) {
+    let counter_opts = Opts::new(name, help);
+    let counter = IntCounter::with_opts(counter_opts).unwrap();
+    registry.register(Box::new(counter.clone())).unwrap();
+    counter.inc_by(slot.as_u64() as i64);
+}

beacon_node/network/src/beacon_chain.rs

@@ -1,9 +1,6 @@
 use beacon_chain::BeaconChain as RawBeaconChain;
 use beacon_chain::{
-    db::ClientDB,
-    fork_choice::ForkChoice,
     parking_lot::RwLockReadGuard,
-    slot_clock::SlotClock,
     types::{BeaconState, ChainSpec},
     AttestationValidationError, CheckPoint,
 };
@@ -12,17 +9,17 @@ use types::{
     Attestation, BeaconBlock, BeaconBlockBody, BeaconBlockHeader, Epoch, EthSpec, Hash256, Slot,
 };
 
-pub use beacon_chain::{BeaconChainError, BlockProcessingOutcome, InvalidBlock};
+pub use beacon_chain::{BeaconChainError, BeaconChainTypes, BlockProcessingOutcome, InvalidBlock};
 
 /// The network's API to the beacon chain.
-pub trait BeaconChain<E: EthSpec>: Send + Sync {
+pub trait BeaconChain<T: BeaconChainTypes>: Send + Sync {
     fn get_spec(&self) -> &ChainSpec;
 
-    fn get_state(&self) -> RwLockReadGuard<BeaconState<E>>;
+    fn get_state(&self) -> RwLockReadGuard<BeaconState<T::EthSpec>>;
 
     fn slot(&self) -> Slot;
 
-    fn head(&self) -> RwLockReadGuard<CheckPoint<E>>;
+    fn head(&self) -> RwLockReadGuard<CheckPoint<T::EthSpec>>;
 
     fn get_block(&self, block_root: &Hash256) -> Result<Option<BeaconBlock>, BeaconChainError>;
 
@@ -30,7 +27,7 @@ pub trait BeaconChain<E: EthSpec>: Send + Sync {
 
     fn best_block_root(&self) -> Hash256;
 
-    fn finalized_head(&self) -> RwLockReadGuard<CheckPoint<E>>;
+    fn finalized_head(&self) -> RwLockReadGuard<CheckPoint<T::EthSpec>>;
 
     fn finalized_epoch(&self) -> Epoch;
 
@@ -64,18 +61,12 @@ pub trait BeaconChain<E: EthSpec>: Send + Sync {
     fn is_new_block_root(&self, beacon_block_root: &Hash256) -> Result<bool, BeaconChainError>;
 }
 
-impl<T, U, F, E> BeaconChain<E> for RawBeaconChain<T, U, F, E>
+impl<T: BeaconChainTypes> BeaconChain<T> for RawBeaconChain<T> {
-where
-    T: ClientDB + Sized,
-    U: SlotClock,
-    F: ForkChoice,
-    E: EthSpec,
-{
     fn get_spec(&self) -> &ChainSpec {
         &self.spec
     }
 
-    fn get_state(&self) -> RwLockReadGuard<BeaconState<E>> {
+    fn get_state(&self) -> RwLockReadGuard<BeaconState<T::EthSpec>> {
         self.state.read()
     }
 
@@ -83,7 +74,7 @@ where
         self.get_state().slot
     }
 
-    fn head(&self) -> RwLockReadGuard<CheckPoint<E>> {
+    fn head(&self) -> RwLockReadGuard<CheckPoint<T::EthSpec>> {
         self.head()
     }
 
@@ -95,7 +86,7 @@ where
         self.get_state().finalized_epoch
     }
 
-    fn finalized_head(&self) -> RwLockReadGuard<CheckPoint<E>> {
+    fn finalized_head(&self) -> RwLockReadGuard<CheckPoint<T::EthSpec>> {
         self.finalized_head()
     }
 

beacon_node/network/src/message_handler.rs

@@ -1,4 +1,4 @@
-use crate::beacon_chain::BeaconChain;
+use crate::beacon_chain::{BeaconChain, BeaconChainTypes};
 use crate::error;
 use crate::service::{NetworkMessage, OutgoingMessage};
 use crate::sync::SimpleSync;
@@ -13,7 +13,6 @@ use slog::{debug, warn};
 use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::Instant;
-use types::EthSpec;
 
 /// Timeout for RPC requests.
 // const REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
@@ -21,11 +20,11 @@ use types::EthSpec;
 // const HELLO_TIMEOUT: Duration = Duration::from_secs(30);
 
 /// Handles messages received from the network and client and organises syncing.
-pub struct MessageHandler<E: EthSpec> {
+pub struct MessageHandler<T: BeaconChainTypes> {
     /// Currently loaded and initialised beacon chain.
-    _chain: Arc<BeaconChain<E>>,
+    _chain: Arc<BeaconChain<T>>,
     /// The syncing framework.
-    sync: SimpleSync<E>,
+    sync: SimpleSync<T>,
     /// The context required to send messages to, and process messages from peers.
     network_context: NetworkContext,
     /// The `MessageHandler` logger.
@@ -45,10 +44,10 @@ pub enum HandlerMessage {
     PubsubMessage(PeerId, Box<PubsubMessage>),
 }
 
-impl<E: EthSpec> MessageHandler<E> {
+impl<T: BeaconChainTypes + 'static> MessageHandler<T> {
     /// Initializes and runs the MessageHandler.
     pub fn spawn(
-        beacon_chain: Arc<BeaconChain<E>>,
+        beacon_chain: Arc<BeaconChain<T>>,
         network_send: crossbeam_channel::Sender<NetworkMessage>,
         executor: &tokio::runtime::TaskExecutor,
         log: slog::Logger,

beacon_node/network/src/service.rs

@@ -1,4 +1,4 @@
-use crate::beacon_chain::BeaconChain;
+use crate::beacon_chain::{BeaconChain, BeaconChainTypes};
 use crate::error;
 use crate::message_handler::{HandlerMessage, MessageHandler};
 use crate::NetworkConfig;
@@ -13,20 +13,20 @@ use slog::{debug, info, o, trace};
 use std::marker::PhantomData;
 use std::sync::Arc;
 use tokio::runtime::TaskExecutor;
-use types::{EthSpec, Topic};
+use types::Topic;
 
 /// Service that handles communication between internal services and the eth2_libp2p network service.
-pub struct Service<E: EthSpec> {
+pub struct Service<T: BeaconChainTypes> {
     //libp2p_service: Arc<Mutex<LibP2PService>>,
     _libp2p_exit: oneshot::Sender<()>,
     network_send: crossbeam_channel::Sender<NetworkMessage>,
-    _phantom: PhantomData<E>, //message_handler: MessageHandler,
+    _phantom: PhantomData<T>, //message_handler: MessageHandler,
                               //message_handler_send: Sender<HandlerMessage>
 }
 
-impl<E: EthSpec> Service<E> {
+impl<T: BeaconChainTypes + 'static> Service<T> {
     pub fn new(
-        beacon_chain: Arc<BeaconChain<E>>,
+        beacon_chain: Arc<BeaconChain<T>>,
         config: &NetworkConfig,
         executor: &TaskExecutor,
         log: slog::Logger,

beacon_node/network/src/sync/import_queue.rs

@@ -1,11 +1,11 @@
-use crate::beacon_chain::BeaconChain;
+use crate::beacon_chain::{BeaconChain, BeaconChainTypes};
 use eth2_libp2p::rpc::methods::*;
 use eth2_libp2p::PeerId;
 use slog::{debug, error};
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 use tree_hash::TreeHash;
-use types::{BeaconBlock, BeaconBlockBody, BeaconBlockHeader, EthSpec, Hash256, Slot};
+use types::{BeaconBlock, BeaconBlockBody, BeaconBlockHeader, Hash256, Slot};
 
 /// Provides a queue for fully and partially built `BeaconBlock`s.
 ///
@@ -19,8 +19,8 @@ use types::{BeaconBlock, BeaconBlockBody, BeaconBlockHeader, EthSpec, Hash256, S
 /// `BeaconBlockBody` as the key.
 /// - It is possible for multiple distinct blocks to have identical `BeaconBlockBodies`. Therefore
 /// we cannot use a `HashMap` keyed by the root of `BeaconBlockBody`.
-pub struct ImportQueue<E: EthSpec> {
+pub struct ImportQueue<T: BeaconChainTypes> {
-    pub chain: Arc<BeaconChain<E>>,
+    pub chain: Arc<BeaconChain<T>>,
     /// Partially imported blocks, keyed by the root of `BeaconBlockBody`.
     pub partials: Vec<PartialBeaconBlock>,
     /// Time before a queue entry is considered state.
@@ -29,9 +29,9 @@ pub struct ImportQueue<E: EthSpec> {
     log: slog::Logger,
 }
 
-impl<E: EthSpec> ImportQueue<E> {
+impl<T: BeaconChainTypes> ImportQueue<T> {
     /// Return a new, empty queue.
-    pub fn new(chain: Arc<BeaconChain<E>>, stale_time: Duration, log: slog::Logger) -> Self {
+    pub fn new(chain: Arc<BeaconChain<T>>, stale_time: Duration, log: slog::Logger) -> Self {
         Self {
             chain,
             partials: vec![],

beacon_node/network/src/sync/simple_sync.rs

@@ -1,5 +1,5 @@
 use super::import_queue::ImportQueue;
-use crate::beacon_chain::{BeaconChain, BlockProcessingOutcome, InvalidBlock};
+use crate::beacon_chain::{BeaconChain, BeaconChainTypes, BlockProcessingOutcome, InvalidBlock};
 use crate::message_handler::NetworkContext;
 use eth2_libp2p::rpc::methods::*;
 use eth2_libp2p::rpc::{RPCRequest, RPCResponse, RequestId};
@@ -9,7 +9,7 @@ use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::Duration;
 use tree_hash::TreeHash;
-use types::{Attestation, BeaconBlock, Epoch, EthSpec, Hash256, Slot};
+use types::{Attestation, BeaconBlock, Epoch, Hash256, Slot};
 
 /// The number of slots that we can import blocks ahead of us, before going into full Sync mode.
 const SLOT_IMPORT_TOLERANCE: u64 = 100;
@@ -88,8 +88,8 @@ impl From<HelloMessage> for PeerSyncInfo {
     }
 }
 
-impl<E: EthSpec> From<&Arc<BeaconChain<E>>> for PeerSyncInfo {
+impl<T: BeaconChainTypes> From<&Arc<BeaconChain<T>>> for PeerSyncInfo {
-    fn from(chain: &Arc<BeaconChain<E>>) -> PeerSyncInfo {
+    fn from(chain: &Arc<BeaconChain<T>>) -> PeerSyncInfo {
         Self::from(chain.hello_message())
     }
 }
@@ -103,22 +103,22 @@ pub enum SyncState {
 }
 
 /// Simple Syncing protocol.
-pub struct SimpleSync<E: EthSpec> {
+pub struct SimpleSync<T: BeaconChainTypes> {
     /// A reference to the underlying beacon chain.
-    chain: Arc<BeaconChain<E>>,
+    chain: Arc<BeaconChain<T>>,
     /// A mapping of Peers to their respective PeerSyncInfo.
     known_peers: HashMap<PeerId, PeerSyncInfo>,
     /// A queue to allow importing of blocks
-    import_queue: ImportQueue<E>,
+    import_queue: ImportQueue<T>,
     /// The current state of the syncing protocol.
     state: SyncState,
     /// Sync logger.
     log: slog::Logger,
 }
 
-impl<E: EthSpec> SimpleSync<E> {
+impl<T: BeaconChainTypes> SimpleSync<T> {
     /// Instantiate a `SimpleSync` instance, with no peers and an empty queue.
-    pub fn new(beacon_chain: Arc<BeaconChain<E>>, log: &slog::Logger) -> Self {
+    pub fn new(beacon_chain: Arc<BeaconChain<T>>, log: &slog::Logger) -> Self {
         let sync_logger = log.new(o!("Service"=> "Sync"));
 
         let queue_item_stale_time = Duration::from_secs(QUEUE_STALE_SECS);

beacon_node/rpc/Cargo.toml

@@ -17,7 +17,7 @@ protos = { path = "../../protos" }
 grpcio = { version = "0.4", default-features = false, features = ["protobuf-codec"] }
 protobuf = "2.0.2"
 clap = "2.32.0"
-db = { path = "../db" }
+store = { path = "../store" }
 dirs = "1.0.3"
 futures = "0.1.23"
 slog = "^2.2.3"

beacon_node/rpc/src/attestation.rs

@@ -1,4 +1,4 @@
-use crate::beacon_chain::BeaconChain;
+use crate::beacon_chain::{BeaconChain, BeaconChainTypes};
 use futures::Future;
 use grpcio::{RpcContext, RpcStatus, RpcStatusCode, UnarySink};
 use protos::services::{
@@ -9,15 +9,15 @@ use protos::services_grpc::AttestationService;
 use slog::{error, info, trace, warn};
 use ssz::{ssz_encode, Decode};
 use std::sync::Arc;
-use types::{Attestation, EthSpec};
+use types::Attestation;
 
 #[derive(Clone)]
-pub struct AttestationServiceInstance<E: EthSpec> {
+pub struct AttestationServiceInstance<T: BeaconChainTypes> {
-    pub chain: Arc<BeaconChain<E>>,
+    pub chain: Arc<BeaconChain<T>>,
     pub log: slog::Logger,
 }
 
-impl<E: EthSpec> AttestationService for AttestationServiceInstance<E> {
+impl<T: BeaconChainTypes> AttestationService for AttestationServiceInstance<T> {
     /// Produce the `AttestationData` for signing by a validator.
     fn produce_attestation_data(
         &mut self,

beacon_node/rpc/src/beacon_block.rs

@@ -1,4 +1,4 @@
-use crate::beacon_chain::BeaconChain;
+use crate::beacon_chain::{BeaconChain, BeaconChainTypes};
 use crossbeam_channel;
 use eth2_libp2p::PubsubMessage;
 use futures::Future;
@@ -13,16 +13,16 @@ use slog::Logger;
 use slog::{error, info, trace, warn};
 use ssz::{ssz_encode, Decode};
 use std::sync::Arc;
-use types::{BeaconBlock, EthSpec, Signature, Slot};
+use types::{BeaconBlock, Signature, Slot};
 
 #[derive(Clone)]
-pub struct BeaconBlockServiceInstance<E: EthSpec> {
+pub struct BeaconBlockServiceInstance<T: BeaconChainTypes> {
-    pub chain: Arc<BeaconChain<E>>,
+    pub chain: Arc<BeaconChain<T>>,
     pub network_chan: crossbeam_channel::Sender<NetworkMessage>,
     pub log: Logger,
 }
 
-impl<E: EthSpec> BeaconBlockService for BeaconBlockServiceInstance<E> {
+impl<T: BeaconChainTypes> BeaconBlockService for BeaconBlockServiceInstance<T> {
     /// Produce a `BeaconBlock` for signing by a validator.
     fn produce_beacon_block(
         &mut self,

beacon_node/rpc/src/beacon_chain.rs

@@ -1,22 +1,19 @@
 use beacon_chain::BeaconChain as RawBeaconChain;
 use beacon_chain::{
-    db::ClientDB,
-    fork_choice::ForkChoice,
     parking_lot::{RwLockReadGuard, RwLockWriteGuard},
-    slot_clock::SlotClock,
     types::{BeaconState, ChainSpec, Signature},
     AttestationValidationError, BlockProductionError,
 };
-pub use beacon_chain::{BeaconChainError, BlockProcessingOutcome};
+pub use beacon_chain::{BeaconChainError, BeaconChainTypes, BlockProcessingOutcome};
 use types::{Attestation, AttestationData, BeaconBlock, EthSpec};
 
 /// The RPC's API to the beacon chain.
-pub trait BeaconChain<E: EthSpec>: Send + Sync {
+pub trait BeaconChain<T: BeaconChainTypes>: Send + Sync {
     fn get_spec(&self) -> &ChainSpec;
 
-    fn get_state(&self) -> RwLockReadGuard<BeaconState<E>>;
+    fn get_state(&self) -> RwLockReadGuard<BeaconState<T::EthSpec>>;
 
-    fn get_mut_state(&self) -> RwLockWriteGuard<BeaconState<E>>;
+    fn get_mut_state(&self) -> RwLockWriteGuard<BeaconState<T::EthSpec>>;
 
     fn process_block(&self, block: BeaconBlock)
         -> Result<BlockProcessingOutcome, BeaconChainError>;
@@ -24,7 +21,7 @@ pub trait BeaconChain<E: EthSpec>: Send + Sync {
     fn produce_block(
         &self,
         randao_reveal: Signature,
-    ) -> Result<(BeaconBlock, BeaconState<E>), BlockProductionError>;
+    ) -> Result<(BeaconBlock, BeaconState<T::EthSpec>), BlockProductionError>;
 
     fn produce_attestation_data(&self, shard: u64) -> Result<AttestationData, BeaconChainError>;
 
@@ -34,22 +31,16 @@ pub trait BeaconChain<E: EthSpec>: Send + Sync {
     ) -> Result<(), AttestationValidationError>;
 }
 
-impl<T, U, F, E> BeaconChain<E> for RawBeaconChain<T, U, F, E>
+impl<T: BeaconChainTypes> BeaconChain<T> for RawBeaconChain<T> {
-where
-    T: ClientDB + Sized,
-    U: SlotClock,
-    F: ForkChoice,
-    E: EthSpec,
-{
     fn get_spec(&self) -> &ChainSpec {
         &self.spec
     }
 
-    fn get_state(&self) -> RwLockReadGuard<BeaconState<E>> {
+    fn get_state(&self) -> RwLockReadGuard<BeaconState<T::EthSpec>> {
         self.state.read()
     }
 
-    fn get_mut_state(&self) -> RwLockWriteGuard<BeaconState<E>> {
+    fn get_mut_state(&self) -> RwLockWriteGuard<BeaconState<T::EthSpec>> {
         self.state.write()
     }
 
@@ -63,7 +54,7 @@ where
     fn produce_block(
         &self,
         randao_reveal: Signature,
-    ) -> Result<(BeaconBlock, BeaconState<E>), BlockProductionError> {
+    ) -> Result<(BeaconBlock, BeaconState<T::EthSpec>), BlockProductionError> {
         self.produce_block(randao_reveal)
     }
 

beacon_node/rpc/src/beacon_node.rs

@@ -1,19 +1,18 @@
-use crate::beacon_chain::BeaconChain;
+use crate::beacon_chain::{BeaconChain, BeaconChainTypes};
 use futures::Future;
 use grpcio::{RpcContext, UnarySink};
 use protos::services::{Empty, Fork, NodeInfoResponse};
 use protos::services_grpc::BeaconNodeService;
 use slog::{trace, warn};
 use std::sync::Arc;
-use types::EthSpec;
 
 #[derive(Clone)]
-pub struct BeaconNodeServiceInstance<E: EthSpec> {
+pub struct BeaconNodeServiceInstance<T: BeaconChainTypes> {
-    pub chain: Arc<BeaconChain<E>>,
+    pub chain: Arc<BeaconChain<T>>,
     pub log: slog::Logger,
 }
 
-impl<E: EthSpec> BeaconNodeService for BeaconNodeServiceInstance<E> {
+impl<T: BeaconChainTypes> BeaconNodeService for BeaconNodeServiceInstance<T> {
     /// Provides basic node information.
     fn info(&mut self, ctx: RpcContext, _req: Empty, sink: UnarySink<NodeInfoResponse>) {
         trace!(self.log, "Node info requested via RPC");

beacon_node/rpc/src/lib.rs

@@ -7,7 +7,7 @@ mod validator;
 
 use self::attestation::AttestationServiceInstance;
 use self::beacon_block::BeaconBlockServiceInstance;
-use self::beacon_chain::BeaconChain;
+use self::beacon_chain::{BeaconChain, BeaconChainTypes};
 use self::beacon_node::BeaconNodeServiceInstance;
 use self::validator::ValidatorServiceInstance;
 pub use config::Config as RPCConfig;
@@ -21,13 +21,12 @@ use protos::services_grpc::{
 use slog::{info, o, warn};
 use std::sync::Arc;
 use tokio::runtime::TaskExecutor;
-use types::EthSpec;
 
-pub fn start_server<E: EthSpec>(
+pub fn start_server<T: BeaconChainTypes + Clone + 'static>(
     config: &RPCConfig,
     executor: &TaskExecutor,
     network_chan: crossbeam_channel::Sender<NetworkMessage>,
-    beacon_chain: Arc<BeaconChain<E>>,
+    beacon_chain: Arc<BeaconChain<T>>,
     log: &slog::Logger,
 ) -> exit_future::Signal {
     let log = log.new(o!("Service"=>"RPC"));

beacon_node/rpc/src/validator.rs

@@ -1,4 +1,4 @@
-use crate::beacon_chain::BeaconChain;
+use crate::beacon_chain::{BeaconChain, BeaconChainTypes};
 use bls::PublicKey;
 use futures::Future;
 use grpcio::{RpcContext, RpcStatus, RpcStatusCode, UnarySink};
@@ -7,16 +7,16 @@ use protos::services_grpc::ValidatorService;
 use slog::{trace, warn};
 use ssz::Decode;
 use std::sync::Arc;
-use types::{Epoch, EthSpec, RelativeEpoch};
+use types::{Epoch, RelativeEpoch};
 
 #[derive(Clone)]
-pub struct ValidatorServiceInstance<E: EthSpec> {
+pub struct ValidatorServiceInstance<T: BeaconChainTypes> {
-    pub chain: Arc<BeaconChain<E>>,
+    pub chain: Arc<BeaconChain<T>>,
     pub log: slog::Logger,
 }
 //TODO: Refactor Errors
 
-impl<E: EthSpec> ValidatorService for ValidatorServiceInstance<E> {
+impl<T: BeaconChainTypes> ValidatorService for ValidatorServiceInstance<T> {
     /// For a list of validator public keys, this function returns the slot at which each
     /// validator must propose a block, attest to a shard, their shard committee and the shard they
     /// need to attest to.

beacon_node/src/main.rs

@@ -68,6 +68,15 @@ fn main() {
                 .help("Listen port for RPC endpoint.")
                 .takes_value(true),
         )
+        .arg(
+            Arg::with_name("db")
+                .long("db")
+                .value_name("DB")
+                .help("Type of database to use.")
+                .takes_value(true)
+                .possible_values(&["disk", "memory"])
+                .default_value("memory"),
+        )
         .get_matches();
 
     // invalid arguments, panic

beacon_node/src/run.rs

@@ -1,15 +1,18 @@
-use client::client_types::TestingClientType;
+use client::{
-use client::error;
+    error, notifier, BeaconChainTypes, Client, ClientConfig, DBType, TestnetDiskBeaconChainTypes,
-use client::{notifier, Client, ClientConfig};
+    TestnetMemoryBeaconChainTypes,
+};
 use futures::sync::oneshot;
 use futures::Future;
 use slog::info;
 use std::cell::RefCell;
 use tokio::runtime::Builder;
+use tokio::runtime::Runtime;
+use tokio::runtime::TaskExecutor;
 use tokio_timer::clock::Clock;
 
 pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Result<()> {
-    let mut runtime = Builder::new()
+    let runtime = Builder::new()
         .name_prefix("main-")
         .clock(Clock::system())
         .build()
@@ -20,8 +23,42 @@ pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Resul
           "data_dir" => &config.data_dir.to_str(),
           "port" => &config.net_conf.listen_port);
 
+    let executor = runtime.executor();
+
+    match config.db_type {
+        DBType::Disk => {
+            info!(
+                log,
+                "BeaconNode starting";
+                "type" => "TestnetDiskBeaconChainTypes"
+            );
+            let client: Client<TestnetDiskBeaconChainTypes> =
+                Client::new(config, log.clone(), &executor)?;
+
+            run(client, executor, runtime, log)
+        }
+        DBType::Memory => {
+            info!(
+                log,
+                "BeaconNode starting";
+                "type" => "TestnetMemoryBeaconChainTypes"
+            );
+            let client: Client<TestnetMemoryBeaconChainTypes> =
+                Client::new(config, log.clone(), &executor)?;
+
+            run(client, executor, runtime, log)
+        }
+    }
+}
+
+pub fn run<T: BeaconChainTypes + Send + Sync + 'static>(
+    client: Client<T>,
+    executor: TaskExecutor,
+    mut runtime: Runtime,
+    log: &slog::Logger,
+) -> error::Result<()> {
     // run service until ctrl-c
-    let (ctrlc_send, ctrlc) = oneshot::channel();
+    let (ctrlc_send, ctrlc_oneshot) = oneshot::channel();
     let ctrlc_send_c = RefCell::new(Some(ctrlc_send));
     ctrlc::set_handler(move || {
         if let Some(ctrlc_send) = ctrlc_send_c.try_borrow_mut().unwrap().take() {
@@ -32,14 +69,10 @@ pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Resul
 
     let (exit_signal, exit) = exit_future::signal();
 
-    let executor = runtime.executor();
-
-    // currently testing - using TestingClientType
-    let client: Client<TestingClientType> = Client::new(config, log.clone(), &executor)?;
     notifier::run(&client, executor, exit);
 
     runtime
-        .block_on(ctrlc)
+        .block_on(ctrlc_oneshot)
         .map_err(|e| format!("Ctrlc oneshot failed: {:?}", e))?;
 
     // perform global shutdown operations.

beacon_node/store/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "store"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+[dev-dependencies]
+tempfile = "3"
+
+[dependencies]
+blake2-rfc = "0.2.18"
+bls = { path = "../../eth2/utils/bls" }
+bytes = "0.4.10"
+db-key = "0.0.5"
+leveldb = "0.8.4"
+parking_lot = "0.7"
+ssz = { path = "../../eth2/utils/ssz" }
+ssz_derive = { path = "../../eth2/utils/ssz_derive" }
+tree_hash = { path = "../../eth2/utils/tree_hash" }
+types = { path =  "../../eth2/types" }

beacon_node/store/src/block_at_slot.rs

@@ -0,0 +1,183 @@
+use super::*;
+use ssz::{Decode, DecodeError};
+
+fn get_block_bytes<T: Store>(store: &T, root: Hash256) -> Result<Option<Vec<u8>>, Error> {
+    store.get_bytes(BeaconBlock::db_column().into(), &root[..])
+}
+
+fn read_slot_from_block_bytes(bytes: &[u8]) -> Result<Slot, DecodeError> {
+    let end = std::cmp::min(Slot::ssz_fixed_len(), bytes.len());
+
+    Slot::from_ssz_bytes(&bytes[0..end])
+}
+
+fn read_previous_block_root_from_block_bytes(bytes: &[u8]) -> Result<Hash256, DecodeError> {
+    let previous_bytes = Slot::ssz_fixed_len();
+    let slice = bytes
+        .get(previous_bytes..previous_bytes + Hash256::ssz_fixed_len())
+        .ok_or_else(|| DecodeError::BytesInvalid("Not enough bytes.".to_string()))?;
+
+    Hash256::from_ssz_bytes(slice)
+}
+
+pub fn get_block_at_preceeding_slot<T: Store>(
+    store: &T,
+    slot: Slot,
+    start_root: Hash256,
+) -> Result<Option<(Hash256, BeaconBlock)>, Error> {
+    let mut root = start_root;
+
+    loop {
+        if let Some(bytes) = get_block_bytes(store, root)? {
+            let this_slot = read_slot_from_block_bytes(&bytes)?;
+
+            if this_slot == slot {
+                let block = BeaconBlock::from_ssz_bytes(&bytes)?;
+                break Ok(Some((root, block)));
+            } else if this_slot < slot {
+                break Ok(None);
+            } else {
+                root = read_previous_block_root_from_block_bytes(&bytes)?;
+            }
+        } else {
+            break Ok(None);
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use ssz::Encode;
+    use tree_hash::TreeHash;
+
+    #[test]
+    fn read_slot() {
+        let spec = FewValidatorsEthSpec::spec();
+
+        let test_slot = |slot: Slot| {
+            let mut block = BeaconBlock::empty(&spec);
+            block.slot = slot;
+            let bytes = block.as_ssz_bytes();
+            assert_eq!(read_slot_from_block_bytes(&bytes).unwrap(), slot);
+        };
+
+        test_slot(Slot::new(0));
+        test_slot(Slot::new(1));
+        test_slot(Slot::new(42));
+        test_slot(Slot::new(u64::max_value()));
+    }
+
+    #[test]
+    fn bad_slot() {
+        for i in 0..8 {
+            assert!(read_slot_from_block_bytes(&vec![0; i]).is_err());
+        }
+    }
+
+    #[test]
+    fn read_previous_block_root() {
+        let spec = FewValidatorsEthSpec::spec();
+
+        let test_root = |root: Hash256| {
+            let mut block = BeaconBlock::empty(&spec);
+            block.previous_block_root = root;
+            let bytes = block.as_ssz_bytes();
+            assert_eq!(
+                read_previous_block_root_from_block_bytes(&bytes).unwrap(),
+                root
+            );
+        };
+
+        test_root(Hash256::random());
+        test_root(Hash256::random());
+        test_root(Hash256::random());
+    }
+
+    fn build_chain(
+        store: &impl Store,
+        slots: &[usize],
+        spec: &ChainSpec,
+    ) -> Vec<(Hash256, BeaconBlock)> {
+        let mut blocks_and_roots: Vec<(Hash256, BeaconBlock)> = vec![];
+
+        for (i, slot) in slots.iter().enumerate() {
+            let mut block = BeaconBlock::empty(spec);
+            block.slot = Slot::from(*slot);
+
+            if i > 0 {
+                block.previous_block_root = blocks_and_roots[i - 1].0;
+            }
+
+            let root = Hash256::from_slice(&block.tree_hash_root());
+
+            store.put(&root, &block).unwrap();
+            blocks_and_roots.push((root, block));
+        }
+
+        blocks_and_roots
+    }
+
+    #[test]
+    fn chain_without_skips() {
+        let n: usize = 10;
+        let store = MemoryStore::open();
+        let spec = FewValidatorsEthSpec::spec();
+
+        let slots: Vec<usize> = (0..n).collect();
+        let blocks_and_roots = build_chain(&store, &slots, &spec);
+
+        for source in 1..n {
+            for target in 0..=source {
+                let (source_root, _source_block) = &blocks_and_roots[source];
+                let (target_root, target_block) = &blocks_and_roots[target];
+
+                let (found_root, found_block) = store
+                    .get_block_at_preceeding_slot(*source_root, target_block.slot)
+                    .unwrap()
+                    .unwrap();
+
+                assert_eq!(found_root, *target_root);
+                assert_eq!(found_block, *target_block);
+            }
+        }
+    }
+
+    #[test]
+    fn chain_with_skips() {
+        let store = MemoryStore::open();
+        let spec = FewValidatorsEthSpec::spec();
+
+        let slots = vec![0, 1, 2, 5];
+
+        let blocks_and_roots = build_chain(&store, &slots, &spec);
+
+        // Valid slots
+        for target in 0..3 {
+            let (source_root, _source_block) = &blocks_and_roots[3];
+            let (target_root, target_block) = &blocks_and_roots[target];
+
+            let (found_root, found_block) = store
+                .get_block_at_preceeding_slot(*source_root, target_block.slot)
+                .unwrap()
+                .unwrap();
+
+            assert_eq!(found_root, *target_root);
+            assert_eq!(found_block, *target_block);
+        }
+
+        // Slot that doesn't exist
+        let (source_root, _source_block) = &blocks_and_roots[3];
+        assert!(store
+            .get_block_at_preceeding_slot(*source_root, Slot::new(3))
+            .unwrap()
+            .is_none());
+
+        // Slot too high
+        let (source_root, _source_block) = &blocks_and_roots[3];
+        assert!(store
+            .get_block_at_preceeding_slot(*source_root, Slot::new(3))
+            .unwrap()
+            .is_none());
+    }
+}

beacon_node/store/src/disk_db.rs

@@ -1,19 +1,20 @@
 extern crate rocksdb;
 
-use super::rocksdb::Error as RocksError;
+// use super::stores::COLUMNS;
-use super::rocksdb::{Options, DB};
 use super::{ClientDB, DBError, DBValue};
+use rocksdb::Error as RocksError;
+use rocksdb::{Options, DB};
 use std::fs;
 use std::path::Path;
 
 /// A on-disk database which implements the ClientDB trait.
 ///
 /// This implementation uses RocksDB with default options.
-pub struct DiskDB {
+pub struct DiskStore {
     db: DB,
 }
 
-impl DiskDB {
+impl DiskStore {
     /// Open the RocksDB database, optionally supplying columns if required.
     ///
     /// The RocksDB database will be contained in a directory titled
@@ -23,31 +24,32 @@ impl DiskDB {
     ///
     /// Panics if the database is unable to be created.
     pub fn open(path: &Path, columns: Option<&[&str]>) -> Self {
-        /*
+        // Rocks options.
-         * Initialise the options
-         */
         let mut options = Options::default();
         options.create_if_missing(true);
 
-        // TODO: ensure that columns are created (and remove
+        // Ensure the path exists.
-        // the dead_code allow)
-
-        /*
-         * Initialise the path
-         */
         fs::create_dir_all(&path).unwrap_or_else(|_| panic!("Unable to create {:?}", &path));
         let db_path = path.join("database");
 
-        /*
+        let columns = columns.unwrap_or(&COLUMNS);
-         * Open the database
-         */
-        let db = match columns {
-            None => DB::open(&options, db_path),
-            Some(columns) => DB::open_cf(&options, db_path, columns),
-        }
-        .expect("Unable to open local database");;
 
-        Self { db }
+        if db_path.exists() {
+            Self {
+                db: DB::open_cf(&options, db_path, &COLUMNS)
+                    .expect("Unable to open local database"),
+            }
+        } else {
+            let mut db = Self {
+                db: DB::open(&options, db_path).expect("Unable to open local database"),
+            };
+
+            for cf in columns {
+                db.create_col(cf).unwrap();
+            }
+
+            db
+        }
     }
 
     /// Create a RocksDB column family. Corresponds to the
@@ -69,7 +71,7 @@ impl From<RocksError> for DBError {
     }
 }
 
-impl ClientDB for DiskDB {
+impl ClientDB for DiskStore {
     /// Get the value for some key on some column.
     ///
     /// Corresponds to the `get_cf()` method on the RocksDB API.
@@ -97,7 +99,7 @@ impl ClientDB for DiskDB {
             None => Err(DBError {
                 message: "Unknown column".to_string(),
             }),
-            Some(handle) => self.db.put_cf(handle, key, val).map_err(Into::into),
+            Some(handle) => self.db.put_cf(handle, key, val).map_err(|e| e.into()),
         }
     }
 
@@ -152,7 +154,7 @@ mod tests {
         let col_name: &str = "TestColumn";
         let column_families = vec![col_name];
 
-        let mut db = DiskDB::open(&path, None);
+        let mut db = DiskStore::open(&path, None);
 
         for cf in column_families {
             db.create_col(&cf).unwrap();

beacon_node/store/src/errors.rs

@@ -0,0 +1,30 @@
+use ssz::DecodeError;
+
+#[derive(Debug, PartialEq)]
+pub enum Error {
+    SszDecodeError(DecodeError),
+    DBError { message: String },
+}
+
+impl From<DecodeError> for Error {
+    fn from(e: DecodeError) -> Error {
+        Error::SszDecodeError(e)
+    }
+}
+
+impl From<DBError> for Error {
+    fn from(e: DBError) -> Error {
+        Error::DBError { message: e.message }
+    }
+}
+
+#[derive(Debug)]
+pub struct DBError {
+    pub message: String,
+}
+
+impl DBError {
+    pub fn new(message: String) -> Self {
+        Self { message }
+    }
+}

beacon_node/store/src/impls.rs

@@ -0,0 +1,30 @@
+use crate::*;
+use ssz::{Decode, Encode};
+
+impl StoreItem for BeaconBlock {
+    fn db_column() -> DBColumn {
+        DBColumn::BeaconBlock
+    }
+
+    fn as_store_bytes(&self) -> Vec<u8> {
+        self.as_ssz_bytes()
+    }
+
+    fn from_store_bytes(bytes: &mut [u8]) -> Result<Self, Error> {
+        Self::from_ssz_bytes(bytes).map_err(Into::into)
+    }
+}
+
+impl<T: EthSpec> StoreItem for BeaconState<T> {
+    fn db_column() -> DBColumn {
+        DBColumn::BeaconState
+    }
+
+    fn as_store_bytes(&self) -> Vec<u8> {
+        self.as_ssz_bytes()
+    }
+
+    fn from_store_bytes(bytes: &mut [u8]) -> Result<Self, Error> {
+        Self::from_ssz_bytes(bytes).map_err(Into::into)
+    }
+}

beacon_node/store/src/leveldb_store.rs

@@ -0,0 +1,100 @@
+use super::*;
+use db_key::Key;
+use leveldb::database::kv::KV;
+use leveldb::database::Database;
+use leveldb::error::Error as LevelDBError;
+use leveldb::options::{Options, ReadOptions, WriteOptions};
+use std::path::Path;
+
+/// A wrapped leveldb database.
+pub struct LevelDB {
+    db: Database<BytesKey>,
+}
+
+impl LevelDB {
+    /// Open a database at `path`, creating a new database if one does not already exist.
+    pub fn open(path: &Path) -> Result<Self, Error> {
+        let mut options = Options::new();
+
+        options.create_if_missing = true;
+
+        let db = Database::open(path, options)?;
+
+        Ok(Self { db })
+    }
+
+    fn read_options(&self) -> ReadOptions<BytesKey> {
+        ReadOptions::new()
+    }
+
+    fn write_options(&self) -> WriteOptions {
+        WriteOptions::new()
+    }
+
+    fn get_key_for_col(col: &str, key: &[u8]) -> BytesKey {
+        let mut col = col.as_bytes().to_vec();
+        col.append(&mut key.to_vec());
+        BytesKey { key: col }
+    }
+}
+
+/// Used for keying leveldb.
+pub struct BytesKey {
+    key: Vec<u8>,
+}
+
+impl Key for BytesKey {
+    fn from_u8(key: &[u8]) -> Self {
+        Self { key: key.to_vec() }
+    }
+
+    fn as_slice<T, F: Fn(&[u8]) -> T>(&self, f: F) -> T {
+        f(self.key.as_slice())
+    }
+}
+
+impl Store for LevelDB {
+    /// Retrieve some bytes in `column` with `key`.
+    fn get_bytes(&self, col: &str, key: &[u8]) -> Result<Option<Vec<u8>>, Error> {
+        let column_key = Self::get_key_for_col(col, key);
+
+        self.db
+            .get(self.read_options(), column_key)
+            .map_err(Into::into)
+    }
+
+    /// Store some `value` in `column`, indexed with `key`.
+    fn put_bytes(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), Error> {
+        let column_key = Self::get_key_for_col(col, key);
+
+        self.db
+            .put(self.write_options(), column_key, val)
+            .map_err(Into::into)
+    }
+
+    /// Return `true` if `key` exists in `column`.
+    fn key_exists(&self, col: &str, key: &[u8]) -> Result<bool, Error> {
+        let column_key = Self::get_key_for_col(col, key);
+
+        self.db
+            .get(self.read_options(), column_key)
+            .map_err(Into::into)
+            .and_then(|val| Ok(val.is_some()))
+    }
+
+    /// Removes `key` from `column`.
+    fn key_delete(&self, col: &str, key: &[u8]) -> Result<(), Error> {
+        let column_key = Self::get_key_for_col(col, key);
+        self.db
+            .delete(self.write_options(), column_key)
+            .map_err(Into::into)
+    }
+}
+
+impl From<LevelDBError> for Error {
+    fn from(e: LevelDBError) -> Error {
+        Error::DBError {
+            message: format!("{:?}", e),
+        }
+    }
+}

beacon_node/store/src/lib.rs

@@ -0,0 +1,223 @@
+//! Storage functionality for Lighthouse.
+//!
+//! Provides the following stores:
+//!
+//! - `DiskStore`: an on-disk store backed by leveldb. Used in production.
+//! - `MemoryStore`: an in-memory store backed by a hash-map. Used for testing.
+//!
+//! Provides a simple API for storing/retrieving all types that sometimes needs type-hints. See
+//! tests for implementation examples.
+
+mod block_at_slot;
+mod errors;
+mod impls;
+mod leveldb_store;
+mod memory_store;
+
+pub use self::leveldb_store::LevelDB as DiskStore;
+pub use self::memory_store::MemoryStore;
+pub use errors::Error;
+pub use types::*;
+
+/// An object capable of storing and retrieving objects implementing `StoreItem`.
+///
+/// A `Store` is fundamentally backed by a key-value database, however it provides support for
+/// columns. A simple column implementation might involve prefixing a key with some bytes unique to
+/// each column.
+pub trait Store: Sync + Send + Sized {
+    /// Store an item in `Self`.
+    fn put(&self, key: &Hash256, item: &impl StoreItem) -> Result<(), Error> {
+        item.db_put(self, key)
+    }
+
+    /// Retrieve an item from `Self`.
+    fn get<I: StoreItem>(&self, key: &Hash256) -> Result<Option<I>, Error> {
+        I::db_get(self, key)
+    }
+
+    /// Returns `true` if the given key represents an item in `Self`.
+    fn exists<I: StoreItem>(&self, key: &Hash256) -> Result<bool, Error> {
+        I::db_exists(self, key)
+    }
+
+    /// Remove an item from `Self`.
+    fn delete<I: StoreItem>(&self, key: &Hash256) -> Result<(), Error> {
+        I::db_delete(self, key)
+    }
+
+    /// Given the root of an existing block in the store (`start_block_root`), return a parent
+    /// block with the specified `slot`.
+    ///
+    /// Returns `None` if no parent block exists at that slot, or if `slot` is greater than the
+    /// slot of `start_block_root`.
+    fn get_block_at_preceeding_slot(
+        &self,
+        start_block_root: Hash256,
+        slot: Slot,
+    ) -> Result<Option<(Hash256, BeaconBlock)>, Error> {
+        block_at_slot::get_block_at_preceeding_slot(self, slot, start_block_root)
+    }
+
+    /// Retrieve some bytes in `column` with `key`.
+    fn get_bytes(&self, column: &str, key: &[u8]) -> Result<Option<Vec<u8>>, Error>;
+
+    /// Store some `value` in `column`, indexed with `key`.
+    fn put_bytes(&self, column: &str, key: &[u8], value: &[u8]) -> Result<(), Error>;
+
+    /// Return `true` if `key` exists in `column`.
+    fn key_exists(&self, column: &str, key: &[u8]) -> Result<bool, Error>;
+
+    /// Removes `key` from `column`.
+    fn key_delete(&self, column: &str, key: &[u8]) -> Result<(), Error>;
+}
+
+/// A unique column identifier.
+pub enum DBColumn {
+    BeaconBlock,
+    BeaconState,
+    BeaconChain,
+}
+
+impl<'a> Into<&'a str> for DBColumn {
+    /// Returns a `&str` that can be used for keying a key-value data base.
+    fn into(self) -> &'a str {
+        match self {
+            DBColumn::BeaconBlock => &"blk",
+            DBColumn::BeaconState => &"ste",
+            DBColumn::BeaconChain => &"bch",
+        }
+    }
+}
+
+/// An item that may be stored in a `Store`.
+///
+/// Provides default methods that are suitable for most applications, however when overridden they
+/// provide full customizability of `Store` operations.
+pub trait StoreItem: Sized {
+    /// Identifies which column this item should be placed in.
+    fn db_column() -> DBColumn;
+
+    /// Serialize `self` as bytes.
+    fn as_store_bytes(&self) -> Vec<u8>;
+
+    /// De-serialize `self` from bytes.
+    fn from_store_bytes(bytes: &mut [u8]) -> Result<Self, Error>;
+
+    /// Store `self`.
+    fn db_put(&self, store: &impl Store, key: &Hash256) -> Result<(), Error> {
+        let column = Self::db_column().into();
+        let key = key.as_bytes();
+
+        store
+            .put_bytes(column, key, &self.as_store_bytes())
+            .map_err(Into::into)
+    }
+
+    /// Retrieve an instance of `Self`.
+    fn db_get(store: &impl Store, key: &Hash256) -> Result<Option<Self>, Error> {
+        let column = Self::db_column().into();
+        let key = key.as_bytes();
+
+        match store.get_bytes(column, key)? {
+            Some(mut bytes) => Ok(Some(Self::from_store_bytes(&mut bytes[..])?)),
+            None => Ok(None),
+        }
+    }
+
+    /// Return `true` if an instance of `Self` exists in `Store`.
+    fn db_exists(store: &impl Store, key: &Hash256) -> Result<bool, Error> {
+        let column = Self::db_column().into();
+        let key = key.as_bytes();
+
+        store.key_exists(column, key)
+    }
+
+    /// Delete `self` from the `Store`.
+    fn db_delete(store: &impl Store, key: &Hash256) -> Result<(), Error> {
+        let column = Self::db_column().into();
+        let key = key.as_bytes();
+
+        store.key_delete(column, key)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use ssz::{Decode, Encode};
+    use ssz_derive::{Decode, Encode};
+    use tempfile::tempdir;
+
+    #[derive(PartialEq, Debug, Encode, Decode)]
+    struct StorableThing {
+        a: u64,
+        b: u64,
+    }
+
+    impl StoreItem for StorableThing {
+        fn db_column() -> DBColumn {
+            DBColumn::BeaconBlock
+        }
+
+        fn as_store_bytes(&self) -> Vec<u8> {
+            self.as_ssz_bytes()
+        }
+
+        fn from_store_bytes(bytes: &mut [u8]) -> Result<Self, Error> {
+            Self::from_ssz_bytes(bytes).map_err(Into::into)
+        }
+    }
+
+    fn test_impl(store: impl Store) {
+        let key = Hash256::random();
+        let item = StorableThing { a: 1, b: 42 };
+
+        assert_eq!(store.exists::<StorableThing>(&key), Ok(false));
+
+        store.put(&key, &item).unwrap();
+
+        assert_eq!(store.exists::<StorableThing>(&key), Ok(true));
+
+        let retrieved = store.get(&key).unwrap().unwrap();
+        assert_eq!(item, retrieved);
+
+        store.delete::<StorableThing>(&key).unwrap();
+
+        assert_eq!(store.exists::<StorableThing>(&key), Ok(false));
+
+        assert_eq!(store.get::<StorableThing>(&key), Ok(None));
+    }
+
+    #[test]
+    fn diskdb() {
+        let dir = tempdir().unwrap();
+        let path = dir.path();
+        let store = DiskStore::open(&path).unwrap();
+
+        test_impl(store);
+    }
+
+    #[test]
+    fn memorydb() {
+        let store = MemoryStore::open();
+
+        test_impl(store);
+    }
+
+    #[test]
+    fn exists() {
+        let store = MemoryStore::open();
+        let key = Hash256::random();
+        let item = StorableThing { a: 1, b: 42 };
+
+        assert_eq!(store.exists::<StorableThing>(&key).unwrap(), false);
+
+        store.put(&key, &item).unwrap();
+
+        assert_eq!(store.exists::<StorableThing>(&key).unwrap(), true);
+
+        store.delete::<StorableThing>(&key).unwrap();
+
+        assert_eq!(store.exists::<StorableThing>(&key).unwrap(), false);
+    }
+}

beacon_node/store/src/memory_store.rs

@@ -0,0 +1,63 @@
+use super::{Error, Store};
+use parking_lot::RwLock;
+use std::collections::HashMap;
+
+type DBHashMap = HashMap<Vec<u8>, Vec<u8>>;
+
+/// A thread-safe `HashMap` wrapper.
+pub struct MemoryStore {
+    db: RwLock<DBHashMap>,
+}
+
+impl MemoryStore {
+    /// Create a new, empty database.
+    pub fn open() -> Self {
+        Self {
+            db: RwLock::new(HashMap::new()),
+        }
+    }
+
+    fn get_key_for_col(col: &str, key: &[u8]) -> Vec<u8> {
+        let mut col = col.as_bytes().to_vec();
+        col.append(&mut key.to_vec());
+        col
+    }
+}
+
+impl Store for MemoryStore {
+    /// Get the value of some key from the database. Returns `None` if the key does not exist.
+    fn get_bytes(&self, col: &str, key: &[u8]) -> Result<Option<Vec<u8>>, Error> {
+        let column_key = MemoryStore::get_key_for_col(col, key);
+
+        Ok(self
+            .db
+            .read()
+            .get(&column_key)
+            .and_then(|val| Some(val.clone())))
+    }
+
+    /// Puts a key in the database.
+    fn put_bytes(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), Error> {
+        let column_key = MemoryStore::get_key_for_col(col, key);
+
+        self.db.write().insert(column_key, val.to_vec());
+
+        Ok(())
+    }
+
+    /// Return true if some key exists in some column.
+    fn key_exists(&self, col: &str, key: &[u8]) -> Result<bool, Error> {
+        let column_key = MemoryStore::get_key_for_col(col, key);
+
+        Ok(self.db.read().contains_key(&column_key))
+    }
+
+    /// Delete some key from the database.
+    fn key_delete(&self, col: &str, key: &[u8]) -> Result<(), Error> {
+        let column_key = MemoryStore::get_key_for_col(col, key);
+
+        self.db.write().remove(&column_key);
+
+        Ok(())
+    }
+}

beacon_node/store/src/store.rs

@@ -0,0 +1,37 @@
+use super::*;
+
+pub type Vec<u8> = Vec<u8>;
+
+pub trait Store: Sync + Send + Sized {
+    fn put(&self, key: &Hash256, item: &impl StoreItem) -> Result<(), Error> {
+        item.db_put(self, key)
+    }
+
+    fn get<I: StoreItem>(&self, key: &Hash256) -> Result<Option<I>, Error> {
+        I::db_get(self, key)
+    }
+
+    fn exists<I: StoreItem>(&self, key: &Hash256) -> Result<bool, Error> {
+        I::db_exists(self, key)
+    }
+
+    fn delete<I: StoreItem>(&self, key: &Hash256) -> Result<(), Error> {
+        I::db_delete(self, key)
+    }
+
+    fn get_block_at_preceeding_slot(
+        &self,
+        start_block_root: Hash256,
+        slot: Slot,
+    ) -> Result<Option<(Hash256, BeaconBlock)>, Error> {
+        block_at_slot::get_block_at_preceeding_slot(self, slot, start_block_root)
+    }
+
+    fn get_bytes(&self, col: &str, key: &[u8]) -> Result<Option<Vec<u8>>, Error>;
+
+    fn put_bytes(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), Error>;
+
+    fn key_exists(&self, col: &str, key: &[u8]) -> Result<bool, Error>;
+
+    fn key_delete(&self, col: &str, key: &[u8]) -> Result<(), Error>;
+}

eth2/fork_choice/Cargo.toml

@@ -5,7 +5,7 @@ authors = ["Age Manning <Age@AgeManning.com>"]
 edition = "2018"
 
 [dependencies]
-db = { path = "../../beacon_node/db" }
+store = { path = "../../beacon_node/store" }
 ssz = { path = "../utils/ssz" }
 types = { path = "../types" }
 log = "0.4.6"

eth2/fork_choice/src/bitwise_lmd_ghost.rs

@@ -1,16 +1,11 @@
 //! The optimised bitwise LMD-GHOST fork choice rule.
-extern crate bit_vec;
-
 use crate::{ForkChoice, ForkChoiceError};
 use bit_vec::BitVec;
-use db::{
-    stores::{BeaconBlockStore, BeaconStateStore},
-    ClientDB,
-};
 use log::{debug, trace};
 use std::collections::HashMap;
 use std::marker::PhantomData;
 use std::sync::Arc;
+use store::Store;
 use types::{BeaconBlock, BeaconState, ChainSpec, EthSpec, Hash256, Slot, SlotHeight};
 
 //TODO: Pruning - Children
@@ -34,7 +29,7 @@ fn power_of_2_below(x: u64) -> u64 {
 }
 
 /// Stores the necessary data structures to run the optimised bitwise lmd ghost algorithm.
-pub struct BitwiseLMDGhost<T: ClientDB + Sized, E> {
+pub struct BitwiseLMDGhost<T, E> {
     /// A cache of known ancestors at given heights for a specific block.
     //TODO: Consider FnvHashMap
     cache: HashMap<CacheKey<u64>, Hash256>,
@@ -46,30 +41,21 @@ pub struct BitwiseLMDGhost<T: ClientDB + Sized, E> {
     /// The latest attestation targets as a map of validator index to block hash.
     //TODO: Could this be a fixed size vec
     latest_attestation_targets: HashMap<u64, Hash256>,
-    /// Block storage access.
+    /// Block and state storage.
-    block_store: Arc<BeaconBlockStore<T>>,
+    store: Arc<T>,
-    /// State storage access.
-    state_store: Arc<BeaconStateStore<T>>,
     max_known_height: SlotHeight,
     _phantom: PhantomData<E>,
 }
 
-impl<T, E: EthSpec> BitwiseLMDGhost<T, E>
+impl<T: Store, E: EthSpec> BitwiseLMDGhost<T, E> {
-where
+    pub fn new(store: Arc<T>) -> Self {
-    T: ClientDB + Sized,
-{
-    pub fn new(
-        block_store: Arc<BeaconBlockStore<T>>,
-        state_store: Arc<BeaconStateStore<T>>,
-    ) -> Self {
         BitwiseLMDGhost {
             cache: HashMap::new(),
             ancestors: vec![HashMap::new(); 16],
             latest_attestation_targets: HashMap::new(),
             children: HashMap::new(),
             max_known_height: SlotHeight::new(0),
-            block_store,
+            store,
-            state_store,
             _phantom: PhantomData,
         }
     }
@@ -89,8 +75,8 @@ where
         let mut latest_votes: HashMap<Hash256, u64> = HashMap::new();
         // gets the current weighted votes
         let current_state: BeaconState<E> = self
-            .state_store
+            .store
-            .get_deserialized(&state_root)?
+            .get(&state_root)?
             .ok_or_else(|| ForkChoiceError::MissingBeaconState(*state_root))?;
 
         let active_validator_indices =
@@ -121,8 +107,8 @@ where
         // return None if we can't get the block from the db.
         let block_height = {
             let block_slot = self
-                .block_store
+                .store
-                .get_deserialized(&block_hash)
+                .get::<BeaconBlock>(&block_hash)
                 .ok()?
                 .expect("Should have returned already if None")
                 .slot;
@@ -243,7 +229,7 @@ where
     }
 }
 
-impl<T: ClientDB + Sized, E: EthSpec> ForkChoice for BitwiseLMDGhost<T, E> {
+impl<T: Store, E: EthSpec> ForkChoice for BitwiseLMDGhost<T, E> {
     fn add_block(
         &mut self,
         block: &BeaconBlock,
@@ -252,8 +238,8 @@ impl<T: ClientDB + Sized, E: EthSpec> ForkChoice for BitwiseLMDGhost<T, E> {
     ) -> Result<(), ForkChoiceError> {
         // get the height of the parent
         let parent_height = self
-            .block_store
+            .store
-            .get_deserialized(&block.previous_block_root)?
+            .get::<BeaconBlock>(&block.previous_block_root)?
             .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(block.previous_block_root))?
             .slot
             .height(spec.genesis_slot);
@@ -304,16 +290,16 @@ impl<T: ClientDB + Sized, E: EthSpec> ForkChoice for BitwiseLMDGhost<T, E> {
             trace!("Old attestation found: {:?}", attestation_target);
             // get the height of the target block
             let block_height = self
-                .block_store
+                .store
-                .get_deserialized(&target_block_root)?
+                .get::<BeaconBlock>(&target_block_root)?
                 .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*target_block_root))?
                 .slot
                 .height(spec.genesis_slot);
 
             // get the height of the past target block
             let past_block_height = self
-                .block_store
+                .store
-                .get_deserialized(&attestation_target)?
+                .get::<BeaconBlock>(&attestation_target)?
                 .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*attestation_target))?
                 .slot
                 .height(spec.genesis_slot);
@@ -337,8 +323,8 @@ impl<T: ClientDB + Sized, E: EthSpec> ForkChoice for BitwiseLMDGhost<T, E> {
             justified_block_start
         );
         let block = self
-            .block_store
+            .store
-            .get_deserialized(&justified_block_start)?
+            .get::<BeaconBlock>(&justified_block_start)?
             .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*justified_block_start))?;
 
         let block_slot = block.slot;
@@ -429,8 +415,8 @@ impl<T: ClientDB + Sized, E: EthSpec> ForkChoice for BitwiseLMDGhost<T, E> {
             // didn't find head yet, proceed to next iteration
             // update block height
             block_height = self
-                .block_store
+                .store
-                .get_deserialized(&current_head)?
+                .get::<BeaconBlock>(&current_head)?
                 .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(current_head))?
                 .slot
                 .height(spec.genesis_slot);

eth2/fork_choice/src/lib.rs

@@ -16,17 +16,14 @@
 //! [`slow_lmd_ghost`]: struct.SlowLmdGhost.html
 //! [`bitwise_lmd_ghost`]: struct.OptimisedLmdGhost.html
 
-extern crate db;
-extern crate ssz;
-extern crate types;
-
 pub mod bitwise_lmd_ghost;
 pub mod longest_chain;
 pub mod optimized_lmd_ghost;
 pub mod slow_lmd_ghost;
 
-use db::stores::BeaconBlockAtSlotError;
+// use store::stores::BeaconBlockAtSlotError;
-use db::DBError;
+// use store::DBError;
+use store::Error as DBError;
 use types::{BeaconBlock, ChainSpec, Hash256};
 
 pub use bitwise_lmd_ghost::BitwiseLMDGhost;
@@ -77,10 +74,11 @@ pub enum ForkChoiceError {
 
 impl From<DBError> for ForkChoiceError {
     fn from(e: DBError) -> ForkChoiceError {
-        ForkChoiceError::StorageError(e.message)
+        ForkChoiceError::StorageError(format!("{:?}", e))
     }
 }
 
+/*
 impl From<BeaconBlockAtSlotError> for ForkChoiceError {
     fn from(e: BeaconBlockAtSlotError) -> ForkChoiceError {
         match e {
@@ -94,6 +92,7 @@ impl From<BeaconBlockAtSlotError> for ForkChoiceError {
         }
     }
 }
+*/
 
 /// Fork choice options that are currently implemented.
 #[derive(Debug, Clone)]

eth2/fork_choice/src/longest_chain.rs

@@ -1,31 +1,25 @@
 use crate::{ForkChoice, ForkChoiceError};
-use db::{stores::BeaconBlockStore, ClientDB};
 use std::sync::Arc;
+use store::Store;
 use types::{BeaconBlock, ChainSpec, Hash256, Slot};
 
-pub struct LongestChain<T>
+pub struct LongestChain<T> {
-where
-    T: ClientDB + Sized,
-{
     /// List of head block hashes
     head_block_hashes: Vec<Hash256>,
-    /// Block storage access.
+    /// Block storage.
-    block_store: Arc<BeaconBlockStore<T>>,
+    store: Arc<T>,
 }
 
-impl<T> LongestChain<T>
+impl<T: Store> LongestChain<T> {
-where
+    pub fn new(store: Arc<T>) -> Self {
-    T: ClientDB + Sized,
-{
-    pub fn new(block_store: Arc<BeaconBlockStore<T>>) -> Self {
         LongestChain {
             head_block_hashes: Vec::new(),
-            block_store,
+            store,
         }
     }
 }
 
-impl<T: ClientDB + Sized> ForkChoice for LongestChain<T> {
+impl<T: Store> ForkChoice for LongestChain<T> {
     fn add_block(
         &mut self,
         block: &BeaconBlock,
@@ -55,9 +49,9 @@ impl<T: ClientDB + Sized> ForkChoice for LongestChain<T> {
          * Load all the head_block hashes from the DB as SszBeaconBlocks.
          */
         for (index, block_hash) in self.head_block_hashes.iter().enumerate() {
-            let block = self
+            let block: BeaconBlock = self
-                .block_store
+                .store
-                .get_deserialized(&block_hash)?
+                .get(&block_hash)?
                 .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*block_hash))?;
             head_blocks.push((index, block));
         }

eth2/fork_choice/src/optimized_lmd_ghost.rs

@@ -1,16 +1,11 @@
 //! The optimised bitwise LMD-GHOST fork choice rule.
-extern crate bit_vec;
-
 use crate::{ForkChoice, ForkChoiceError};
-use db::{
-    stores::{BeaconBlockStore, BeaconStateStore},
-    ClientDB,
-};
 use log::{debug, trace};
 use std::cmp::Ordering;
 use std::collections::HashMap;
 use std::marker::PhantomData;
 use std::sync::Arc;
+use store::Store;
 use types::{BeaconBlock, BeaconState, ChainSpec, EthSpec, Hash256, Slot, SlotHeight};
 
 //TODO: Pruning - Children
@@ -34,7 +29,7 @@ fn power_of_2_below(x: u64) -> u64 {
 }
 
 /// Stores the necessary data structures to run the optimised lmd ghost algorithm.
-pub struct OptimizedLMDGhost<T: ClientDB + Sized, E> {
+pub struct OptimizedLMDGhost<T, E> {
     /// A cache of known ancestors at given heights for a specific block.
     //TODO: Consider FnvHashMap
     cache: HashMap<CacheKey<u64>, Hash256>,
@@ -46,30 +41,21 @@ pub struct OptimizedLMDGhost<T: ClientDB + Sized, E> {
     /// The latest attestation targets as a map of validator index to block hash.
     //TODO: Could this be a fixed size vec
     latest_attestation_targets: HashMap<u64, Hash256>,
-    /// Block storage access.
+    /// Block and state storage.
-    block_store: Arc<BeaconBlockStore<T>>,
+    store: Arc<T>,
-    /// State storage access.
-    state_store: Arc<BeaconStateStore<T>>,
     max_known_height: SlotHeight,
     _phantom: PhantomData<E>,
 }
 
-impl<T, E: EthSpec> OptimizedLMDGhost<T, E>
+impl<T: Store, E: EthSpec> OptimizedLMDGhost<T, E> {
-where
+    pub fn new(store: Arc<T>) -> Self {
-    T: ClientDB + Sized,
-{
-    pub fn new(
-        block_store: Arc<BeaconBlockStore<T>>,
-        state_store: Arc<BeaconStateStore<T>>,
-    ) -> Self {
         OptimizedLMDGhost {
             cache: HashMap::new(),
             ancestors: vec![HashMap::new(); 16],
             latest_attestation_targets: HashMap::new(),
             children: HashMap::new(),
             max_known_height: SlotHeight::new(0),
-            block_store,
+            store,
-            state_store,
             _phantom: PhantomData,
         }
     }
@@ -89,8 +75,8 @@ where
         let mut latest_votes: HashMap<Hash256, u64> = HashMap::new();
         // gets the current weighted votes
         let current_state: BeaconState<E> = self
-            .state_store
+            .store
-            .get_deserialized(&state_root)?
+            .get(&state_root)?
             .ok_or_else(|| ForkChoiceError::MissingBeaconState(*state_root))?;
 
         let active_validator_indices =
@@ -121,8 +107,8 @@ where
         // return None if we can't get the block from the db.
         let block_height = {
             let block_slot = self
-                .block_store
+                .store
-                .get_deserialized(&block_hash)
+                .get::<BeaconBlock>(&block_hash)
                 .ok()?
                 .expect("Should have returned already if None")
                 .slot;
@@ -214,7 +200,7 @@ where
     }
 }
 
-impl<T: ClientDB + Sized, E: EthSpec> ForkChoice for OptimizedLMDGhost<T, E> {
+impl<T: Store, E: EthSpec> ForkChoice for OptimizedLMDGhost<T, E> {
     fn add_block(
         &mut self,
         block: &BeaconBlock,
@@ -223,8 +209,8 @@ impl<T: ClientDB + Sized, E: EthSpec> ForkChoice for OptimizedLMDGhost<T, E> {
     ) -> Result<(), ForkChoiceError> {
         // get the height of the parent
         let parent_height = self
-            .block_store
+            .store
-            .get_deserialized(&block.previous_block_root)?
+            .get::<BeaconBlock>(&block.previous_block_root)?
             .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(block.previous_block_root))?
             .slot
             .height(spec.genesis_slot);
@@ -275,16 +261,16 @@ impl<T: ClientDB + Sized, E: EthSpec> ForkChoice for OptimizedLMDGhost<T, E> {
             trace!("Old attestation found: {:?}", attestation_target);
             // get the height of the target block
             let block_height = self
-                .block_store
+                .store
-                .get_deserialized(&target_block_root)?
+                .get::<BeaconBlock>(&target_block_root)?
                 .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*target_block_root))?
                 .slot
                 .height(spec.genesis_slot);
 
             // get the height of the past target block
             let past_block_height = self
-                .block_store
+                .store
-                .get_deserialized(&attestation_target)?
+                .get::<BeaconBlock>(&attestation_target)?
                 .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*attestation_target))?
                 .slot
                 .height(spec.genesis_slot);
@@ -308,8 +294,8 @@ impl<T: ClientDB + Sized, E: EthSpec> ForkChoice for OptimizedLMDGhost<T, E> {
             justified_block_start
         );
         let block = self
-            .block_store
+            .store
-            .get_deserialized(&justified_block_start)?
+            .get::<BeaconBlock>(&justified_block_start)?
             .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*justified_block_start))?;
 
         let block_slot = block.slot;
@@ -400,8 +386,8 @@ impl<T: ClientDB + Sized, E: EthSpec> ForkChoice for OptimizedLMDGhost<T, E> {
             // didn't find head yet, proceed to next iteration
             // update block height
             block_height = self
-                .block_store
+                .store
-                .get_deserialized(&current_head)?
+                .get::<BeaconBlock>(&current_head)?
                 .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(current_head))?
                 .slot
                 .height(spec.genesis_slot);

eth2/fork_choice/src/slow_lmd_ghost.rs

@@ -1,44 +1,30 @@
-extern crate db;
-
 use crate::{ForkChoice, ForkChoiceError};
-use db::{
-    stores::{BeaconBlockStore, BeaconStateStore},
-    ClientDB,
-};
 use log::{debug, trace};
 use std::collections::HashMap;
 use std::marker::PhantomData;
 use std::sync::Arc;
+use store::Store;
 use types::{BeaconBlock, BeaconState, ChainSpec, EthSpec, Hash256, Slot};
 
 //TODO: Pruning and syncing
 
-pub struct SlowLMDGhost<T: ClientDB + Sized, E> {
+pub struct SlowLMDGhost<T, E> {
     /// The latest attestation targets as a map of validator index to block hash.
     //TODO: Could this be a fixed size vec
     latest_attestation_targets: HashMap<u64, Hash256>,
     /// Stores the children for any given parent.
     children: HashMap<Hash256, Vec<Hash256>>,
-    /// Block storage access.
+    /// Block and state storage.
-    block_store: Arc<BeaconBlockStore<T>>,
+    store: Arc<T>,
-    /// State storage access.
-    state_store: Arc<BeaconStateStore<T>>,
     _phantom: PhantomData<E>,
 }
 
-impl<T, E: EthSpec> SlowLMDGhost<T, E>
+impl<T: Store, E: EthSpec> SlowLMDGhost<T, E> {
-where
+    pub fn new(store: Arc<T>) -> Self {
-    T: ClientDB + Sized,
-{
-    pub fn new(
-        block_store: Arc<BeaconBlockStore<T>>,
-        state_store: Arc<BeaconStateStore<T>>,
-    ) -> Self {
         SlowLMDGhost {
             latest_attestation_targets: HashMap::new(),
             children: HashMap::new(),
-            block_store,
+            store,
-            state_store,
             _phantom: PhantomData,
         }
     }
@@ -58,8 +44,8 @@ where
         let mut latest_votes: HashMap<Hash256, u64> = HashMap::new();
         // gets the current weighted votes
         let current_state: BeaconState<E> = self
-            .state_store
+            .store
-            .get_deserialized(&state_root)?
+            .get(state_root)?
             .ok_or_else(|| ForkChoiceError::MissingBeaconState(*state_root))?;
 
         let active_validator_indices =
@@ -90,15 +76,15 @@ where
     ) -> Result<u64, ForkChoiceError> {
         let mut count = 0;
         let block_slot = self
-            .block_store
+            .store
-            .get_deserialized(&block_root)?
+            .get::<BeaconBlock>(&block_root)?
             .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*block_root))?
             .slot;
 
         for (vote_hash, votes) in latest_votes.iter() {
             let (root_at_slot, _) = self
-                .block_store
+                .store
-                .block_at_slot(&vote_hash, block_slot)?
+                .get_block_at_preceeding_slot(*vote_hash, block_slot)?
                 .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*block_root))?;
             if root_at_slot == *block_root {
                 count += votes;
@@ -108,7 +94,7 @@ where
     }
 }
 
-impl<T: ClientDB + Sized, E: EthSpec> ForkChoice for SlowLMDGhost<T, E> {
+impl<T: Store, E: EthSpec> ForkChoice for SlowLMDGhost<T, E> {
     /// Process when a block is added
     fn add_block(
         &mut self,
@@ -150,16 +136,16 @@ impl<T: ClientDB + Sized, E: EthSpec> ForkChoice for SlowLMDGhost<T, E> {
             trace!("Old attestation found: {:?}", attestation_target);
             // get the height of the target block
             let block_height = self
-                .block_store
+                .store
-                .get_deserialized(&target_block_root)?
+                .get::<BeaconBlock>(&target_block_root)?
                 .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*target_block_root))?
                 .slot
                 .height(spec.genesis_slot);
 
             // get the height of the past target block
             let past_block_height = self
-                .block_store
+                .store
-                .get_deserialized(&attestation_target)?
+                .get::<BeaconBlock>(&attestation_target)?
                 .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*attestation_target))?
                 .slot
                 .height(spec.genesis_slot);
@@ -180,8 +166,8 @@ impl<T: ClientDB + Sized, E: EthSpec> ForkChoice for SlowLMDGhost<T, E> {
     ) -> Result<Hash256, ForkChoiceError> {
         debug!("Running LMD Ghost Fork-choice rule");
         let start = self
-            .block_store
+            .store
-            .get_deserialized(&justified_block_start)?
+            .get::<BeaconBlock>(&justified_block_start)?
             .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*justified_block_start))?;
 
         let start_state_root = start.state_root;

eth2/fork_choice/tests/tests.rs

@@ -1,26 +1,14 @@
 #![cfg(not(debug_assertions))]
 // Tests the available fork-choice algorithms
 
-extern crate beacon_chain;
-extern crate bls;
-extern crate db;
-// extern crate env_logger; // for debugging
-extern crate fork_choice;
-extern crate hex;
-extern crate log;
-extern crate slot_clock;
-extern crate types;
-extern crate yaml_rust;
-
 pub use beacon_chain::BeaconChain;
 use bls::Signature;
-use db::stores::{BeaconBlockStore, BeaconStateStore};
+use store::MemoryStore;
-use db::MemoryDB;
+use store::Store;
 // use env_logger::{Builder, Env};
 use fork_choice::{
     BitwiseLMDGhost, ForkChoice, ForkChoiceAlgorithm, LongestChain, OptimizedLMDGhost, SlowLMDGhost,
 };
-use ssz::ssz_encode;
 use std::collections::HashMap;
 use std::sync::Arc;
 use std::{fs::File, io::prelude::*, path::PathBuf};
@@ -106,7 +94,7 @@ fn test_yaml_vectors(
     // process the tests
     for test_case in test_cases {
         // setup a fresh test
-        let (mut fork_choice, block_store, state_root) =
+        let (mut fork_choice, store, state_root) =
             setup_inital_state(&fork_choice_algo, emulated_validators);
 
         // keep a hashmap of block_id's to block_hashes (random hashes to abstract block_id)
@@ -149,9 +137,7 @@ fn test_yaml_vectors(
             };
 
             // Store the block.
-            block_store
+            store.put(&block_hash, &beacon_block).unwrap();
-                .put(&block_hash, &ssz_encode(&beacon_block)[..])
-                .unwrap();
 
             // run add block for fork choice if not genesis
             if parent_id != block_id {
@@ -222,29 +208,26 @@ fn load_test_cases_from_yaml(file_path: &str) -> Vec<yaml_rust::Yaml> {
 fn setup_inital_state(
     fork_choice_algo: &ForkChoiceAlgorithm,
     num_validators: usize,
-) -> (Box<ForkChoice>, Arc<BeaconBlockStore<MemoryDB>>, Hash256) {
+) -> (Box<ForkChoice>, Arc<MemoryStore>, Hash256) {
-    let db = Arc::new(MemoryDB::open());
+    let store = Arc::new(MemoryStore::open());
-    let block_store = Arc::new(BeaconBlockStore::new(db.clone()));
-    let state_store = Arc::new(BeaconStateStore::new(db.clone()));
 
     // the fork choice instantiation
     let fork_choice: Box<ForkChoice> = match fork_choice_algo {
         ForkChoiceAlgorithm::OptimizedLMDGhost => {
-            let f: OptimizedLMDGhost<MemoryDB, FoundationEthSpec> =
+            let f: OptimizedLMDGhost<MemoryStore, FoundationEthSpec> =
-                OptimizedLMDGhost::new(block_store.clone(), state_store.clone());
+                OptimizedLMDGhost::new(store.clone());
             Box::new(f)
         }
         ForkChoiceAlgorithm::BitwiseLMDGhost => {
-            let f: BitwiseLMDGhost<MemoryDB, FoundationEthSpec> =
+            let f: BitwiseLMDGhost<MemoryStore, FoundationEthSpec> =
-                BitwiseLMDGhost::new(block_store.clone(), state_store.clone());
+                BitwiseLMDGhost::new(store.clone());
             Box::new(f)
         }
         ForkChoiceAlgorithm::SlowLMDGhost => {
-            let f: SlowLMDGhost<MemoryDB, FoundationEthSpec> =
+            let f: SlowLMDGhost<MemoryStore, FoundationEthSpec> = SlowLMDGhost::new(store.clone());
-                SlowLMDGhost::new(block_store.clone(), state_store.clone());
             Box::new(f)
         }
-        ForkChoiceAlgorithm::LongestChain => Box::new(LongestChain::new(block_store.clone())),
+        ForkChoiceAlgorithm::LongestChain => Box::new(LongestChain::new(store.clone())),
     };
 
     let spec = FoundationEthSpec::spec();
@@ -255,12 +238,10 @@ fn setup_inital_state(
     let (state, _keypairs) = state_builder.build();
 
     let state_root = state.canonical_root();
-    state_store
+    store.put(&state_root, &state).unwrap();
-        .put(&state_root, &ssz_encode(&state)[..])
-        .unwrap();
 
     // return initialised vars
-    (fork_choice, block_store, state_root)
+    (fork_choice, store, state_root)
 }
 
 // convert a block_id into a Hash256 -- assume input is hex encoded;

eth2/state_processing/Cargo.toml

@@ -30,3 +30,6 @@ tree_hash = { path = "../utils/tree_hash" }
 tree_hash_derive = { path = "../utils/tree_hash_derive" }
 types = { path = "../types" }
 rayon = "1.0"
+
+[features]
+fake_crypto = ["bls/fake_crypto"]

eth2/state_processing/src/per_block_processing.rs

@@ -22,7 +22,9 @@ pub use verify_transfer::{
     execute_transfer, verify_transfer, verify_transfer_time_independent_only,
 };
 
+pub mod block_processing_builder;
 pub mod errors;
+pub mod tests;
 mod validate_attestation;
 mod verify_attester_slashing;
 mod verify_deposit;

eth2/state_processing/src/per_block_processing/block_processing_builder.rs

@@ -0,0 +1,66 @@
+use tree_hash::SignedRoot;
+use types::test_utils::{TestingBeaconBlockBuilder, TestingBeaconStateBuilder};
+use types::*;
+
+pub struct BlockProcessingBuilder<T: EthSpec> {
+    pub state_builder: TestingBeaconStateBuilder<T>,
+    pub block_builder: TestingBeaconBlockBuilder,
+
+    pub num_validators: usize,
+}
+
+impl<T: EthSpec> BlockProcessingBuilder<T> {
+    pub fn new(num_validators: usize, spec: &ChainSpec) -> Self {
+        let state_builder =
+            TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(num_validators, &spec);
+        let block_builder = TestingBeaconBlockBuilder::new(spec);
+
+        Self {
+            state_builder,
+            block_builder,
+            num_validators: 0,
+        }
+    }
+
+    pub fn set_slot(&mut self, slot: Slot, spec: &ChainSpec) {
+        self.state_builder.teleport_to_slot(slot, &spec);
+    }
+
+    pub fn build_caches(&mut self, spec: &ChainSpec) {
+        // Builds all caches; benches will not contain shuffling/committee building times.
+        self.state_builder.build_caches(&spec).unwrap();
+    }
+
+    pub fn build(
+        mut self,
+        randao_sk: Option<SecretKey>,
+        previous_block_root: Option<Hash256>,
+        spec: &ChainSpec,
+    ) -> (BeaconBlock, BeaconState<T>) {
+        let (state, keypairs) = self.state_builder.build();
+        let builder = &mut self.block_builder;
+
+        builder.set_slot(state.slot);
+
+        match previous_block_root {
+            Some(root) => builder.set_previous_block_root(root),
+            None => builder.set_previous_block_root(Hash256::from_slice(
+                &state.latest_block_header.signed_root(),
+            )),
+        }
+
+        let proposer_index = state
+            .get_beacon_proposer_index(state.slot, RelativeEpoch::Current, spec)
+            .unwrap();
+        let keypair = &keypairs[proposer_index];
+
+        match randao_sk {
+            Some(sk) => builder.set_randao_reveal(&sk, &state.fork, spec),
+            None => builder.set_randao_reveal(&keypair.sk, &state.fork, spec),
+        }
+
+        let block = self.block_builder.build(&keypair.sk, &state.fork, spec);
+
+        (block, state)
+    }
+}

eth2/state_processing/src/per_block_processing/tests.rs

@@ -0,0 +1,112 @@
+#![cfg(all(test, not(feature = "fake_crypto")))]
+use super::block_processing_builder::BlockProcessingBuilder;
+use super::errors::*;
+use crate::per_block_processing;
+use tree_hash::SignedRoot;
+use types::*;
+
+pub const VALIDATOR_COUNT: usize = 10;
+
+#[test]
+fn valid_block_ok() {
+    let spec = FoundationEthSpec::spec();
+    let builder = get_builder(&spec);
+    let (block, mut state) = builder.build(None, None, &spec);
+
+    let result = per_block_processing(&mut state, &block, &spec);
+
+    assert_eq!(result, Ok(()));
+}
+
+#[test]
+fn invalid_block_header_state_slot() {
+    let spec = FoundationEthSpec::spec();
+    let builder = get_builder(&spec);
+    let (mut block, mut state) = builder.build(None, None, &spec);
+
+    state.slot = Slot::new(133713);
+    block.slot = Slot::new(424242);
+
+    let result = per_block_processing(&mut state, &block, &spec);
+
+    assert_eq!(
+        result,
+        Err(BlockProcessingError::Invalid(
+            BlockInvalid::StateSlotMismatch
+        ))
+    );
+}
+
+#[test]
+fn invalid_parent_block_root() {
+    let spec = FoundationEthSpec::spec();
+    let builder = get_builder(&spec);
+    let invalid_parent_root = Hash256::from([0xAA; 32]);
+    let (block, mut state) = builder.build(None, Some(invalid_parent_root), &spec);
+
+    let result = per_block_processing(&mut state, &block, &spec);
+
+    assert_eq!(
+        result,
+        Err(BlockProcessingError::Invalid(
+            BlockInvalid::ParentBlockRootMismatch {
+                state: Hash256::from_slice(&state.latest_block_header.signed_root()),
+                block: block.previous_block_root
+            }
+        ))
+    );
+}
+
+#[test]
+fn invalid_block_signature() {
+    let spec = FoundationEthSpec::spec();
+    let builder = get_builder(&spec);
+    let (mut block, mut state) = builder.build(None, None, &spec);
+
+    // sign the block with a keypair that is not the expected proposer
+    let keypair = Keypair::random();
+    let message = block.signed_root();
+    let epoch = block.slot.epoch(spec.slots_per_epoch);
+    let domain = spec.get_domain(epoch, Domain::BeaconBlock, &state.fork);
+    block.signature = Signature::new(&message, domain, &keypair.sk);
+
+    // process block with invalid block signature
+    let result = per_block_processing(&mut state, &block, &spec);
+
+    // should get a BadSignature error
+    assert_eq!(
+        result,
+        Err(BlockProcessingError::Invalid(BlockInvalid::BadSignature))
+    );
+}
+
+#[test]
+fn invalid_randao_reveal_signature() {
+    let spec = FoundationEthSpec::spec();
+    let builder = get_builder(&spec);
+
+    // sign randao reveal with random keypair
+    let keypair = Keypair::random();
+    let (block, mut state) = builder.build(Some(keypair.sk), None, &spec);
+
+    let result = per_block_processing(&mut state, &block, &spec);
+
+    // should get a BadRandaoSignature error
+    assert_eq!(
+        result,
+        Err(BlockProcessingError::Invalid(
+            BlockInvalid::BadRandaoSignature
+        ))
+    );
+}
+
+fn get_builder(spec: &ChainSpec) -> (BlockProcessingBuilder<FoundationEthSpec>) {
+    let mut builder = BlockProcessingBuilder::new(VALIDATOR_COUNT, &spec);
+
+    // Set the state and block to be in the last slot of the 4th epoch.
+    let last_slot_of_epoch = (spec.genesis_epoch + 4).end_slot(spec.slots_per_epoch);
+    builder.set_slot(last_slot_of_epoch, &spec);
+    builder.build_caches(&spec);
+
+    (builder)
+}

eth2/types/src/test_utils/builders/testing_beacon_block_builder.rs

@@ -23,6 +23,11 @@ impl TestingBeaconBlockBuilder {
         }
     }
 
+    /// Set the previous block root
+    pub fn set_previous_block_root(&mut self, root: Hash256) {
+        self.block.previous_block_root = root;
+    }
+
     /// Set the slot of the block.
     pub fn set_slot(&mut self, slot: Slot) {
         self.block.slot = slot;
@@ -48,6 +53,11 @@ impl TestingBeaconBlockBuilder {
         self.block.body.randao_reveal = Signature::new(&message, domain, sk);
     }
 
+    /// Has the randao reveal been set?
+    pub fn randao_reveal_not_set(&mut self) -> bool {
+        self.block.body.randao_reveal.is_empty()
+    }
+
     /// Inserts a signed, valid `ProposerSlashing` for the validator.
     pub fn insert_proposer_slashing(
         &mut self,

eth2/utils/bls/src/fake_signature.rs

@@ -14,6 +14,7 @@ use tree_hash::tree_hash_ssz_encoding_as_vector;
 #[derive(Debug, PartialEq, Clone, Eq)]
 pub struct FakeSignature {
     bytes: Vec<u8>,
+    is_empty: bool,
 }
 
 impl FakeSignature {
@@ -26,6 +27,7 @@ impl FakeSignature {
     pub fn zero() -> Self {
         Self {
             bytes: vec![0; BLS_SIG_BYTE_SIZE],
+            is_empty: true,
         }
     }
 
@@ -59,6 +61,7 @@ impl FakeSignature {
         } else {
             Ok(Self {
                 bytes: bytes.to_vec(),
+                is_empty: false,
             })
         }
     }
@@ -71,6 +74,11 @@ impl FakeSignature {
     pub fn empty_signature() -> Self {
         FakeSignature::zero()
     }
+
+    // Check for empty Signature
+    pub fn is_empty(&self) -> bool {
+        self.is_empty
+    }
 }
 
 impl_ssz!(FakeSignature, BLS_SIG_BYTE_SIZE, "FakeSignature");