Refactor beacon chain start code

beacon_node/Cargo.toml

@@ -6,6 +6,7 @@ edition = "2018"
 
 [dependencies]
 eth2_config = { path = "../eth2/utils/eth2_config" }
+beacon_chain = { path = "beacon_chain" }
 types = { path = "../eth2/types" }
 store = { path = "./store" }
 client = { path = "client" }

beacon_node/beacon_chain/Cargo.toml

@@ -11,9 +11,11 @@ lazy_static = "1.3.0"
 lighthouse_metrics = { path = "../../eth2/utils/lighthouse_metrics" }
 log = "0.4"
 operation_pool = { path = "../../eth2/operation_pool" }
+reqwest = "0.9"
 serde = "1.0"
 serde_derive = "1.0"
 serde_yaml = "0.8"
+eth2-libp2p = { path = "../eth2-libp2p" }
 slog = { version = "^2.2.3" , features = ["max_level_trace"] }
 sloggers = { version = "^0.3" }
 slot_clock = { path = "../../eth2/utils/slot_clock" }
@@ -22,6 +24,7 @@ eth2_ssz_derive = "0.1"
 state_processing = { path = "../../eth2/state_processing" }
 tree_hash = "0.1"
 types = { path = "../../eth2/types" }
+url = "1.2"
 lmd_ghost = { path = "../../eth2/lmd_ghost" }
 
 [dev-dependencies]

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -114,7 +114,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     /// Instantiate a new Beacon Chain, from genesis.
     pub fn from_genesis(
         store: Arc<T::Store>,
-        slot_clock: T::SlotClock,
         mut genesis_state: BeaconState<T::EthSpec>,
         mut genesis_block: BeaconBlock<T::EthSpec>,
         spec: ChainSpec,
@@ -147,6 +146,13 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
               "genesis_block_root" => format!("{}", genesis_block_root),
         );
 
+        // Slot clock
+        let slot_clock = T::SlotClock::new(
+            spec.genesis_slot,
+            genesis_state.genesis_time,
+            spec.seconds_per_slot,
+        );
+
         Ok(Self {
             spec,
             slot_clock,

beacon_node/beacon_chain/src/beacon_chain_builder.rs

@@ -1,49 +1,115 @@
-use crate::BeaconChainTypes;
+use super::bootstrapper::Bootstrapper;
+use crate::{BeaconChain, BeaconChainTypes};
+use slog::Logger;
 use std::fs::File;
 use std::path::PathBuf;
+use std::sync::Arc;
 use std::time::SystemTime;
-use types::{
+use types::{test_utils::TestingBeaconStateBuilder, BeaconBlock, BeaconState, ChainSpec, EthSpec};
-    test_utils::TestingBeaconStateBuilder, BeaconBlock, BeaconState, ChainSpec, EthSpec, Hash256,
+
-};
+enum BuildStrategy<T: BeaconChainTypes> {
+    FromGenesis {
+        genesis_state: Box<BeaconState<T::EthSpec>>,
+        genesis_block: Box<BeaconBlock<T::EthSpec>>,
+    },
+    LoadFromStore,
+}
 
 pub struct BeaconChainBuilder<T: BeaconChainTypes> {
-    genesis_state: BeaconState<T::EthSpec>,
+    build_strategy: BuildStrategy<T>,
-    genesis_block: BeaconBlock<T::EthSpec>,
     spec: ChainSpec,
+    log: Logger,
 }
 
 impl<T: BeaconChainTypes> BeaconChainBuilder<T> {
-    pub fn recent_genesis(validator_count: usize, spec: ChainSpec) -> Self {
+    pub fn recent_genesis(validator_count: usize, spec: ChainSpec, log: Logger) -> Self {
-        Self::quick_start(recent_genesis_time(), validator_count, spec)
+        Self::quick_start(recent_genesis_time(), validator_count, spec, log)
     }
 
-    pub fn quick_start(genesis_time: u64, validator_count: usize, spec: ChainSpec) -> Self {
+    pub fn quick_start(
+        genesis_time: u64,
+        validator_count: usize,
+        spec: ChainSpec,
+        log: Logger,
+    ) -> Self {
         let (mut genesis_state, _keypairs) =
             TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(validator_count, &spec)
                 .build();
 
         genesis_state.genesis_time = genesis_time;
 
-        Self::from_genesis_state(genesis_state, spec)
+        Self::from_genesis_state(genesis_state, spec, log)
     }
 
-    pub fn yaml_state(file: PathBuf, spec: ChainSpec) -> Result<Self, String> {
+    pub fn yaml_state(file: &PathBuf, spec: ChainSpec, log: Logger) -> Result<Self, String> {
         let file = File::open(file.clone())
             .map_err(|e| format!("Unable to open YAML genesis state file {:?}: {:?}", file, e))?;
 
         let genesis_state = serde_yaml::from_reader(file)
             .map_err(|e| format!("Unable to parse YAML genesis state file: {:?}", e))?;
 
-        Ok(Self::from_genesis_state(genesis_state, spec))
+        Ok(Self::from_genesis_state(genesis_state, spec, log))
     }
 
-    pub fn from_genesis_state(genesis_state: BeaconState<T::EthSpec>, spec: ChainSpec) -> Self {
+    pub fn http_bootstrap(server: &str, spec: ChainSpec, log: Logger) -> Result<Self, String> {
-        Self {
+        let bootstrapper = Bootstrapper::from_server_string(server.to_string())
-            genesis_block: genesis_block(&genesis_state, &spec),
+            .map_err(|e| format!("Failed to initialize bootstrap client: {}", e))?;
-            genesis_state,
+
+        let (genesis_state, genesis_block) = bootstrapper
+            .genesis()
+            .map_err(|e| format!("Failed to bootstrap genesis state: {}", e))?;
+
+        Ok(Self {
+            build_strategy: BuildStrategy::FromGenesis {
+                genesis_block: Box::new(genesis_block),
+                genesis_state: Box::new(genesis_state),
+            },
             spec,
+            log,
+        })
+    }
+
+    fn from_genesis_state(
+        genesis_state: BeaconState<T::EthSpec>,
+        spec: ChainSpec,
+        log: Logger,
+    ) -> Self {
+        Self {
+            build_strategy: BuildStrategy::FromGenesis {
+                genesis_block: Box::new(genesis_block(&genesis_state, &spec)),
+                genesis_state: Box::new(genesis_state),
+            },
+            spec,
+            log,
         }
     }
+
+    pub fn from_store(spec: ChainSpec, log: Logger) -> Self {
+        Self {
+            build_strategy: BuildStrategy::LoadFromStore,
+            spec,
+            log,
+        }
+    }
+
+    pub fn build(self, store: Arc<T::Store>) -> Result<BeaconChain<T>, String> {
+        Ok(match self.build_strategy {
+            BuildStrategy::LoadFromStore => BeaconChain::from_store(store, self.spec, self.log)
+                .map_err(|e| format!("Error loading BeaconChain from database: {:?}", e))?
+                .ok_or_else(|| format!("Unable to find exising BeaconChain in database."))?,
+            BuildStrategy::FromGenesis {
+                genesis_block,
+                genesis_state,
+            } => BeaconChain::from_genesis(
+                store,
+                genesis_state.as_ref().clone(),
+                genesis_block.as_ref().clone(),
+                self.spec,
+                self.log,
+            )
+            .map_err(|e| format!("Failed to initialize new beacon chain: {:?}", e))?,
+        })
+    }
 }
 
 fn genesis_block<T: EthSpec>(genesis_state: &BeaconState<T>, spec: &ChainSpec) -> BeaconBlock<T> {

beacon_node/beacon_chain/src/lib.rs

@@ -4,6 +4,7 @@ extern crate lazy_static;
 
 mod beacon_chain;
 mod beacon_chain_builder;
+mod bootstrapper;
 mod checkpoint;
 mod errors;
 mod fork_choice;
@@ -18,6 +19,7 @@ pub use self::beacon_chain::{
 pub use self::checkpoint::CheckPoint;
 pub use self::errors::{BeaconChainError, BlockProductionError};
 pub use beacon_chain_builder::BeaconChainBuilder;
+pub use bootstrapper::Bootstrapper;
 pub use lmd_ghost;
 pub use metrics::scrape_for_metrics;
 pub use parking_lot;

beacon_node/beacon_chain/src/test_utils.rs

@@ -1,7 +1,6 @@
 use crate::{BeaconChain, BeaconChainTypes, BlockProcessingOutcome};
 use lmd_ghost::LmdGhost;
 use sloggers::{null::NullLoggerBuilder, Build};
-use slot_clock::SlotClock;
 use slot_clock::TestingSlotClock;
 use state_processing::per_slot_processing;
 use std::marker::PhantomData;
@@ -114,22 +113,9 @@ where
         let builder = NullLoggerBuilder;
         let log = builder.build().expect("logger should build");
 
-        // Slot clock
+        let chain =
-        let slot_clock = TestingSlotClock::new(
+            BeaconChain::from_genesis(store, genesis_state, genesis_block, spec.clone(), log)
-            spec.genesis_slot,
+                .expect("Terminate if beacon chain generation fails");
-            genesis_state.genesis_time,
-            spec.seconds_per_slot,
-        );
-
-        let chain = BeaconChain::from_genesis(
-            store,
-            slot_clock,
-            genesis_state,
-            genesis_block,
-            spec.clone(),
-            log,
-        )
-        .expect("Terminate if beacon chain generation fails");
 
         Self {
             chain,

beacon_node/client/Cargo.toml

@@ -27,5 +27,3 @@ clap = "2.32.0"
 dirs = "1.0.3"
 exit-future = "0.1.3"
 futures = "0.1.25"
-reqwest = "0.9"
-url = "1.2"

beacon_node/client/src/beacon_chain_types.rs

@@ -1,170 +0,0 @@
-use crate::bootstrapper::Bootstrapper;
-use crate::error::Result;
-use crate::{config::BeaconChainStartMethod, ClientConfig};
-use beacon_chain::{
-    lmd_ghost::{LmdGhost, ThreadSafeReducedTree},
-    slot_clock::SystemTimeSlotClock,
-    store::Store,
-    BeaconChain, BeaconChainTypes,
-};
-use slog::{crit, info, Logger};
-use slot_clock::SlotClock;
-use std::fs::File;
-use std::marker::PhantomData;
-use std::sync::Arc;
-use std::time::SystemTime;
-use tree_hash::TreeHash;
-use types::{
-    test_utils::TestingBeaconStateBuilder, BeaconBlock, BeaconState, ChainSpec, EthSpec, Hash256,
-};
-
-/// Provides a new, initialized `BeaconChain`
-pub trait InitialiseBeaconChain<T: BeaconChainTypes> {
-    fn initialise_beacon_chain(
-        store: Arc<T::Store>,
-        config: &ClientConfig,
-        spec: ChainSpec,
-        log: Logger,
-    ) -> Result<BeaconChain<T>> {
-        maybe_load_from_store_for_testnet::<_, T::Store, T::EthSpec>(store, config, spec, log)
-    }
-}
-
-#[derive(Clone)]
-pub struct ClientType<S: Store, E: EthSpec> {
-    _phantom_t: PhantomData<S>,
-    _phantom_u: PhantomData<E>,
-}
-
-impl<S, E> BeaconChainTypes for ClientType<S, E>
-where
-    S: Store + 'static,
-    E: EthSpec,
-{
-    type Store = S;
-    type SlotClock = SystemTimeSlotClock;
-    type LmdGhost = ThreadSafeReducedTree<S, E>;
-    type EthSpec = E;
-}
-impl<T: Store, E: EthSpec, X: BeaconChainTypes> InitialiseBeaconChain<X> for ClientType<T, E> {}
-
-/// Loads a `BeaconChain` from `store`, if it exists. Otherwise, create a new chain from genesis.
-fn maybe_load_from_store_for_testnet<T, U: Store, V: EthSpec>(
-    store: Arc<U>,
-    config: &ClientConfig,
-    spec: ChainSpec,
-    log: Logger,
-) -> Result<BeaconChain<T>>
-where
-    T: BeaconChainTypes<Store = U, EthSpec = V>,
-    T::LmdGhost: LmdGhost<U, V>,
-{
-    let genesis_state = match &config.beacon_chain_start_method {
-        BeaconChainStartMethod::Resume => unimplemented!("No resume code yet"),
-        BeaconChainStartMethod::Mainnet => {
-            crit!(log, "No mainnet beacon chain startup specification.");
-            return Err("Mainnet is not yet specified. We're working on it.".into());
-        }
-        BeaconChainStartMethod::RecentGenesis { validator_count } => {
-            generate_testnet_genesis_state(*validator_count, recent_genesis_time(), &spec)
-        }
-        BeaconChainStartMethod::Generated {
-            validator_count,
-            genesis_time,
-        } => generate_testnet_genesis_state(*validator_count, *genesis_time, &spec),
-        BeaconChainStartMethod::Yaml { file } => {
-            let file = File::open(file).map_err(|e| {
-                format!("Unable to open YAML genesis state file {:?}: {:?}", file, e)
-            })?;
-
-            serde_yaml::from_reader(file)
-                .map_err(|e| format!("Unable to parse YAML genesis state file: {:?}", e))?
-        }
-        BeaconChainStartMethod::HttpBootstrap { server, .. } => {
-            let bootstrapper = Bootstrapper::from_server_string(server.to_string())
-                .map_err(|e| format!("Failed to initialize bootstrap client: {}", e))?;
-
-            let (state, _block) = bootstrapper
-                .genesis()
-                .map_err(|e| format!("Failed to bootstrap genesis state: {}", e))?;
-
-            state
-        }
-    };
-
-    let mut genesis_block = BeaconBlock::empty(&spec);
-    genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root());
-    let genesis_block_root = genesis_block.canonical_root();
-
-    // Slot clock
-    let slot_clock = T::SlotClock::new(
-        spec.genesis_slot,
-        genesis_state.genesis_time,
-        spec.seconds_per_slot,
-    );
-
-    // Try load an existing `BeaconChain` from the store. If unable, create a new one.
-    if let Ok(Some(beacon_chain)) =
-        BeaconChain::from_store(store.clone(), spec.clone(), log.clone())
-    {
-        // Here we check to ensure that the `BeaconChain` loaded from store has the expected
-        // genesis block.
-        //
-        // Without this check, it's possible that there will be an existing DB with a `BeaconChain`
-        // that has different parameters than provided to this executable.
-        if beacon_chain.genesis_block_root == genesis_block_root {
-            info!(
-                log,
-                "Loaded BeaconChain from store";
-                "slot" => beacon_chain.head().beacon_state.slot,
-                "best_slot" => beacon_chain.best_slot(),
-            );
-
-            Ok(beacon_chain)
-        } else {
-            crit!(
-                log,
-                "The BeaconChain loaded from disk has an incorrect genesis root. \
-                 This may be caused by an old database in located in datadir."
-            );
-            Err("Incorrect genesis root".into())
-        }
-    } else {
-        BeaconChain::from_genesis(
-            store,
-            slot_clock,
-            genesis_state,
-            genesis_block,
-            spec,
-            log.clone(),
-        )
-        .map_err(|e| format!("Failed to initialize new beacon chain: {:?}", e).into())
-    }
-}
-
-fn generate_testnet_genesis_state<E: EthSpec>(
-    validator_count: usize,
-    genesis_time: u64,
-    spec: &ChainSpec,
-) -> BeaconState<E> {
-    let (mut genesis_state, _keypairs) =
-        TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(validator_count, spec)
-            .build();
-
-    genesis_state.genesis_time = genesis_time;
-
-    genesis_state
-}
-
-/// Returns the system time, mod 30 minutes.
-///
-/// Used for easily creating testnets.
-fn recent_genesis_time() -> u64 {
-    let now = SystemTime::now()
-        .duration_since(SystemTime::UNIX_EPOCH)
-        .unwrap()
-        .as_secs();
-    let secs_after_last_period = now.checked_rem(30 * 60).unwrap_or(0);
-    // genesis is now the last 30 minute block.
-    now - secs_after_last_period
-}

beacon_node/client/src/lib.rs

@@ -1,31 +1,47 @@
 extern crate slog;
 
-mod beacon_chain_types;
-mod bootstrapper;
 mod config;
 
 pub mod error;
 pub mod notifier;
 
-use beacon_chain::BeaconChain;
+use beacon_chain::{
+    lmd_ghost::ThreadSafeReducedTree, slot_clock::SystemTimeSlotClock, store::Store, BeaconChain,
+    BeaconChainBuilder,
+};
 use exit_future::Signal;
 use futures::{future::Future, Stream};
 use network::Service as NetworkService;
-use slog::{error, info, o};
+use slog::{crit, error, info, o};
 use slot_clock::SlotClock;
 use std::marker::PhantomData;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 use tokio::runtime::TaskExecutor;
 use tokio::timer::Interval;
+use types::EthSpec;
 
 pub use beacon_chain::BeaconChainTypes;
-pub use beacon_chain_types::ClientType;
-pub use beacon_chain_types::InitialiseBeaconChain;
-pub use bootstrapper::Bootstrapper;
 pub use config::{BeaconChainStartMethod, Config as ClientConfig};
 pub use eth2_config::Eth2Config;
 
+#[derive(Clone)]
+pub struct ClientType<S: Store, E: EthSpec> {
+    _phantom_t: PhantomData<S>,
+    _phantom_u: PhantomData<E>,
+}
+
+impl<S, E> BeaconChainTypes for ClientType<S, E>
+where
+    S: Store + 'static,
+    E: EthSpec,
+{
+    type Store = S;
+    type SlotClock = SystemTimeSlotClock;
+    type LmdGhost = ThreadSafeReducedTree<S, E>;
+    type EthSpec = E;
+}
+
 /// Main beacon node client service. This provides the connection and initialisation of the clients
 /// sub-services in multiple threads.
 pub struct Client<T: BeaconChainTypes> {
@@ -49,7 +65,7 @@ pub struct Client<T: BeaconChainTypes> {
 
 impl<T> Client<T>
 where
-    T: BeaconChainTypes + InitialiseBeaconChain<T> + Clone,
+    T: BeaconChainTypes + Clone,
 {
     /// Generate an instance of the client. Spawn and link all internal sub-processes.
     pub fn new(
@@ -62,13 +78,41 @@ where
         let store = Arc::new(store);
         let seconds_per_slot = eth2_config.spec.seconds_per_slot;
 
-        // Load a `BeaconChain` from the store, or create a new one if it does not exist.
+        let spec = &eth2_config.spec.clone();
-        let beacon_chain = Arc::new(T::initialise_beacon_chain(
+
-            store,
+        let beacon_chain_builder = match &client_config.beacon_chain_start_method {
-            &client_config,
+            BeaconChainStartMethod::Resume => {
-            eth2_config.spec.clone(),
+                BeaconChainBuilder::from_store(spec.clone(), log.clone())
-            log.clone(),
+            }
-        )?);
+            BeaconChainStartMethod::Mainnet => {
+                crit!(log, "No mainnet beacon chain startup specification.");
+                return Err("Mainnet is not yet specified. We're working on it.".into());
+            }
+            BeaconChainStartMethod::RecentGenesis { validator_count } => {
+                BeaconChainBuilder::recent_genesis(*validator_count, spec.clone(), log.clone())
+            }
+            BeaconChainStartMethod::Generated {
+                validator_count,
+                genesis_time,
+            } => BeaconChainBuilder::quick_start(
+                *genesis_time,
+                *validator_count,
+                spec.clone(),
+                log.clone(),
+            ),
+            BeaconChainStartMethod::Yaml { file } => {
+                BeaconChainBuilder::yaml_state(file, spec.clone(), log.clone())?
+            }
+            BeaconChainStartMethod::HttpBootstrap { server, .. } => {
+                BeaconChainBuilder::http_bootstrap(server, spec.clone(), log.clone())?
+            }
+        };
+
+        let beacon_chain: Arc<BeaconChain<T>> = Arc::new(
+            beacon_chain_builder
+                .build(store)
+                .map_err(error::Error::from)?,
+        );
 
         if beacon_chain.read_slot_clock().is_none() {
             panic!("Cannot start client before genesis!")

beacon_node/src/config.rs

@@ -1,5 +1,6 @@
+use beacon_chain::Bootstrapper;
 use clap::ArgMatches;
-use client::{BeaconChainStartMethod, Bootstrapper, ClientConfig, Eth2Config};
+use client::{BeaconChainStartMethod, ClientConfig, Eth2Config};
 use eth2_config::{read_from_file, write_to_file};
 use rand::{distributions::Alphanumeric, Rng};
 use slog::{crit, info, warn, Logger};

beacon_node/src/run.rs

@@ -1,7 +1,4 @@
-use client::{
+use client::{error, notifier, BeaconChainTypes, Client, ClientConfig, ClientType, Eth2Config};
-    error, notifier, BeaconChainTypes, Client, ClientConfig, ClientType, Eth2Config,
-    InitialiseBeaconChain,
-};
 use futures::sync::oneshot;
 use futures::Future;
 use slog::{error, info};
@@ -117,7 +114,7 @@ fn run<T>(
     log: &slog::Logger,
 ) -> error::Result<()>
 where
-    T: BeaconChainTypes + InitialiseBeaconChain<T> + Clone,
+    T: BeaconChainTypes + Clone,
     T::Store: OpenDatabase,
 {
     let store = T::Store::open_database(&db_path)?;