Merge branch 'interop' into millsecond-slots

beacon_node/Cargo.toml

@@ -6,11 +6,13 @@ edition = "2018"
 
 [dependencies]
 eth2_config = { path = "../eth2/utils/eth2_config" }
+beacon_chain = { path = "beacon_chain" }
 types = { path = "../eth2/types" }
 store = { path = "./store" }
 client = { path = "client" }
 version = { path = "version" }
 clap = "2.32.0"
+rand = "0.7"
 slog = { version = "^2.2.3" , features = ["max_level_trace", "release_max_level_trace"] }
 slog-term = "^2.4.0"
 slog-async = "^2.3.0"

beacon_node/beacon_chain/Cargo.toml

@@ -5,14 +5,18 @@ authors = ["Paul Hauner <paul@paulhauner.com>", "Age Manning <Age@AgeManning.com
 edition = "2018"
 
 [dependencies]
+eth2_config = { path = "../../eth2/utils/eth2_config" }
 store = { path = "../store" }
 parking_lot = "0.7"
 lazy_static = "1.3.0"
 lighthouse_metrics = { path = "../../eth2/utils/lighthouse_metrics" }
 log = "0.4"
 operation_pool = { path = "../../eth2/operation_pool" }
+reqwest = "0.9"
 serde = "1.0"
 serde_derive = "1.0"
+serde_yaml = "0.8"
+eth2-libp2p = { path = "../eth2-libp2p" }
 slog = { version = "^2.2.3" , features = ["max_level_trace"] }
 sloggers = { version = "^0.3" }
 slot_clock = { path = "../../eth2/utils/slot_clock" }
@@ -21,6 +25,7 @@ eth2_ssz_derive = "0.1"
 state_processing = { path = "../../eth2/state_processing" }
 tree_hash = "0.1"
 types = { path = "../../eth2/types" }
+url = "1.2"
 lmd_ghost = { path = "../../eth2/lmd_ghost" }
 
 [dev-dependencies]

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -115,7 +115,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     /// Instantiate a new Beacon Chain, from genesis.
     pub fn from_genesis(
         store: Arc<T::Store>,
-        slot_clock: T::SlotClock,
         mut genesis_state: BeaconState<T::EthSpec>,
         mut genesis_block: BeaconBlock<T::EthSpec>,
         spec: ChainSpec,
@@ -148,6 +147,13 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
               "genesis_block_root" => format!("{}", genesis_block_root),
         );
 
+        // Slot clock
+        let slot_clock = T::SlotClock::new(
+            spec.genesis_slot,
+            genesis_state.genesis_time,
+            spec.seconds_per_slot,
+        );
+
         Ok(Self {
             spec,
             slot_clock,

beacon_node/beacon_chain/src/beacon_chain_builder.rs

@@ -0,0 +1,134 @@
+use super::bootstrapper::Bootstrapper;
+use crate::{BeaconChain, BeaconChainTypes};
+use slog::Logger;
+use std::fs::File;
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::time::SystemTime;
+use types::{test_utils::TestingBeaconStateBuilder, BeaconBlock, BeaconState, ChainSpec, EthSpec};
+
+enum BuildStrategy<T: BeaconChainTypes> {
+    FromGenesis {
+        genesis_state: Box<BeaconState<T::EthSpec>>,
+        genesis_block: Box<BeaconBlock<T::EthSpec>>,
+    },
+    LoadFromStore,
+}
+
+pub struct BeaconChainBuilder<T: BeaconChainTypes> {
+    build_strategy: BuildStrategy<T>,
+    spec: ChainSpec,
+    log: Logger,
+}
+
+impl<T: BeaconChainTypes> BeaconChainBuilder<T> {
+    pub fn recent_genesis(validator_count: usize, spec: ChainSpec, log: Logger) -> Self {
+        Self::quick_start(recent_genesis_time(), validator_count, spec, log)
+    }
+
+    pub fn quick_start(
+        genesis_time: u64,
+        validator_count: usize,
+        spec: ChainSpec,
+        log: Logger,
+    ) -> Self {
+        let (mut genesis_state, _keypairs) =
+            TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(validator_count, &spec)
+                .build();
+
+        genesis_state.genesis_time = genesis_time;
+
+        Self::from_genesis_state(genesis_state, spec, log)
+    }
+
+    pub fn yaml_state(file: &PathBuf, spec: ChainSpec, log: Logger) -> Result<Self, String> {
+        let file = File::open(file.clone())
+            .map_err(|e| format!("Unable to open YAML genesis state file {:?}: {:?}", file, e))?;
+
+        let genesis_state = serde_yaml::from_reader(file)
+            .map_err(|e| format!("Unable to parse YAML genesis state file: {:?}", e))?;
+
+        Ok(Self::from_genesis_state(genesis_state, spec, log))
+    }
+
+    pub fn http_bootstrap(server: &str, spec: ChainSpec, log: Logger) -> Result<Self, String> {
+        let bootstrapper = Bootstrapper::from_server_string(server.to_string())
+            .map_err(|e| format!("Failed to initialize bootstrap client: {}", e))?;
+
+        let (genesis_state, genesis_block) = bootstrapper
+            .genesis()
+            .map_err(|e| format!("Failed to bootstrap genesis state: {}", e))?;
+
+        Ok(Self {
+            build_strategy: BuildStrategy::FromGenesis {
+                genesis_block: Box::new(genesis_block),
+                genesis_state: Box::new(genesis_state),
+            },
+            spec,
+            log,
+        })
+    }
+
+    fn from_genesis_state(
+        genesis_state: BeaconState<T::EthSpec>,
+        spec: ChainSpec,
+        log: Logger,
+    ) -> Self {
+        Self {
+            build_strategy: BuildStrategy::FromGenesis {
+                genesis_block: Box::new(genesis_block(&genesis_state, &spec)),
+                genesis_state: Box::new(genesis_state),
+            },
+            spec,
+            log,
+        }
+    }
+
+    pub fn from_store(spec: ChainSpec, log: Logger) -> Self {
+        Self {
+            build_strategy: BuildStrategy::LoadFromStore,
+            spec,
+            log,
+        }
+    }
+
+    pub fn build(self, store: Arc<T::Store>) -> Result<BeaconChain<T>, String> {
+        Ok(match self.build_strategy {
+            BuildStrategy::LoadFromStore => BeaconChain::from_store(store, self.spec, self.log)
+                .map_err(|e| format!("Error loading BeaconChain from database: {:?}", e))?
+                .ok_or_else(|| format!("Unable to find exising BeaconChain in database."))?,
+            BuildStrategy::FromGenesis {
+                genesis_block,
+                genesis_state,
+            } => BeaconChain::from_genesis(
+                store,
+                genesis_state.as_ref().clone(),
+                genesis_block.as_ref().clone(),
+                self.spec,
+                self.log,
+            )
+            .map_err(|e| format!("Failed to initialize new beacon chain: {:?}", e))?,
+        })
+    }
+}
+
+fn genesis_block<T: EthSpec>(genesis_state: &BeaconState<T>, spec: &ChainSpec) -> BeaconBlock<T> {
+    let mut genesis_block = BeaconBlock::empty(&spec);
+
+    genesis_block.state_root = genesis_state.canonical_root();
+
+    genesis_block
+}
+
+/// Returns the system time, mod 30 minutes.
+///
+/// Used for easily creating testnets.
+fn recent_genesis_time() -> u64 {
+    let now = SystemTime::now()
+        .duration_since(SystemTime::UNIX_EPOCH)
+        .unwrap()
+        .as_secs();
+    let secs_after_last_period = now.checked_rem(30 * 60).unwrap_or(0);
+    // genesis is now the last 30 minute block.
+    now - secs_after_last_period
+}

beacon_node/beacon_chain/src/bootstrapper.rs

@@ -0,0 +1,233 @@
+use eth2_config::Eth2Config;
+use eth2_libp2p::{
+    multiaddr::{Multiaddr, Protocol},
+    Enr,
+};
+use reqwest::{Error as HttpError, Url};
+use serde::Deserialize;
+use std::borrow::Cow;
+use std::net::Ipv4Addr;
+use types::{BeaconBlock, BeaconState, Checkpoint, EthSpec, Hash256, Slot};
+use url::Host;
+
+#[derive(Debug)]
+enum Error {
+    InvalidUrl,
+    HttpError(HttpError),
+}
+
+impl From<HttpError> for Error {
+    fn from(e: HttpError) -> Error {
+        Error::HttpError(e)
+    }
+}
+
+/// Used to load "bootstrap" information from the HTTP API of another Lighthouse beacon node.
+///
+/// Bootstrapping information includes things like genesis and finalized states and blocks, and
+/// libp2p connection details.
+pub struct Bootstrapper {
+    url: Url,
+}
+
+impl Bootstrapper {
+    /// Parses the given `server` as a URL, instantiating `Self`.
+    pub fn from_server_string(server: String) -> Result<Self, String> {
+        Ok(Self {
+            url: Url::parse(&server).map_err(|e| format!("Invalid bootstrap server url: {}", e))?,
+        })
+    }
+
+    /// Build a multiaddr using the HTTP server URL that is not guaranteed to be correct.
+    ///
+    /// The address is created by querying the HTTP server for its listening libp2p addresses.
+    /// Then, we find the first TCP port in those addresses and combine the port with the URL of
+    /// the server.
+    ///
+    /// For example, the server `http://192.168.0.1` might end up with a `best_effort_multiaddr` of
+    /// `/ipv4/192.168.0.1/tcp/9000` if the server advertises a listening address of
+    /// `/ipv4/172.0.0.1/tcp/9000`.
+    pub fn best_effort_multiaddr(&self, port: Option<u16>) -> Option<Multiaddr> {
+        let tcp_port = if let Some(port) = port {
+            port
+        } else {
+            self.listen_port().ok()?
+        };
+
+        let mut multiaddr = Multiaddr::with_capacity(2);
+
+        match self.url.host()? {
+            Host::Ipv4(addr) => multiaddr.push(Protocol::Ip4(addr)),
+            Host::Domain(s) => multiaddr.push(Protocol::Dns4(Cow::Borrowed(s))),
+            _ => return None,
+        };
+
+        multiaddr.push(Protocol::Tcp(tcp_port));
+
+        Some(multiaddr)
+    }
+
+    /// Returns the IPv4 address of the server URL, unless it contains a FQDN.
+    pub fn server_ipv4_addr(&self) -> Option<Ipv4Addr> {
+        match self.url.host()? {
+            Host::Ipv4(addr) => Some(addr),
+            _ => None,
+        }
+    }
+
+    /// Returns the servers Eth2Config.
+    pub fn eth2_config(&self) -> Result<Eth2Config, String> {
+        get_eth2_config(self.url.clone()).map_err(|e| format!("Unable to get Eth2Config: {:?}", e))
+    }
+
+    /// Returns the servers ENR address.
+    pub fn enr(&self) -> Result<Enr, String> {
+        get_enr(self.url.clone()).map_err(|e| format!("Unable to get ENR: {:?}", e))
+    }
+
+    /// Returns the servers listening libp2p addresses.
+    pub fn listen_port(&self) -> Result<u16, String> {
+        get_listen_port(self.url.clone()).map_err(|e| format!("Unable to get listen port: {:?}", e))
+    }
+
+    /// Returns the genesis block and state.
+    pub fn genesis<T: EthSpec>(&self) -> Result<(BeaconState<T>, BeaconBlock<T>), String> {
+        let genesis_slot = Slot::new(0);
+
+        let block = get_block(self.url.clone(), genesis_slot)
+            .map_err(|e| format!("Unable to get genesis block: {:?}", e))?
+            .beacon_block;
+        let state = get_state(self.url.clone(), genesis_slot)
+            .map_err(|e| format!("Unable to get genesis state: {:?}", e))?
+            .beacon_state;
+
+        Ok((state, block))
+    }
+
+    /// Returns the most recent finalized state and block.
+    pub fn finalized<T: EthSpec>(&self) -> Result<(BeaconState<T>, BeaconBlock<T>), String> {
+        let slots_per_epoch = get_slots_per_epoch(self.url.clone())
+            .map_err(|e| format!("Unable to get slots per epoch: {:?}", e))?;
+        let finalized_slot = get_finalized_slot(self.url.clone(), slots_per_epoch.as_u64())
+            .map_err(|e| format!("Unable to get finalized slot: {:?}", e))?;
+
+        let block = get_block(self.url.clone(), finalized_slot)
+            .map_err(|e| format!("Unable to get finalized block: {:?}", e))?
+            .beacon_block;
+        let state = get_state(self.url.clone(), finalized_slot)
+            .map_err(|e| format!("Unable to get finalized state: {:?}", e))?
+            .beacon_state;
+
+        Ok((state, block))
+    }
+}
+
+fn get_slots_per_epoch(mut url: Url) -> Result<Slot, Error> {
+    url.path_segments_mut()
+        .map(|mut url| {
+            url.push("spec").push("slots_per_epoch");
+        })
+        .map_err(|_| Error::InvalidUrl)?;
+
+    reqwest::get(url)?
+        .error_for_status()?
+        .json()
+        .map_err(Into::into)
+}
+
+fn get_eth2_config(mut url: Url) -> Result<Eth2Config, Error> {
+    url.path_segments_mut()
+        .map(|mut url| {
+            url.push("spec").push("eth2_config");
+        })
+        .map_err(|_| Error::InvalidUrl)?;
+
+    reqwest::get(url)?
+        .error_for_status()?
+        .json()
+        .map_err(Into::into)
+}
+
+fn get_finalized_slot(mut url: Url, slots_per_epoch: u64) -> Result<Slot, Error> {
+    url.path_segments_mut()
+        .map(|mut url| {
+            url.push("beacon").push("latest_finalized_checkpoint");
+        })
+        .map_err(|_| Error::InvalidUrl)?;
+
+    let checkpoint: Checkpoint = reqwest::get(url)?.error_for_status()?.json()?;
+
+    Ok(checkpoint.epoch.start_slot(slots_per_epoch))
+}
+
+#[derive(Deserialize)]
+#[serde(bound = "T: EthSpec")]
+pub struct StateResponse<T: EthSpec> {
+    pub root: Hash256,
+    pub beacon_state: BeaconState<T>,
+}
+
+fn get_state<T: EthSpec>(mut url: Url, slot: Slot) -> Result<StateResponse<T>, Error> {
+    url.path_segments_mut()
+        .map(|mut url| {
+            url.push("beacon").push("state");
+        })
+        .map_err(|_| Error::InvalidUrl)?;
+
+    url.query_pairs_mut()
+        .append_pair("slot", &format!("{}", slot.as_u64()));
+
+    reqwest::get(url)?
+        .error_for_status()?
+        .json()
+        .map_err(Into::into)
+}
+
+#[derive(Deserialize)]
+#[serde(bound = "T: EthSpec")]
+pub struct BlockResponse<T: EthSpec> {
+    pub root: Hash256,
+    pub beacon_block: BeaconBlock<T>,
+}
+
+fn get_block<T: EthSpec>(mut url: Url, slot: Slot) -> Result<BlockResponse<T>, Error> {
+    url.path_segments_mut()
+        .map(|mut url| {
+            url.push("beacon").push("block");
+        })
+        .map_err(|_| Error::InvalidUrl)?;
+
+    url.query_pairs_mut()
+        .append_pair("slot", &format!("{}", slot.as_u64()));
+
+    reqwest::get(url)?
+        .error_for_status()?
+        .json()
+        .map_err(Into::into)
+}
+
+fn get_enr(mut url: Url) -> Result<Enr, Error> {
+    url.path_segments_mut()
+        .map(|mut url| {
+            url.push("network").push("enr");
+        })
+        .map_err(|_| Error::InvalidUrl)?;
+
+    reqwest::get(url)?
+        .error_for_status()?
+        .json()
+        .map_err(Into::into)
+}
+
+fn get_listen_port(mut url: Url) -> Result<u16, Error> {
+    url.path_segments_mut()
+        .map(|mut url| {
+            url.push("network").push("listen_port");
+        })
+        .map_err(|_| Error::InvalidUrl)?;
+
+    reqwest::get(url)?
+        .error_for_status()?
+        .json()
+        .map_err(Into::into)
+}

beacon_node/beacon_chain/src/lib.rs

@@ -3,6 +3,8 @@
 extern crate lazy_static;
 
 mod beacon_chain;
+mod beacon_chain_builder;
+mod bootstrapper;
 mod checkpoint;
 mod errors;
 mod fork_choice;
@@ -16,6 +18,8 @@ pub use self::beacon_chain::{
 };
 pub use self::checkpoint::CheckPoint;
 pub use self::errors::{BeaconChainError, BlockProductionError};
+pub use beacon_chain_builder::BeaconChainBuilder;
+pub use bootstrapper::Bootstrapper;
 pub use lmd_ghost;
 pub use metrics::scrape_for_metrics;
 pub use parking_lot;

beacon_node/beacon_chain/src/test_utils.rs

@@ -1,12 +1,10 @@
 use crate::{BeaconChain, BeaconChainTypes, BlockProcessingOutcome};
 use lmd_ghost::LmdGhost;
 use sloggers::{null::NullLoggerBuilder, Build};
-use slot_clock::SlotClock;
 use slot_clock::TestingSlotClock;
 use state_processing::per_slot_processing;
 use std::marker::PhantomData;
 use std::sync::Arc;
-use std::time::Duration;
 use store::MemoryStore;
 use store::Store;
 use tree_hash::{SignedRoot, TreeHash};
@@ -115,23 +113,9 @@ where
         let builder = NullLoggerBuilder;
         let log = builder.build().expect("logger should build");
 
-        // Slot clock
+        let chain =
-        let slot_clock = TestingSlotClock::from_eth2_genesis(
+            BeaconChain::from_genesis(store, genesis_state, genesis_block, spec.clone(), log)
-            spec.genesis_slot,
+                .expect("Terminate if beacon chain generation fails");
-            genesis_state.genesis_time,
-            Duration::from_secs(spec.seconds_per_slot),
-        )
-        .expect("Slot clock should start");
-
-        let chain = BeaconChain::from_genesis(
-            store,
-            slot_clock,
-            genesis_state,
-            genesis_block,
-            spec.clone(),
-            log,
-        )
-        .expect("Terminate if beacon chain generation fails");
 
         Self {
             chain,

beacon_node/client/src/beacon_chain_types.rs

@@ -1,169 +0,0 @@
-use crate::bootstrapper::Bootstrapper;
-use crate::error::Result;
-use crate::{config::GenesisState, ClientConfig};
-use beacon_chain::{
-    lmd_ghost::{LmdGhost, ThreadSafeReducedTree},
-    slot_clock::SystemTimeSlotClock,
-    store::Store,
-    BeaconChain, BeaconChainTypes,
-};
-use slog::{crit, info, Logger};
-use slot_clock::SlotClock;
-use std::fs::File;
-use std::marker::PhantomData;
-use std::sync::Arc;
-use std::time::SystemTime;
-use tree_hash::TreeHash;
-use types::{
-    test_utils::TestingBeaconStateBuilder, BeaconBlock, BeaconState, ChainSpec, EthSpec, Hash256,
-};
-
-/// Provides a new, initialized `BeaconChain`
-pub trait InitialiseBeaconChain<T: BeaconChainTypes> {
-    fn initialise_beacon_chain(
-        store: Arc<T::Store>,
-        config: &ClientConfig,
-        spec: ChainSpec,
-        log: Logger,
-    ) -> Result<BeaconChain<T>> {
-        maybe_load_from_store_for_testnet::<_, T::Store, T::EthSpec>(store, config, spec, log)
-    }
-}
-
-#[derive(Clone)]
-pub struct ClientType<S: Store, E: EthSpec> {
-    _phantom_t: PhantomData<S>,
-    _phantom_u: PhantomData<E>,
-}
-
-impl<S, E> BeaconChainTypes for ClientType<S, E>
-where
-    S: Store + 'static,
-    E: EthSpec,
-{
-    type Store = S;
-    type SlotClock = SystemTimeSlotClock;
-    type LmdGhost = ThreadSafeReducedTree<S, E>;
-    type EthSpec = E;
-}
-impl<T: Store, E: EthSpec, X: BeaconChainTypes> InitialiseBeaconChain<X> for ClientType<T, E> {}
-
-/// Loads a `BeaconChain` from `store`, if it exists. Otherwise, create a new chain from genesis.
-fn maybe_load_from_store_for_testnet<T, U: Store, V: EthSpec>(
-    store: Arc<U>,
-    config: &ClientConfig,
-    spec: ChainSpec,
-    log: Logger,
-) -> Result<BeaconChain<T>>
-where
-    T: BeaconChainTypes<Store = U, EthSpec = V>,
-    T::LmdGhost: LmdGhost<U, V>,
-{
-    let genesis_state = match &config.genesis_state {
-        GenesisState::Mainnet => {
-            crit!(log, "This release does not support mainnet genesis state.");
-            return Err("Mainnet is unsupported".into());
-        }
-        GenesisState::RecentGenesis { validator_count } => {
-            generate_testnet_genesis_state(*validator_count, recent_genesis_time(), &spec)
-        }
-        GenesisState::Generated {
-            validator_count,
-            genesis_time,
-        } => generate_testnet_genesis_state(*validator_count, *genesis_time, &spec),
-        GenesisState::Yaml { file } => {
-            let file = File::open(file).map_err(|e| {
-                format!("Unable to open YAML genesis state file {:?}: {:?}", file, e)
-            })?;
-
-            serde_yaml::from_reader(file)
-                .map_err(|e| format!("Unable to parse YAML genesis state file: {:?}", e))?
-        }
-        GenesisState::HttpBootstrap { server } => {
-            let bootstrapper = Bootstrapper::from_server_string(server.to_string())
-                .map_err(|e| format!("Failed to initialize bootstrap client: {}", e))?;
-
-            let (state, _block) = bootstrapper
-                .genesis()
-                .map_err(|e| format!("Failed to bootstrap genesis state: {}", e))?;
-
-            state
-        }
-    };
-
-    let mut genesis_block = BeaconBlock::empty(&spec);
-    genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root());
-    let genesis_block_root = genesis_block.canonical_root();
-
-    // Slot clock
-    let slot_clock = T::SlotClock::new(
-        spec.genesis_slot,
-        genesis_state.genesis_time,
-        spec.seconds_per_slot,
-    );
-
-    // Try load an existing `BeaconChain` from the store. If unable, create a new one.
-    if let Ok(Some(beacon_chain)) =
-        BeaconChain::from_store(store.clone(), spec.clone(), log.clone())
-    {
-        // Here we check to ensure that the `BeaconChain` loaded from store has the expected
-        // genesis block.
-        //
-        // Without this check, it's possible that there will be an existing DB with a `BeaconChain`
-        // that has different parameters than provided to this executable.
-        if beacon_chain.genesis_block_root == genesis_block_root {
-            info!(
-                log,
-                "Loaded BeaconChain from store";
-                "slot" => beacon_chain.head().beacon_state.slot,
-                "best_slot" => beacon_chain.best_slot(),
-            );
-
-            Ok(beacon_chain)
-        } else {
-            crit!(
-                log,
-                "The BeaconChain loaded from disk has an incorrect genesis root. \
-                 This may be caused by an old database in located in datadir."
-            );
-            Err("Incorrect genesis root".into())
-        }
-    } else {
-        BeaconChain::from_genesis(
-            store,
-            slot_clock,
-            genesis_state,
-            genesis_block,
-            spec,
-            log.clone(),
-        )
-        .map_err(|e| format!("Failed to initialize new beacon chain: {:?}", e).into())
-    }
-}
-
-fn generate_testnet_genesis_state<E: EthSpec>(
-    validator_count: usize,
-    genesis_time: u64,
-    spec: &ChainSpec,
-) -> BeaconState<E> {
-    let (mut genesis_state, _keypairs) =
-        TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(validator_count, spec)
-            .build();
-
-    genesis_state.genesis_time = genesis_time;
-
-    genesis_state
-}
-
-/// Returns the system time, mod 30 minutes.
-///
-/// Used for easily creating testnets.
-fn recent_genesis_time() -> u64 {
-    let now = SystemTime::now()
-        .duration_since(SystemTime::UNIX_EPOCH)
-        .unwrap()
-        .as_secs();
-    let secs_after_last_period = now.checked_rem(30 * 60).unwrap_or(0);
-    // genesis is now the last 30 minute block.
-    now - secs_after_last_period
-}

beacon_node/client/src/config.rs

@@ -1,15 +1,11 @@
-use crate::{Bootstrapper, Eth2Config};
 use clap::ArgMatches;
 use network::NetworkConfig;
 use serde_derive::{Deserialize, Serialize};
-use slog::{info, o, warn, Drain};
+use slog::{info, o, Drain};
 use std::fs::{self, OpenOptions};
 use std::path::PathBuf;
 use std::sync::Mutex;
 
-/// The number initial validators when starting the `Minimal`.
-const TESTNET_VALIDATOR_COUNT: usize = 16;
-
 /// The number initial validators when starting the `Minimal`.
 const TESTNET_SPEC_CONSTANTS: &str = "minimal";
 
@@ -21,33 +17,49 @@ pub struct Config {
     db_name: String,
     pub log_file: PathBuf,
     pub spec_constants: String,
-    pub genesis_state: GenesisState,
+    /// Defines how we should initialize a BeaconChain instances.
+    ///
+    /// This field is not serialized, there for it will not be written to (or loaded from) config
+    /// files. It can only be configured via the CLI.
+    #[serde(skip)]
+    pub beacon_chain_start_method: BeaconChainStartMethod,
     pub network: network::NetworkConfig,
     pub rpc: rpc::RPCConfig,
     pub rest_api: rest_api::ApiConfig,
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+/// Defines how the client should initialize a BeaconChain.
-#[serde(tag = "type")]
+///
-pub enum GenesisState {
+/// In general, there are two methods:
-    /// Use the mainnet genesis state.
+///  - resuming a new chain, or
-    ///
+///  - initializing a new one.
-    /// Mainnet genesis state is not presently known, so this is a place-holder.
+#[derive(Debug, Clone)]
+pub enum BeaconChainStartMethod {
+    /// Resume from an existing BeaconChain, loaded from the existing local database.
+    Resume,
+    /// Resume from an existing BeaconChain, loaded from the existing local database.
     Mainnet,
-    /// Generate a state with `validator_count` validators, all with well-known secret keys.
+    /// Create a new beacon chain that can connect to mainnet.
     ///
     /// Set the genesis time to be the start of the previous 30-minute window.
     RecentGenesis { validator_count: usize },
-    /// Generate a state with `genesis_time` and `validator_count` validators, all with well-known
+    /// Create a new beacon chain with `genesis_time` and `validator_count` validators, all with well-known
     /// secret keys.
     Generated {
         validator_count: usize,
         genesis_time: u64,
     },
-    /// Load a YAML-encoded genesis state from a file.
+    /// Create a new beacon chain by loading a YAML-encoded genesis state from a file.
     Yaml { file: PathBuf },
-    /// Use a HTTP server (running our REST-API) to load genesis and finalized states and blocks.
+    /// Create a new beacon chain by using a HTTP server (running our REST-API) to load genesis and
-    HttpBootstrap { server: String },
+    /// finalized states and blocks.
+    HttpBootstrap { server: String, port: Option<u16> },
+}
+
+impl Default for BeaconChainStartMethod {
+    fn default() -> Self {
+        BeaconChainStartMethod::Resume
+    }
 }
 
 impl Default for Config {
@@ -58,12 +70,10 @@ impl Default for Config {
             db_type: "disk".to_string(),
             db_name: "chain_db".to_string(),
             network: NetworkConfig::new(),
-            rpc: rpc::RPCConfig::default(),
+            rpc: <_>::default(),
-            rest_api: rest_api::ApiConfig::default(),
+            rest_api: <_>::default(),
             spec_constants: TESTNET_SPEC_CONSTANTS.into(),
-            genesis_state: GenesisState::RecentGenesis {
+            beacon_chain_start_method: <_>::default(),
-                validator_count: TESTNET_VALIDATOR_COUNT,
-            },
         }
     }
 }
@@ -127,15 +137,6 @@ impl Config {
             self.data_dir = PathBuf::from(dir);
         };
 
-        if let Some(default_spec) = args.value_of("default-spec") {
-            match default_spec {
-                "mainnet" => self.spec_constants = Eth2Config::mainnet().spec_constants,
-                "minimal" => self.spec_constants = Eth2Config::minimal().spec_constants,
-                "interop" => self.spec_constants = Eth2Config::interop().spec_constants,
-                _ => {} // not supported
-            }
-        }
-
         if let Some(dir) = args.value_of("db") {
             self.db_type = dir.to_string();
         };
@@ -149,40 +150,6 @@ impl Config {
             self.update_logger(log)?;
         };
 
-        // If the `--bootstrap` flag is provided, overwrite the default configuration.
-        if let Some(server) = args.value_of("bootstrap") {
-            do_bootstrapping(self, server.to_string(), &log)?;
-        }
-
         Ok(())
     }
 }
-
-/// Perform the HTTP bootstrapping procedure, reading an ENR and multiaddr from the HTTP server and
-/// adding them to the `config`.
-fn do_bootstrapping(config: &mut Config, server: String, log: &slog::Logger) -> Result<(), String> {
-    // Set the genesis state source.
-    config.genesis_state = GenesisState::HttpBootstrap {
-        server: server.to_string(),
-    };
-
-    let bootstrapper = Bootstrapper::from_server_string(server.to_string())?;
-
-    config.network.boot_nodes.push(bootstrapper.enr()?);
-
-    if let Some(server_multiaddr) = bootstrapper.best_effort_multiaddr() {
-        info!(
-            log,
-            "Estimated bootstrapper libp2p address";
-            "multiaddr" => format!("{:?}", server_multiaddr)
-        );
-        config.network.libp2p_nodes.push(server_multiaddr);
-    } else {
-        warn!(
-            log,
-            "Unable to estimate a bootstrapper libp2p address, this node may not find any peers."
-        );
-    }
-
-    Ok(())
-}

beacon_node/client/src/lib.rs

@@ -1,31 +1,47 @@
 extern crate slog;
 
-mod beacon_chain_types;
-mod bootstrapper;
 mod config;
 
 pub mod error;
 pub mod notifier;
 
-use beacon_chain::BeaconChain;
+use beacon_chain::{
+    lmd_ghost::ThreadSafeReducedTree, slot_clock::SystemTimeSlotClock, store::Store, BeaconChain,
+    BeaconChainBuilder,
+};
 use exit_future::Signal;
 use futures::{future::Future, Stream};
 use network::Service as NetworkService;
-use slog::{error, info, o};
+use slog::{crit, error, info, o};
 use slot_clock::SlotClock;
 use std::marker::PhantomData;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 use tokio::runtime::TaskExecutor;
 use tokio::timer::Interval;
+use types::EthSpec;
 
 pub use beacon_chain::BeaconChainTypes;
-pub use beacon_chain_types::ClientType;
+pub use config::{BeaconChainStartMethod, Config as ClientConfig};
-pub use beacon_chain_types::InitialiseBeaconChain;
-pub use bootstrapper::Bootstrapper;
-pub use config::{Config as ClientConfig, GenesisState};
 pub use eth2_config::Eth2Config;
 
+#[derive(Clone)]
+pub struct ClientType<S: Store, E: EthSpec> {
+    _phantom_t: PhantomData<S>,
+    _phantom_u: PhantomData<E>,
+}
+
+impl<S, E> BeaconChainTypes for ClientType<S, E>
+where
+    S: Store + 'static,
+    E: EthSpec,
+{
+    type Store = S;
+    type SlotClock = SystemTimeSlotClock;
+    type LmdGhost = ThreadSafeReducedTree<S, E>;
+    type EthSpec = E;
+}
+
 /// Main beacon node client service. This provides the connection and initialisation of the clients
 /// sub-services in multiple threads.
 pub struct Client<T: BeaconChainTypes> {
@@ -49,7 +65,7 @@ pub struct Client<T: BeaconChainTypes> {
 
 impl<T> Client<T>
 where
-    T: BeaconChainTypes + InitialiseBeaconChain<T> + Clone,
+    T: BeaconChainTypes + Clone,
 {
     /// Generate an instance of the client. Spawn and link all internal sub-processes.
     pub fn new(
@@ -62,13 +78,41 @@ where
         let store = Arc::new(store);
         let seconds_per_slot = eth2_config.spec.seconds_per_slot;
 
-        // Load a `BeaconChain` from the store, or create a new one if it does not exist.
+        let spec = &eth2_config.spec.clone();
-        let beacon_chain = Arc::new(T::initialise_beacon_chain(
+
-            store,
+        let beacon_chain_builder = match &client_config.beacon_chain_start_method {
-            &client_config,
+            BeaconChainStartMethod::Resume => {
-            eth2_config.spec.clone(),
+                BeaconChainBuilder::from_store(spec.clone(), log.clone())
-            log.clone(),
+            }
-        )?);
+            BeaconChainStartMethod::Mainnet => {
+                crit!(log, "No mainnet beacon chain startup specification.");
+                return Err("Mainnet is not yet specified. We're working on it.".into());
+            }
+            BeaconChainStartMethod::RecentGenesis { validator_count } => {
+                BeaconChainBuilder::recent_genesis(*validator_count, spec.clone(), log.clone())
+            }
+            BeaconChainStartMethod::Generated {
+                validator_count,
+                genesis_time,
+            } => BeaconChainBuilder::quick_start(
+                *genesis_time,
+                *validator_count,
+                spec.clone(),
+                log.clone(),
+            ),
+            BeaconChainStartMethod::Yaml { file } => {
+                BeaconChainBuilder::yaml_state(file, spec.clone(), log.clone())?
+            }
+            BeaconChainStartMethod::HttpBootstrap { server, .. } => {
+                BeaconChainBuilder::http_bootstrap(server, spec.clone(), log.clone())?
+            }
+        };
+
+        let beacon_chain: Arc<BeaconChain<T>> = Arc::new(
+            beacon_chain_builder
+                .build(store)
+                .map_err(error::Error::from)?,
+        );
 
         if beacon_chain.read_slot_clock().is_none() {
             panic!("Cannot start client before genesis!")
@@ -118,6 +162,7 @@ where
                 beacon_chain.clone(),
                 network.clone(),
                 client_config.db_path().expect("unable to read datadir"),
+                eth2_config.clone(),
                 &log,
             ) {
                 Ok(s) => Some(s),

beacon_node/rest_api/Cargo.toml

@@ -27,5 +27,6 @@ exit-future = "0.1.3"
 tokio = "0.1.17"
 url = "2.0"
 lazy_static = "1.3.0"
+eth2_config = { path = "../../eth2/utils/eth2_config" }
 lighthouse_metrics = { path = "../../eth2/utils/lighthouse_metrics" }
 slot_clock = { path = "../../eth2/utils/slot_clock" }

beacon_node/rest_api/src/lib.rs

@@ -13,6 +13,7 @@ mod url_query;
 
 use beacon_chain::{BeaconChain, BeaconChainTypes};
 use client_network::Service as NetworkService;
+use eth2_config::Eth2Config;
 use hyper::rt::Future;
 use hyper::service::service_fn_ok;
 use hyper::{Body, Method, Response, Server, StatusCode};
@@ -79,6 +80,7 @@ pub fn start_server<T: BeaconChainTypes>(
     beacon_chain: Arc<BeaconChain<T>>,
     network_service: Arc<NetworkService<T>>,
     db_path: PathBuf,
+    eth2_config: Eth2Config,
     log: &slog::Logger,
 ) -> Result<exit_future::Signal, hyper::Error> {
     let log = log.new(o!("Service" => "Api"));
@@ -100,12 +102,14 @@ pub fn start_server<T: BeaconChainTypes>(
     // Clone our stateful objects, for use in service closure.
     let server_log = log.clone();
     let server_bc = beacon_chain.clone();
+    let eth2_config = Arc::new(eth2_config);
 
     let service = move || {
         let log = server_log.clone();
         let beacon_chain = server_bc.clone();
         let db_path = db_path.clone();
         let network_service = network_service.clone();
+        let eth2_config = eth2_config.clone();
 
         // Create a simple handler for the router, inject our stateful objects into the request.
         service_fn_ok(move |mut req| {
@@ -118,6 +122,8 @@ pub fn start_server<T: BeaconChainTypes>(
             req.extensions_mut().insert::<DBPath>(db_path.clone());
             req.extensions_mut()
                 .insert::<Arc<NetworkService<T>>>(network_service.clone());
+            req.extensions_mut()
+                .insert::<Arc<Eth2Config>>(eth2_config.clone());
 
             let path = req.uri().path().to_string();
 
@@ -144,6 +150,7 @@ pub fn start_server<T: BeaconChainTypes>(
                 (&Method::GET, "/node/genesis_time") => node::get_genesis_time::<T>(req),
                 (&Method::GET, "/spec") => spec::get_spec::<T>(req),
                 (&Method::GET, "/spec/slots_per_epoch") => spec::get_slots_per_epoch::<T>(req),
+                (&Method::GET, "/spec/eth2_config") => spec::get_eth2_config::<T>(req),
                 _ => Err(ApiError::MethodNotAllowed(path.clone())),
             };
 

beacon_node/rest_api/src/spec.rs

@@ -1,6 +1,7 @@
 use super::{success_response, ApiResult};
 use crate::ApiError;
 use beacon_chain::{BeaconChain, BeaconChainTypes};
+use eth2_config::Eth2Config;
 use hyper::{Body, Request};
 use std::sync::Arc;
 use types::EthSpec;
@@ -18,6 +19,19 @@ pub fn get_spec<T: BeaconChainTypes + 'static>(req: Request<Body>) -> ApiResult
     Ok(success_response(Body::from(json)))
 }
 
+/// HTTP handler to return the full Eth2Config object.
+pub fn get_eth2_config<T: BeaconChainTypes + 'static>(req: Request<Body>) -> ApiResult {
+    let eth2_config = req
+        .extensions()
+        .get::<Arc<Eth2Config>>()
+        .ok_or_else(|| ApiError::ServerError("Eth2Config extension missing".to_string()))?;
+
+    let json: String = serde_json::to_string(eth2_config.as_ref())
+        .map_err(|e| ApiError::ServerError(format!("Unable to serialize Eth2Config: {:?}", e)))?;
+
+    Ok(success_response(Body::from(json)))
+}
+
 /// HTTP handler to return the full spec object.
 pub fn get_slots_per_epoch<T: BeaconChainTypes + 'static>(_req: Request<Body>) -> ApiResult {
     let json: String = serde_json::to_string(&T::EthSpec::slots_per_epoch())

beacon_node/src/config.rs

@@ -0,0 +1,435 @@
+use beacon_chain::Bootstrapper;
+use clap::ArgMatches;
+use client::{BeaconChainStartMethod, ClientConfig, Eth2Config};
+use eth2_config::{read_from_file, write_to_file};
+use rand::{distributions::Alphanumeric, Rng};
+use slog::{crit, info, warn, Logger};
+use std::fs;
+use std::path::{Path, PathBuf};
+
+pub const DEFAULT_DATA_DIR: &str = ".lighthouse";
+pub const CLIENT_CONFIG_FILENAME: &str = "beacon-node.toml";
+pub const ETH2_CONFIG_FILENAME: &str = "eth2-spec.toml";
+
+type Result<T> = std::result::Result<T, String>;
+type Config = (ClientConfig, Eth2Config);
+
+/// Gets the fully-initialized global client and eth2 configuration objects.
+pub fn get_configs(cli_args: &ArgMatches, log: &Logger) -> Result<Config> {
+    let mut builder = ConfigBuilder::new(cli_args, log)?;
+
+    match cli_args.subcommand() {
+        ("testnet", Some(sub_cmd_args)) => {
+            process_testnet_subcommand(&mut builder, sub_cmd_args, log)?
+        }
+        // No sub-command assumes a resume operation.
+        _ => {
+            info!(
+                log,
+                "Resuming from existing datadir";
+                "path" => format!("{:?}", builder.data_dir)
+            );
+
+            // If no primary subcommand was given, start the beacon chain from an existing
+            // database.
+            builder.set_beacon_chain_start_method(BeaconChainStartMethod::Resume);
+
+            // Whilst there is no large testnet or mainnet force the user to specify how they want
+            // to start a new chain (e.g., from a genesis YAML file, another node, etc).
+            if !builder.data_dir.exists() {
+                return Err(
+                    "No datadir found. To start a new beacon chain, see `testnet --help`. \
+                     Use `--datadir` to specify a different directory"
+                        .into(),
+                );
+            }
+
+            // If the `testnet` command was not provided, attempt to load an existing datadir and
+            // continue with an existing chain.
+            builder.load_from_datadir()?;
+        }
+    };
+
+    builder.build(cli_args)
+}
+
+/// Process the `testnet` CLI subcommand arguments, updating the `builder`.
+fn process_testnet_subcommand(
+    builder: &mut ConfigBuilder,
+    cli_args: &ArgMatches,
+    log: &Logger,
+) -> Result<()> {
+    if cli_args.is_present("random-datadir") {
+        builder.set_random_datadir()?;
+    }
+
+    let is_bootstrap = cli_args.subcommand_name() == Some("bootstrap");
+
+    if let Some(path_string) = cli_args.value_of("eth2-config") {
+        if is_bootstrap {
+            return Err("Cannot supply --eth2-config when using bootsrap".to_string());
+        }
+
+        let path = path_string
+            .parse::<PathBuf>()
+            .map_err(|e| format!("Unable to parse eth2-config path: {:?}", e))?;
+        builder.load_eth2_config(path)?;
+    } else {
+        builder.update_spec_from_subcommand(&cli_args)?;
+    }
+
+    if let Some(path_string) = cli_args.value_of("client-config") {
+        let path = path_string
+            .parse::<PathBuf>()
+            .map_err(|e| format!("Unable to parse client config path: {:?}", e))?;
+        builder.load_client_config(path)?;
+    }
+
+    if cli_args.is_present("force") {
+        builder.clean_datadir()?;
+    }
+
+    info!(
+        log,
+        "Creating new datadir";
+        "path" => format!("{:?}", builder.data_dir)
+    );
+
+    // Start matching on the second subcommand (e.g., `testnet bootstrap ...`)
+    match cli_args.subcommand() {
+        ("bootstrap", Some(cli_args)) => {
+            let server = cli_args
+                .value_of("server")
+                .ok_or_else(|| "No bootstrap server specified")?;
+            let port: Option<u16> = cli_args
+                .value_of("libp2p-port")
+                .and_then(|s| s.parse::<u16>().ok());
+
+            builder.import_bootstrap_libp2p_address(server, port)?;
+            builder.import_bootstrap_eth2_config(server)?;
+
+            builder.set_beacon_chain_start_method(BeaconChainStartMethod::HttpBootstrap {
+                server: server.to_string(),
+                port,
+            })
+        }
+        ("recent", Some(cli_args)) => {
+            let validator_count = cli_args
+                .value_of("validator_count")
+                .ok_or_else(|| "No validator_count specified")?
+                .parse::<usize>()
+                .map_err(|e| format!("Unable to parse validator_count: {:?}", e))?;
+
+            builder.set_beacon_chain_start_method(BeaconChainStartMethod::RecentGenesis {
+                validator_count,
+            })
+        }
+        _ => return Err("No testnet method specified. See 'testnet --help'.".into()),
+    };
+
+    builder.write_configs_to_new_datadir()?;
+
+    Ok(())
+}
+
+/// Allows for building a set of configurations based upon `clap` arguments.
+struct ConfigBuilder<'a> {
+    log: &'a Logger,
+    pub data_dir: PathBuf,
+    eth2_config: Eth2Config,
+    client_config: ClientConfig,
+}
+
+impl<'a> ConfigBuilder<'a> {
+    /// Create a new builder with default settings.
+    pub fn new(cli_args: &'a ArgMatches, log: &'a Logger) -> Result<Self> {
+        // Read the `--datadir` flag.
+        //
+        // If it's not present, try and find the home directory (`~`) and push the default data
+        // directory onto it.
+        let data_dir: PathBuf = cli_args
+            .value_of("datadir")
+            .map(|string| PathBuf::from(string))
+            .or_else(|| {
+                dirs::home_dir().map(|mut home| {
+                    home.push(DEFAULT_DATA_DIR);
+                    home
+                })
+            })
+            .ok_or_else(|| "Unable to find a home directory for the datadir".to_string())?;
+
+        Ok(Self {
+            log,
+            data_dir,
+            eth2_config: Eth2Config::minimal(),
+            client_config: ClientConfig::default(),
+        })
+    }
+
+    /// Clears any configuration files that would interfere with writing new configs.
+    ///
+    /// Moves the following files in `data_dir` into a backup directory:
+    ///
+    /// - Client config
+    /// - Eth2 config
+    /// - The entire database directory
+    pub fn clean_datadir(&mut self) -> Result<()> {
+        let backup_dir = {
+            let mut s = String::from("backup_");
+            s.push_str(&random_string(6));
+            self.data_dir.join(s)
+        };
+
+        fs::create_dir_all(&backup_dir)
+            .map_err(|e| format!("Unable to create config backup dir: {:?}", e))?;
+
+        let move_to_backup_dir = |path: &Path| -> Result<()> {
+            let file_name = path
+                .file_name()
+                .ok_or_else(|| "Invalid path found during datadir clean (no filename).")?;
+
+            let mut new = path.to_path_buf();
+            new.pop();
+            new.push(backup_dir.clone());
+            new.push(file_name);
+
+            let _ = fs::rename(path, new);
+
+            Ok(())
+        };
+
+        move_to_backup_dir(&self.data_dir.join(CLIENT_CONFIG_FILENAME))?;
+        move_to_backup_dir(&self.data_dir.join(ETH2_CONFIG_FILENAME))?;
+
+        if let Some(db_path) = self.client_config.db_path() {
+            move_to_backup_dir(&db_path)?;
+        }
+
+        Ok(())
+    }
+
+    /// Sets the method for starting the beacon chain.
+    pub fn set_beacon_chain_start_method(&mut self, method: BeaconChainStartMethod) {
+        self.client_config.beacon_chain_start_method = method;
+    }
+
+    /// Import the libp2p address for `server` into the list of bootnodes in `self`.
+    ///
+    /// If `port` is `Some`, it is used as the port for the `Multiaddr`. If `port` is `None`,
+    /// attempts to connect to the `server` via HTTP and retrieve it's libp2p listen port.
+    pub fn import_bootstrap_libp2p_address(
+        &mut self,
+        server: &str,
+        port: Option<u16>,
+    ) -> Result<()> {
+        let bootstrapper = Bootstrapper::from_server_string(server.to_string())?;
+
+        if let Some(server_multiaddr) = bootstrapper.best_effort_multiaddr(port) {
+            info!(
+                self.log,
+                "Estimated bootstrapper libp2p address";
+                "multiaddr" => format!("{:?}", server_multiaddr)
+            );
+
+            self.client_config
+                .network
+                .libp2p_nodes
+                .push(server_multiaddr);
+        } else {
+            warn!(
+                self.log,
+                "Unable to estimate a bootstrapper libp2p address, this node may not find any peers."
+            );
+        };
+
+        Ok(())
+    }
+
+    /// Set the config data_dir to be an random directory.
+    ///
+    /// Useful for easily spinning up ephemeral testnets.
+    pub fn set_random_datadir(&mut self) -> Result<()> {
+        let mut s = DEFAULT_DATA_DIR.to_string();
+        s.push_str("_random_");
+        s.push_str(&random_string(6));
+
+        self.data_dir.pop();
+        self.data_dir.push(s);
+
+        Ok(())
+    }
+
+    /// Imports an `Eth2Config` from `server`, returning an error if this fails.
+    pub fn import_bootstrap_eth2_config(&mut self, server: &str) -> Result<()> {
+        let bootstrapper = Bootstrapper::from_server_string(server.to_string())?;
+
+        self.update_eth2_config(bootstrapper.eth2_config()?);
+
+        Ok(())
+    }
+
+    fn update_eth2_config(&mut self, eth2_config: Eth2Config) {
+        self.eth2_config = eth2_config;
+    }
+
+    /// Reads the subcommand and tries to update `self.eth2_config` based up on the `--spec` flag.
+    ///
+    /// Returns an error if the `--spec` flag is not present in the given `cli_args`.
+    pub fn update_spec_from_subcommand(&mut self, cli_args: &ArgMatches) -> Result<()> {
+        // Re-initialise the `Eth2Config`.
+        //
+        // If a CLI parameter is set, overwrite any config file present.
+        // If a parameter is not set, use either the config file present or default to minimal.
+        let eth2_config = match cli_args.value_of("spec") {
+            Some("mainnet") => Eth2Config::mainnet(),
+            Some("minimal") => Eth2Config::minimal(),
+            Some("interop") => Eth2Config::interop(),
+            _ => return Err("Unable to determine specification type.".into()),
+        };
+
+        self.client_config.spec_constants = cli_args
+            .value_of("spec")
+            .expect("Guarded by prior match statement")
+            .to_string();
+        self.eth2_config = eth2_config;
+
+        Ok(())
+    }
+
+    /// Writes the configs in `self` to `self.data_dir`.
+    ///
+    /// Returns an error if `self.data_dir` already exists.
+    pub fn write_configs_to_new_datadir(&mut self) -> Result<()> {
+        let db_exists = self
+            .client_config
+            .db_path()
+            .map(|d| d.exists())
+            .unwrap_or_else(|| false);
+
+        // Do not permit creating a new config when the datadir exists.
+        if db_exists {
+            return Err("Database already exists. See `-f` in `testnet --help`".into());
+        }
+
+        // Create `datadir` and any non-existing parent directories.
+        fs::create_dir_all(&self.data_dir).map_err(|e| {
+            crit!(self.log, "Failed to initialize data dir"; "error" => format!("{}", e));
+            format!("{}", e)
+        })?;
+
+        let client_config_file = self.data_dir.join(CLIENT_CONFIG_FILENAME);
+        if client_config_file.exists() {
+            return Err(format!(
+                "Datadir is not clean, {} exists. See `-f` in `testnet --help`.",
+                CLIENT_CONFIG_FILENAME
+            ));
+        } else {
+            // Write the onfig to a TOML file in the datadir.
+            write_to_file(
+                self.data_dir.join(CLIENT_CONFIG_FILENAME),
+                &self.client_config,
+            )
+            .map_err(|e| format!("Unable to write {} file: {:?}", CLIENT_CONFIG_FILENAME, e))?;
+        }
+
+        let eth2_config_file = self.data_dir.join(ETH2_CONFIG_FILENAME);
+        if eth2_config_file.exists() {
+            return Err(format!(
+                "Datadir is not clean, {} exists. See `-f` in `testnet --help`.",
+                ETH2_CONFIG_FILENAME
+            ));
+        } else {
+            // Write the config to a TOML file in the datadir.
+            write_to_file(self.data_dir.join(ETH2_CONFIG_FILENAME), &self.eth2_config)
+                .map_err(|e| format!("Unable to write {} file: {:?}", ETH2_CONFIG_FILENAME, e))?;
+        }
+
+        Ok(())
+    }
+
+    /// Attempts to load the client and eth2 configs from `self.data_dir`.
+    ///
+    /// Returns an error if any files are not found or are invalid.
+    pub fn load_from_datadir(&mut self) -> Result<()> {
+        // Check to ensure the datadir exists.
+        //
+        // For now we return an error. In the future we may decide to boot a default (e.g.,
+        // public testnet or mainnet).
+        if !self.data_dir.exists() {
+            return Err(
+                "No datadir found. Either create a new testnet or specify a different `--datadir`."
+                    .into(),
+            );
+        }
+
+        // If there is a path to a databse in the config, ensure it exists.
+        if !self
+            .client_config
+            .db_path()
+            .map(|path| path.exists())
+            .unwrap_or_else(|| true)
+        {
+            return Err(
+                "No database found in datadir. Use 'testnet -f' to overwrite the existing \
+                 datadir, or specify a different `--datadir`."
+                    .into(),
+            );
+        }
+
+        self.load_eth2_config(self.data_dir.join(ETH2_CONFIG_FILENAME))?;
+        self.load_client_config(self.data_dir.join(CLIENT_CONFIG_FILENAME))?;
+
+        Ok(())
+    }
+
+    /// Attempts to load the client config from `path`.
+    ///
+    /// Returns an error if any files are not found or are invalid.
+    pub fn load_client_config(&mut self, path: PathBuf) -> Result<()> {
+        self.client_config = read_from_file::<ClientConfig>(path.clone())
+            .map_err(|e| format!("Unable to parse {:?} file: {:?}", path, e))?
+            .ok_or_else(|| format!("{:?} file does not exist", path))?;
+
+        Ok(())
+    }
+
+    /// Attempts to load the eth2 config from `path`.
+    ///
+    /// Returns an error if any files are not found or are invalid.
+    pub fn load_eth2_config(&mut self, path: PathBuf) -> Result<()> {
+        self.eth2_config = read_from_file::<Eth2Config>(path.clone())
+            .map_err(|e| format!("Unable to parse {:?} file: {:?}", path, e))?
+            .ok_or_else(|| format!("{:?} file does not exist", path))?;
+
+        Ok(())
+    }
+
+    /// Consumes self, returning the configs.
+    ///
+    /// The supplied `cli_args` should be the base-level `clap` cli_args (i.e., not a subcommand
+    /// cli_args).
+    pub fn build(mut self, cli_args: &ArgMatches) -> Result<Config> {
+        self.eth2_config.apply_cli_args(cli_args)?;
+        self.client_config
+            .apply_cli_args(cli_args, &mut self.log.clone())?;
+
+        if self.eth2_config.spec_constants != self.client_config.spec_constants {
+            crit!(self.log, "Specification constants do not match.";
+                  "client_config" => format!("{}", self.client_config.spec_constants),
+                  "eth2_config" => format!("{}", self.eth2_config.spec_constants)
+            );
+            return Err("Specification constant mismatch".into());
+        }
+
+        self.client_config.data_dir = self.data_dir;
+
+        Ok((self.client_config, self.eth2_config))
+    }
+}
+
+fn random_string(len: usize) -> String {
+    rand::thread_rng()
+        .sample_iter(&Alphanumeric)
+        .take(len)
+        .collect::<String>()
+}

beacon_node/src/main.rs

@@ -1,12 +1,10 @@
+mod config;
 mod run;
 
-use clap::{App, Arg};
+use clap::{App, Arg, SubCommand};
-use client::{ClientConfig, Eth2Config};
+use config::get_configs;
 use env_logger::{Builder, Env};
-use eth2_config::{read_from_file, write_to_file};
 use slog::{crit, o, warn, Drain, Level};
-use std::fs;
-use std::path::PathBuf;
 
 pub const DEFAULT_DATA_DIR: &str = ".lighthouse";
 pub const CLIENT_CONFIG_FILENAME: &str = "beacon-node.toml";
@@ -30,6 +28,7 @@ fn main() {
                 .value_name("DIR")
                 .help("Data directory for keys and databases.")
                 .takes_value(true)
+                .global(true)
         )
         .arg(
             Arg::with_name("logfile")
@@ -44,6 +43,7 @@ fn main() {
                 .value_name("NETWORK-DIR")
                 .help("Data directory for network keys.")
                 .takes_value(true)
+                .global(true)
         )
         /*
          * Network parameters.
@@ -160,25 +160,7 @@ fn main() {
                 .help("Type of database to use.")
                 .takes_value(true)
                 .possible_values(&["disk", "memory"])
-                .default_value("memory"),
+                .default_value("disk"),
-        )
-        /*
-         * Specification/testnet params.
-         */
-        .arg(
-            Arg::with_name("default-spec")
-                .long("default-spec")
-                .value_name("TITLE")
-                .short("default-spec")
-                .help("Specifies the default eth2 spec to be used. This will override any spec written to disk and will therefore be used by default in future instances.")
-                .takes_value(true)
-                .possible_values(&["mainnet", "minimal", "interop"])
-        )
-        .arg(
-            Arg::with_name("recent-genesis")
-                .long("recent-genesis")
-                .short("r")
-                .help("When present, genesis will be within 30 minutes prior. Only for testing"),
         )
         /*
          * Logging.
@@ -200,14 +182,97 @@ fn main() {
                 .takes_value(true),
         )
         /*
-         * Bootstrap.
+         * The "testnet" sub-command.
+         *
+         * Allows for creating a new datadir with testnet-specific configs.
          */
-        .arg(
+        .subcommand(SubCommand::with_name("testnet")
-            Arg::with_name("bootstrap")
+            .about("Create a new Lighthouse datadir using a testnet strategy.")
-                .long("bootstrap")
+            .arg(
-                .value_name("HTTP_SERVER")
+                Arg::with_name("spec")
-                .help("Load the genesis state and libp2p address from the HTTP API of another Lighthouse node.")
+                    .short("s")
-                .takes_value(true)
+                    .long("spec")
+                    .value_name("TITLE")
+                    .help("Specifies the default eth2 spec type. Only effective when creating a new datadir.")
+                    .takes_value(true)
+                    .required(true)
+                    .possible_values(&["mainnet", "minimal", "interop"])
+                    .default_value("minimal")
+            )
+            .arg(
+                Arg::with_name("eth2-config")
+                    .long("eth2-config")
+                    .value_name("TOML_FILE")
+                    .help("A existing eth2_spec TOML file (e.g., eth2_spec.toml).")
+                    .takes_value(true)
+                    .conflicts_with("spec")
+            )
+            .arg(
+                Arg::with_name("client-config")
+                    .long("client-config")
+                    .value_name("TOML_FILE")
+                    .help("An existing beacon_node TOML file (e.g., beacon_node.toml).")
+                    .takes_value(true)
+            )
+            .arg(
+                Arg::with_name("random-datadir")
+                    .long("random-datadir")
+                    .short("r")
+                    .help("If present, append a random string to the datadir path. Useful for fast development \
+                          iteration.")
+            )
+            .arg(
+                Arg::with_name("force")
+                    .long("force")
+                    .short("f")
+                    .help("If present, will create new config and database files and move the any existing to a \
+                           backup directory.")
+                    .conflicts_with("random-datadir")
+            )
+            /*
+             * Testnet sub-commands.
+             *
+             * `boostrap`
+             *
+             * Start a new node by downloading genesis and network info from another node via the
+             * HTTP API.
+             */
+            .subcommand(SubCommand::with_name("bootstrap")
+                .about("Connects to the given HTTP server, downloads a genesis state and attempts to peer with it.")
+                .arg(Arg::with_name("server")
+                    .value_name("HTTP_SERVER")
+                    .required(true)
+                    .help("A HTTP server, with a http:// prefix"))
+                .arg(Arg::with_name("libp2p-port")
+                    .short("p")
+                    .long("port")
+                    .value_name("TCP_PORT")
+                    .help("A libp2p listen port used to peer with the bootstrap server. This flag is useful \
+                           when port-fowarding is used: you may connect using a different port than \
+                           the one the server is immediately listening on."))
+            )
+            /*
+             * `recent`
+             *
+             * Start a new node, with a specified number of validators with a genesis time in the last
+             * 30-minutes.
+             */
+            .subcommand(SubCommand::with_name("recent")
+                .about("Creates a new genesis state where the genesis time was at the previous \
+                       30-minute boundary (e.g., 12:00, 12:30, 13:00, etc.)")
+                .arg(Arg::with_name("validator_count")
+                    .value_name("VALIDATOR_COUNT")
+                    .required(true)
+                    .help("The number of validators in the genesis state"))
+            )
+            .subcommand(SubCommand::with_name("yaml-genesis-state")
+                .about("Creates a new datadir where the genesis state is read from YAML. Will fail to parse \
+                       a YAML state that was generated to a different spec than that specified by --spec.")
+                .arg(Arg::with_name("file")
+                    .value_name("YAML_FILE")
+                    .required(true)
+                    .help("A YAML file from which to read the state"))
+            )
         )
         .get_matches();
 
@@ -227,143 +292,31 @@ fn main() {
         _ => unreachable!("guarded by clap"),
     };
 
-    let mut log = slog::Logger::root(drain.fuse(), o!());
+    let drain = match matches.occurrences_of("verbosity") {
+        0 => drain.filter_level(Level::Info),
+        1 => drain.filter_level(Level::Debug),
+        2 => drain.filter_level(Level::Trace),
+        _ => drain.filter_level(Level::Trace),
+    };
+
+    let log = slog::Logger::root(drain.fuse(), o!());
 
     warn!(
         log,
         "Ethereum 2.0 is pre-release. This software is experimental."
     );
 
-    let data_dir = match matches
+    // Load the process-wide configuration.
-        .value_of("datadir")
-        .and_then(|v| Some(PathBuf::from(v)))
-    {
-        Some(v) => v,
-        None => {
-            // use the default
-            let mut default_dir = match dirs::home_dir() {
-                Some(v) => v,
-                None => {
-                    crit!(log, "Failed to find a home directory");
-                    return;
-                }
-            };
-            default_dir.push(DEFAULT_DATA_DIR);
-            default_dir
-        }
-    };
-
-    // create the directory if needed
-    match fs::create_dir_all(&data_dir) {
-        Ok(_) => {}
-        Err(e) => {
-            crit!(log, "Failed to initialize data dir"; "error" => format!("{}", e));
-            return;
-        }
-    }
-
-    let client_config_path = data_dir.join(CLIENT_CONFIG_FILENAME);
-
-    // Attempt to load the `ClientConfig` from disk.
     //
-    // If file doesn't exist, create a new, default one.
+    // May load this from disk or create a new configuration, depending on the CLI flags supplied.
-    let mut client_config = match read_from_file::<ClientConfig>(client_config_path.clone()) {
+    let (client_config, eth2_config) = match get_configs(&matches, &log) {
-        Ok(Some(c)) => c,
+        Ok(configs) => configs,
-        Ok(None) => {
-            let default = ClientConfig::default();
-            if let Err(e) = write_to_file(client_config_path, &default) {
-                crit!(log, "Failed to write default ClientConfig to file"; "error" => format!("{:?}", e));
-                return;
-            }
-            default
-        }
         Err(e) => {
-            crit!(log, "Failed to load a ChainConfig file"; "error" => format!("{:?}", e));
+            crit!(log, "Failed to load configuration"; "error" => e);
             return;
         }
     };
 
-    // Ensure the `data_dir` in the config matches that supplied to the CLI.
-    client_config.data_dir = data_dir.clone();
-
-    // Update the client config with any CLI args.
-    match client_config.apply_cli_args(&matches, &mut log) {
-        Ok(()) => (),
-        Err(s) => {
-            crit!(log, "Failed to parse ClientConfig CLI arguments"; "error" => s);
-            return;
-        }
-    };
-
-    let eth2_config_path = data_dir.join(ETH2_CONFIG_FILENAME);
-
-    // Initialise the `Eth2Config`.
-    //
-    // If a CLI parameter is set, overwrite any config file present.
-    // If a parameter is not set, use either the config file present or default to minimal.
-    let cli_config = match matches.value_of("default-spec") {
-        Some("mainnet") => Some(Eth2Config::mainnet()),
-        Some("minimal") => Some(Eth2Config::minimal()),
-        Some("interop") => Some(Eth2Config::interop()),
-        _ => None,
-    };
-    // if a CLI flag is specified, write the new config if it doesn't exist,
-    // otherwise notify the user that the file will not be written.
-    let eth2_config_from_file = match read_from_file::<Eth2Config>(eth2_config_path.clone()) {
-        Ok(config) => config,
-        Err(e) => {
-            crit!(log, "Failed to read the Eth2Config from file"; "error" => format!("{:?}", e));
-            return;
-        }
-    };
-
-    let mut eth2_config = {
-        if let Some(cli_config) = cli_config {
-            if eth2_config_from_file.is_none() {
-                // write to file if one doesn't exist
-                if let Err(e) = write_to_file(eth2_config_path, &cli_config) {
-                    crit!(log, "Failed to write default Eth2Config to file"; "error" => format!("{:?}", e));
-                    return;
-                }
-            } else {
-                warn!(
-                    log,
-                    "Eth2Config file exists. Configuration file is ignored, using default"
-                );
-            }
-            cli_config
-        } else {
-            // CLI config not specified, read from disk
-            match eth2_config_from_file {
-                Some(config) => config,
-                None => {
-                    // set default to minimal
-                    let eth2_config = Eth2Config::minimal();
-                    if let Err(e) = write_to_file(eth2_config_path, &eth2_config) {
-                        crit!(log, "Failed to write default Eth2Config to file"; "error" => format!("{:?}", e));
-                        return;
-                    }
-                    eth2_config
-                }
-            }
-        }
-    };
-
-    // Update the eth2 config with any CLI flags.
-    match eth2_config.apply_cli_args(&matches) {
-        Ok(()) => (),
-        Err(s) => {
-            crit!(log, "Failed to parse Eth2Config CLI arguments"; "error" => s);
-            return;
-        }
-    };
-
-    // check to ensure the spec constants between the client and eth2_config match
-    if eth2_config.spec_constants != client_config.spec_constants {
-        crit!(log, "Specification constants do not match."; "client_config" => format!("{}", client_config.spec_constants), "eth2_config" => format!("{}", eth2_config.spec_constants));
-        return;
-    }
-
     // Start the node using a `tokio` executor.
     match run::run_beacon_node(client_config, eth2_config, &log) {
         Ok(_) => {}

beacon_node/src/run.rs

@@ -1,7 +1,4 @@
-use client::{
+use client::{error, notifier, BeaconChainTypes, Client, ClientConfig, ClientType, Eth2Config};
-    error, notifier, BeaconChainTypes, Client, ClientConfig, ClientType, Eth2Config,
-    InitialiseBeaconChain,
-};
 use futures::sync::oneshot;
 use futures::Future;
 use slog::{error, info};
@@ -46,7 +43,6 @@ pub fn run_beacon_node(
         log,
         "BeaconNode init";
         "p2p_listen_address" => format!("{:?}", &other_client_config.network.listen_address),
-        "data_dir" => format!("{:?}", other_client_config.data_dir()),
         "network_dir" => format!("{:?}", other_client_config.network.network_dir),
         "spec_constants" => &spec_constants,
         "db_type" => &other_client_config.db_type,
@@ -118,7 +114,7 @@ fn run<T>(
     log: &slog::Logger,
 ) -> error::Result<()>
 where
-    T: BeaconChainTypes + InitialiseBeaconChain<T> + Clone,
+    T: BeaconChainTypes + Clone,
     T::Store: OpenDatabase,
 {
     let store = T::Store::open_database(&db_path)?;