Move bootstrapper into own crate

Cargo.toml

@@ -12,6 +12,7 @@ members = [
 	"eth2/utils/logging",
 	"eth2/utils/eth2_hashing",
 	"eth2/utils/lighthouse_metrics",
+	"eth2/utils/lighthouse_bootstrap",
 	"eth2/utils/merkle_proof",
 	"eth2/utils/int_to_bytes",
 	"eth2/utils/serde_hex",

beacon_node/Cargo.toml

@@ -6,6 +6,7 @@ edition = "2018"
 
 [dependencies]
 eth2_config = { path = "../eth2/utils/eth2_config" }
+lighthouse_bootstrap = { path = "../eth2/utils/lighthouse_bootstrap" }
 beacon_chain = { path = "beacon_chain" }
 types = { path = "../eth2/types" }
 store = { path = "./store" }

beacon_node/beacon_chain/Cargo.toml

@@ -11,14 +11,13 @@ store = { path = "../store" }
 parking_lot = "0.7"
 lazy_static = "1.3.0"
 lighthouse_metrics = { path = "../../eth2/utils/lighthouse_metrics" }
+lighthouse_bootstrap = { path = "../../eth2/utils/lighthouse_bootstrap" }
 log = "0.4"
 operation_pool = { path = "../../eth2/operation_pool" }
-reqwest = "0.9"
 rayon = "1.0"
 serde = "1.0"
 serde_derive = "1.0"
 serde_yaml = "0.8"
-eth2-libp2p = { path = "../eth2-libp2p" }
 slog = { version = "^2.2.3" , features = ["max_level_trace"] }
 sloggers = { version = "^0.3" }
 slot_clock = { path = "../../eth2/utils/slot_clock" }
@@ -28,7 +27,6 @@ eth2_ssz_derive = "0.1"
 state_processing = { path = "../../eth2/state_processing" }
 tree_hash = "0.1"
 types = { path = "../../eth2/types" }
-url = "1.2"
 lmd_ghost = { path = "../../eth2/lmd_ghost" }
 
 [dev-dependencies]

beacon_node/beacon_chain/src/beacon_chain_builder.rs

@@ -1,6 +1,6 @@
-use super::bootstrapper::Bootstrapper;
 use crate::{BeaconChain, BeaconChainTypes};
 use eth2_hashing::hash;
+use lighthouse_bootstrap::Bootstrapper;
 use merkle_proof::MerkleTree;
 use rayon::prelude::*;
 use slog::Logger;

beacon_node/beacon_chain/src/lib.rs

@@ -4,7 +4,6 @@ extern crate lazy_static;
 
 mod beacon_chain;
 mod beacon_chain_builder;
-mod bootstrapper;
 mod checkpoint;
 mod errors;
 mod fork_choice;
@@ -19,7 +18,6 @@ pub use self::beacon_chain::{
 pub use self::checkpoint::CheckPoint;
 pub use self::errors::{BeaconChainError, BlockProductionError};
 pub use beacon_chain_builder::BeaconChainBuilder;
-pub use bootstrapper::Bootstrapper;
 pub use lmd_ghost;
 pub use metrics::scrape_for_metrics;
 pub use parking_lot;

beacon_node/client/src/bootstrapper.rs

@@ -1,210 +0,0 @@
-use eth2_libp2p::{
-    multiaddr::{Multiaddr, Protocol},
-    Enr,
-};
-use reqwest::{Error as HttpError, Url};
-use serde::Deserialize;
-use std::borrow::Cow;
-use std::net::Ipv4Addr;
-use types::{BeaconBlock, BeaconState, Checkpoint, EthSpec, Hash256, Slot};
-use url::Host;
-
-#[derive(Debug)]
-enum Error {
-    InvalidUrl,
-    HttpError(HttpError),
-}
-
-impl From<HttpError> for Error {
-    fn from(e: HttpError) -> Error {
-        Error::HttpError(e)
-    }
-}
-
-/// Used to load "bootstrap" information from the HTTP API of another Lighthouse beacon node.
-///
-/// Bootstrapping information includes things like genesis and finalized states and blocks, and
-/// libp2p connection details.
-pub struct Bootstrapper {
-    url: Url,
-}
-
-impl Bootstrapper {
-    /// Parses the given `server` as a URL, instantiating `Self`.
-    pub fn from_server_string(server: String) -> Result<Self, String> {
-        Ok(Self {
-            url: Url::parse(&server).map_err(|e| format!("Invalid bootstrap server url: {}", e))?,
-        })
-    }
-
-    /// Build a multiaddr using the HTTP server URL that is not guaranteed to be correct.
-    ///
-    /// The address is created by querying the HTTP server for its listening libp2p addresses.
-    /// Then, we find the first TCP port in those addresses and combine the port with the URL of
-    /// the server.
-    ///
-    /// For example, the server `http://192.168.0.1` might end up with a `best_effort_multiaddr` of
-    /// `/ipv4/192.168.0.1/tcp/9000` if the server advertises a listening address of
-    /// `/ipv4/172.0.0.1/tcp/9000`.
-    pub fn best_effort_multiaddr(&self) -> Option<Multiaddr> {
-        let tcp_port = self.listen_port().ok()?;
-
-        let mut multiaddr = Multiaddr::with_capacity(2);
-
-        match self.url.host()? {
-            Host::Ipv4(addr) => multiaddr.push(Protocol::Ip4(addr)),
-            Host::Domain(s) => multiaddr.push(Protocol::Dns4(Cow::Borrowed(s))),
-            _ => return None,
-        };
-
-        multiaddr.push(Protocol::Tcp(tcp_port));
-
-        Some(multiaddr)
-    }
-
-    /// Returns the IPv4 address of the server URL, unless it contains a FQDN.
-    pub fn server_ipv4_addr(&self) -> Option<Ipv4Addr> {
-        match self.url.host()? {
-            Host::Ipv4(addr) => Some(addr),
-            _ => None,
-        }
-    }
-
-    /// Returns the servers ENR address.
-    pub fn enr(&self) -> Result<Enr, String> {
-        get_enr(self.url.clone()).map_err(|e| format!("Unable to get ENR: {:?}", e))
-    }
-
-    /// Returns the servers listening libp2p addresses.
-    pub fn listen_port(&self) -> Result<u16, String> {
-        get_listen_port(self.url.clone()).map_err(|e| format!("Unable to get listen port: {:?}", e))
-    }
-
-    /// Returns the genesis block and state.
-    pub fn genesis<T: EthSpec>(&self) -> Result<(BeaconState<T>, BeaconBlock<T>), String> {
-        let genesis_slot = Slot::new(0);
-
-        let block = get_block(self.url.clone(), genesis_slot)
-            .map_err(|e| format!("Unable to get genesis block: {:?}", e))?
-            .beacon_block;
-        let state = get_state(self.url.clone(), genesis_slot)
-            .map_err(|e| format!("Unable to get genesis state: {:?}", e))?
-            .beacon_state;
-
-        Ok((state, block))
-    }
-
-    /// Returns the most recent finalized state and block.
-    pub fn finalized<T: EthSpec>(&self) -> Result<(BeaconState<T>, BeaconBlock<T>), String> {
-        let slots_per_epoch = get_slots_per_epoch(self.url.clone())
-            .map_err(|e| format!("Unable to get slots per epoch: {:?}", e))?;
-        let finalized_slot = get_finalized_slot(self.url.clone(), slots_per_epoch.as_u64())
-            .map_err(|e| format!("Unable to get finalized slot: {:?}", e))?;
-
-        let block = get_block(self.url.clone(), finalized_slot)
-            .map_err(|e| format!("Unable to get finalized block: {:?}", e))?
-            .beacon_block;
-        let state = get_state(self.url.clone(), finalized_slot)
-            .map_err(|e| format!("Unable to get finalized state: {:?}", e))?
-            .beacon_state;
-
-        Ok((state, block))
-    }
-}
-
-fn get_slots_per_epoch(mut url: Url) -> Result<Slot, Error> {
-    url.path_segments_mut()
-        .map(|mut url| {
-            url.push("spec").push("slots_per_epoch");
-        })
-        .map_err(|_| Error::InvalidUrl)?;
-
-    reqwest::get(url)?
-        .error_for_status()?
-        .json()
-        .map_err(Into::into)
-}
-
-fn get_finalized_slot(mut url: Url, slots_per_epoch: u64) -> Result<Slot, Error> {
-    url.path_segments_mut()
-        .map(|mut url| {
-            url.push("beacon").push("latest_finalized_checkpoint");
-        })
-        .map_err(|_| Error::InvalidUrl)?;
-
-    let checkpoint: Checkpoint = reqwest::get(url)?.error_for_status()?.json()?;
-
-    Ok(checkpoint.epoch.start_slot(slots_per_epoch))
-}
-
-#[derive(Deserialize)]
-#[serde(bound = "T: EthSpec")]
-pub struct StateResponse<T: EthSpec> {
-    pub root: Hash256,
-    pub beacon_state: BeaconState<T>,
-}
-
-fn get_state<T: EthSpec>(mut url: Url, slot: Slot) -> Result<StateResponse<T>, Error> {
-    url.path_segments_mut()
-        .map(|mut url| {
-            url.push("beacon").push("state");
-        })
-        .map_err(|_| Error::InvalidUrl)?;
-
-    url.query_pairs_mut()
-        .append_pair("slot", &format!("{}", slot.as_u64()));
-
-    reqwest::get(url)?
-        .error_for_status()?
-        .json()
-        .map_err(Into::into)
-}
-
-#[derive(Deserialize)]
-#[serde(bound = "T: EthSpec")]
-pub struct BlockResponse<T: EthSpec> {
-    pub root: Hash256,
-    pub beacon_block: BeaconBlock<T>,
-}
-
-fn get_block<T: EthSpec>(mut url: Url, slot: Slot) -> Result<BlockResponse<T>, Error> {
-    url.path_segments_mut()
-        .map(|mut url| {
-            url.push("beacon").push("block");
-        })
-        .map_err(|_| Error::InvalidUrl)?;
-
-    url.query_pairs_mut()
-        .append_pair("slot", &format!("{}", slot.as_u64()));
-
-    reqwest::get(url)?
-        .error_for_status()?
-        .json()
-        .map_err(Into::into)
-}
-
-fn get_enr(mut url: Url) -> Result<Enr, Error> {
-    url.path_segments_mut()
-        .map(|mut url| {
-            url.push("network").push("enr");
-        })
-        .map_err(|_| Error::InvalidUrl)?;
-
-    reqwest::get(url)?
-        .error_for_status()?
-        .json()
-        .map_err(Into::into)
-}
-
-fn get_listen_port(mut url: Url) -> Result<u16, Error> {
-    url.path_segments_mut()
-        .map(|mut url| {
-            url.push("network").push("listen_port");
-        })
-        .map_err(|_| Error::InvalidUrl)?;
-
-    reqwest::get(url)?
-        .error_for_status()?
-        .json()
-        .map_err(Into::into)
-}

beacon_node/src/config.rs

@@ -1,7 +1,7 @@
-use beacon_chain::Bootstrapper;
 use clap::ArgMatches;
 use client::{BeaconChainStartMethod, ClientConfig, Eth2Config};
 use eth2_config::{read_from_file, write_to_file};
+use lighthouse_bootstrap::Bootstrapper;
 use rand::{distributions::Alphanumeric, Rng};
 use slog::{crit, info, warn, Logger};
 use std::fs;

eth2/utils/lighthouse_bootstrap/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "lighthouse_bootstrap"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+eth2_config = { path = "../eth2_config" }
+eth2-libp2p = { path = "../../../beacon_node/eth2-libp2p" }
+reqwest = "0.9"
+url = "1.2"
+types = { path = "../../types" }
+serde = "1.0"

validator_client/Cargo.toml

@@ -18,6 +18,7 @@ eth2_ssz = "0.1"
 eth2_config = { path = "../eth2/utils/eth2_config" }
 tree_hash = "0.1"
 clap = "2.32.0"
+lighthouse_bootstrap = { path = "../eth2/utils/lighthouse_bootstrap" }
 grpcio = { version = "0.4", default-features = false, features = ["protobuf-codec"] }
 protos = { path = "../protos" }
 slot_clock = { path = "../eth2/utils/slot_clock" }