Add first changes to validator CLI

beacon_node/client/src/config.rs

@@ -89,6 +89,8 @@ impl Config {
     }
 
     /// Returns the core path for the client.
+    ///
+    /// Creates the directory if it does not exist.
     pub fn data_dir(&self) -> Option<PathBuf> {
         let path = dirs::home_dir()?.join(&self.data_dir);
         fs::create_dir_all(&path).ok()?;

validator_client/src/config.rs

@@ -5,19 +5,45 @@ use serde_derive::{Deserialize, Serialize};
 use slog::{debug, error, info, o, Drain};
 use std::fs::{self, File, OpenOptions};
 use std::io::{Error, ErrorKind};
+use std::ops::Range;
 use std::path::PathBuf;
 use std::sync::Mutex;
 use types::{EthSpec, MainnetEthSpec};
 
+pub const DEFAULT_SERVER: &str = "localhost";
+pub const DEFAULT_SERVER_GRPC_PORT: &str = "5051";
+pub const DEFAULT_SERVER_HTTP_PORT: &str = "5052";
+
+#[derive(Clone)]
+pub enum KeySource {
+    /// Load the keypairs from disk.
+    Disk,
+    /// Generate the keypairs (insecure, generates predictable keys).
+    TestingKeypairRange(Range<usize>),
+}
+
+impl Default for KeySource {
+    fn default() -> Self {
+        KeySource::Disk
+    }
+}
+
 /// Stores the core configuration for this validator instance.
 #[derive(Clone, Serialize, Deserialize)]
 pub struct Config {
     /// The data directory, which stores all validator databases
     pub data_dir: PathBuf,
+    /// The source for loading keypairs
+    #[serde(skip)]
+    pub key_source: KeySource,
     /// The path where the logs will be outputted
     pub log_file: PathBuf,
     /// The server at which the Beacon Node can be contacted
     pub server: String,
+    /// The gRPC port on the server
+    pub server_grpc_port: u16,
+    /// The HTTP port on the server, for the REST API.
+    pub server_http_port: u16,
     /// The number of slots per epoch.
     pub slots_per_epoch: u64,
 }
@@ -29,14 +55,33 @@ impl Default for Config {
     fn default() -> Self {
         Self {
             data_dir: PathBuf::from(".lighthouse-validator"),
+            key_source: <_>::default(),
             log_file: PathBuf::from(""),
-            server: "localhost:5051".to_string(),
+            server: DEFAULT_SERVER.into(),
+            server_grpc_port: DEFAULT_SERVER_GRPC_PORT
+                .parse::<u16>()
+                .expect("gRPC port constant should be valid"),
+            server_http_port: DEFAULT_SERVER_GRPC_PORT
+                .parse::<u16>()
+                .expect("HTTP port constant should be valid"),
             slots_per_epoch: MainnetEthSpec::slots_per_epoch(),
         }
     }
 }
 
 impl Config {
+    /// Returns the full path for the client data directory (not just the name of the directory).
+    pub fn full_data_dir(&self) -> Option<PathBuf> {
+        dirs::home_dir().map(|path| path.join(&self.data_dir))
+    }
+
+    /// Creates the data directory (and any non-existing parent directories).
+    pub fn create_data_dir(&self) -> Option<PathBuf> {
+        let path = dirs::home_dir()?.join(&self.data_dir);
+        fs::create_dir_all(&path).ok()?;
+        Some(path)
+    }
+
     /// Apply the following arguments to `self`, replacing values if they are specified in `args`.
     ///
     /// Returns an error if arguments are obviously invalid. May succeed even if some values are
@@ -94,61 +139,106 @@ impl Config {
         Ok(())
     }
 
+    /// Reads a single keypair from the given `path`.
+    ///
+    /// `path` should be the path to a directory containing a private key. The file name of `path`
+    /// must align with the public key loaded from it, otherwise an error is returned.
+    ///
+    /// An error will be returned if `path` is a file (not a directory).
+    fn read_keypair_file(&self, path: PathBuf) -> Result<Keypair, String> {
+        if !path.is_dir() {
+            return Err("Is not a directory".into());
+        }
+
+        let key_filename: PathBuf = path.join(DEFAULT_PRIVATE_KEY_FILENAME);
+
+        if !key_filename.is_file() {
+            return Err(format!(
+                "Private key is not a file: {:?}",
+                key_filename.to_str()
+            ));
+        }
+
+        let mut key_file = File::open(key_filename.clone())
+            .map_err(|e| format!("Unable to open private key file: {}", e))?;
+
+        let key: Keypair = bincode::deserialize_from(&mut key_file)
+            .map_err(|e| format!("Unable to deserialize private key: {:?}", e))?;
+
+        let ki = key.identifier();
+        if &ki
+            != &path
+                .file_name()
+                .ok_or_else(|| "Invalid path".to_string())?
+                .to_string_lossy()
+        {
+            return Err(format!(
+                "The validator key ({:?}) did not match the directory filename {:?}.",
+                ki,
+                path.to_str()
+            ));
+        } else {
+            Ok(key)
+        }
+    }
+
     /// Try to load keys from validator_dir, returning None if none are found or an error.
     #[allow(dead_code)]
     pub fn fetch_keys(&self, log: &slog::Logger) -> Option<Vec<Keypair>> {
-        let key_pairs: Vec<Keypair> = fs::read_dir(&self.data_dir)
+        let key_pairs: Vec<Keypair> =
-            .ok()?
+            fs::read_dir(&self.full_data_dir().expect("Data dir must exist"))
-            .filter_map(|validator_dir| {
+                .ok()?
-                let validator_dir = validator_dir.ok()?;
+                .filter_map(|validator_dir| {
+                    let validator_dir = validator_dir.ok()?;
 
-                if !(validator_dir.file_type().ok()?.is_dir()) {
+                    if !(validator_dir.file_type().ok()?.is_dir()) {
-                    // Skip non-directories (i.e. no files/symlinks)
+                        // Skip non-directories (i.e. no files/symlinks)
-                    return None;
+                        return None;
-                }
+                    }
 
-                let key_filename = validator_dir.path().join(DEFAULT_PRIVATE_KEY_FILENAME);
+                    let key_filename = validator_dir.path().join(DEFAULT_PRIVATE_KEY_FILENAME);
 
-                if !(key_filename.is_file()) {
+                    if !(key_filename.is_file()) {
-                    info!(
+                        info!(
+                            log,
+                            "Private key is not a file: {:?}",
+                            key_filename.to_str()
+                        );
+                        return None;
+                    }
+
+                    debug!(
                         log,
-                        "Private key is not a file: {:?}",
+                        "Deserializing private key from file: {:?}",
                         key_filename.to_str()
                     );
-                    return None;
-                }
 
-                debug!(
+                    let mut key_file = File::open(key_filename.clone()).ok()?;
-                    log,
-                    "Deserializing private key from file: {:?}",
-                    key_filename.to_str()
-                );
 
-                let mut key_file = File::open(key_filename.clone()).ok()?;
+                    let key: Keypair = if let Ok(key_ok) = bincode::deserialize_from(&mut key_file)
+                    {
+                        key_ok
+                    } else {
+                        error!(
+                            log,
+                            "Unable to deserialize the private key file: {:?}", key_filename
+                        );
+                        return None;
+                    };
 
-                let key: Keypair = if let Ok(key_ok) = bincode::deserialize_from(&mut key_file) {
+                    let ki = key.identifier();
-                    key_ok
+                    if ki != validator_dir.file_name().into_string().ok()? {
-                } else {
+                        error!(
-                    error!(
+                            log,
-                        log,
+                            "The validator key ({:?}) did not match the directory filename {:?}.",
-                        "Unable to deserialize the private key file: {:?}", key_filename
+                            ki,
-                    );
+                            &validator_dir.path().to_string_lossy()
-                    return None;
+                        );
-                };
+                        return None;
-
+                    }
-                let ki = key.identifier();
+                    Some(key)
-                if ki != validator_dir.file_name().into_string().ok()? {
+                })
-                    error!(
+                .collect();
-                        log,
-                        "The validator key ({:?}) did not match the directory filename {:?}.",
-                        ki,
-                        &validator_dir.path().to_string_lossy()
-                    );
-                    return None;
-                }
-                Some(key)
-            })
-            .collect();
 
         // Check if it's an empty vector, and return none.
         if key_pairs.is_empty() {

validator_client/src/main.rs

@@ -6,12 +6,16 @@ pub mod error;
 mod service;
 mod signer;
 
-use crate::config::Config as ValidatorClientConfig;
+use crate::config::{
+    Config as ClientConfig, KeySource, DEFAULT_SERVER, DEFAULT_SERVER_GRPC_PORT,
+    DEFAULT_SERVER_HTTP_PORT,
+};
 use crate::service::Service as ValidatorService;
-use clap::{App, Arg};
+use clap::{App, Arg, ArgMatches, SubCommand};
 use eth2_config::{read_from_file, write_to_file, Eth2Config};
+use lighthouse_bootstrap::Bootstrapper;
 use protos::services_grpc::ValidatorServiceClient;
-use slog::{crit, error, info, o, warn, Drain, Level};
+use slog::{crit, error, info, o, warn, Drain, Level, Logger};
 use std::fs;
 use std::path::PathBuf;
 use types::{InteropEthSpec, Keypair, MainnetEthSpec, MinimalEthSpec};
@@ -21,6 +25,8 @@ pub const DEFAULT_DATA_DIR: &str = ".lighthouse-validator";
 pub const CLIENT_CONFIG_FILENAME: &str = "validator-client.toml";
 pub const ETH2_CONFIG_FILENAME: &str = "eth2-spec.toml";
 
+type Result<T> = core::result::Result<T, String>;
+
 fn main() {
     // Logging
     let decorator = slog_term::TermDecorator::new().build();
@@ -49,28 +55,36 @@ fn main() {
                 .takes_value(true),
         )
         .arg(
-            Arg::with_name("eth2-spec")
+            Arg::with_name("eth2-config")
-                .long("eth2-spec")
+                .long("eth2-config")
                 .short("e")
                 .value_name("TOML_FILE")
-                .help("Path to Ethereum 2.0 specifications file.")
+                .help("Path to Ethereum 2.0 config and specification file (e.g., eth2_spec.toml).")
                 .takes_value(true),
         )
         .arg(
             Arg::with_name("server")
                 .long("server")
-                .value_name("server")
+                .value_name("NETWORK_ADDRESS")
                 .help("Address to connect to BeaconNode.")
+                .default_value(DEFAULT_SERVER)
                 .takes_value(true),
         )
         .arg(
-            Arg::with_name("default-spec")
+            Arg::with_name("server-grpc-port")
-                .long("default-spec")
+                .long("g")
-                .value_name("TITLE")
+                .value_name("PORT")
-                .short("default-spec")
+                .help("Port to use for gRPC API connection to the server.")
-                .help("Specifies the default eth2 spec to be used. This will override any spec written to disk and will therefore be used by default in future instances.")
+                .default_value(DEFAULT_SERVER_GRPC_PORT)
-                .takes_value(true)
+                .takes_value(true),
-                .possible_values(&["mainnet", "minimal", "interop"])
+        )
+        .arg(
+            Arg::with_name("server-http-port")
+                .long("h")
+                .value_name("PORT")
+                .help("Port to use for HTTP API connection to the server.")
+                .default_value(DEFAULT_SERVER_HTTP_PORT)
+                .takes_value(true),
         )
         .arg(
             Arg::with_name("debug-level")
@@ -82,6 +96,33 @@ fn main() {
                 .possible_values(&["info", "debug", "trace", "warn", "error", "crit"])
                 .default_value("info"),
         )
+        /*
+         * The "testnet" sub-command.
+         *
+         * Used for starting testnet validator clients.
+         */
+        .subcommand(SubCommand::with_name("testnet")
+            .about("Starts a testnet validator using INSECURE, predicatable private keys, based off the canonical \
+                   validator index. ONLY USE FOR TESTING PURPOSES!")
+            .arg(
+                Arg::with_name("bootstrap")
+                    .short("b")
+                    .long("bootstrap")
+                    .help("Connect to the RPC server to download the eth2_config via the HTTP API.")
+            )
+            .subcommand(SubCommand::with_name("range")
+                .about("Uses the standard, predicatable `interop` keygen method to produce a range \
+                        of predicatable private keys and starts performing their validator duties.")
+                .arg(Arg::with_name("first_validator")
+                    .value_name("VALIDATOR_INDEX")
+                    .required(true)
+                    .help("The first validator public key to be generated for this client."))
+                .arg(Arg::with_name("validator_count")
+                    .value_name("COUNT")
+                    .required(true)
+                    .help("The number of validators."))
+            )
+        )
         .get_matches();
 
     let drain = match matches.value_of("debug-level") {
@@ -93,8 +134,9 @@ fn main() {
         Some("crit") => drain.filter_level(Level::Critical),
         _ => unreachable!("guarded by clap"),
     };
-    let mut log = slog::Logger::root(drain.fuse(), o!());
+    let log = slog::Logger::root(drain.fuse(), o!());
 
+    /*
     let data_dir = match matches
         .value_of("datadir")
         .and_then(|v| Some(PathBuf::from(v)))
@@ -128,12 +170,10 @@ fn main() {
     // Attempt to load the `ClientConfig` from disk.
     //
     // If file doesn't exist, create a new, default one.
-    let mut client_config = match read_from_file::<ValidatorClientConfig>(
+    let mut client_config = match read_from_file::<ClientConfig>(client_config_path.clone()) {
-        client_config_path.clone(),
-    ) {
         Ok(Some(c)) => c,
         Ok(None) => {
-            let default = ValidatorClientConfig::default();
+            let default = ClientConfig::default();
             if let Err(e) = write_to_file(client_config_path.clone(), &default) {
                 crit!(log, "Failed to write default ClientConfig to file"; "error" => format!("{:?}", e));
                 return;
@@ -223,12 +263,23 @@ fn main() {
             return;
         }
     };
+    */
+    let (client_config, eth2_config) = match get_configs(&matches, &log) {
+        Ok(tuple) => tuple,
+        Err(e) => {
+            crit!(
+                log,
+                "Unable to initialize configuration";
+                "error" => e
+            );
+            return;
+        }
+    };
 
     info!(
         log,
         "Starting validator client";
-        "datadir" => client_config.data_dir.to_str(),
+        "datadir" => client_config.full_data_dir().expect("Unable to find datadir").to_str(),
-        "spec_constants" => &eth2_config.spec_constants,
     );
 
     let result = match eth2_config.spec_constants.as_str() {
@@ -260,3 +311,103 @@ fn main() {
         Err(e) => crit!(log, "Validator client exited with error"; "error" => e.to_string()),
     }
 }
+
+/// Parses the CLI arguments and attempts to load the client and eth2 configuration.
+///
+/// This is not a pure function, it reads from disk and may contact network servers.
+pub fn get_configs(cli_args: &ArgMatches, log: &Logger) -> Result<(ClientConfig, Eth2Config)> {
+    let mut client_config = ClientConfig::default();
+
+    if let Some(server) = cli_args.value_of("server") {
+        client_config.server = server.to_string();
+    }
+
+    if let Some(port) = cli_args.value_of("server-http-port") {
+        client_config.server_http_port = port
+            .parse::<u16>()
+            .map_err(|e| format!("Unable to parse HTTP port: {:?}", e))?;
+    }
+
+    if let Some(port) = cli_args.value_of("server-grpc-port") {
+        client_config.server_grpc_port = port
+            .parse::<u16>()
+            .map_err(|e| format!("Unable to parse gRPC port: {:?}", e))?;
+    }
+
+    info!(
+        log,
+        "Beacon node connection info";
+        "grpc_port" => client_config.server_grpc_port,
+        "http_port" => client_config.server_http_port,
+        "server" => &client_config.server,
+    );
+
+    match cli_args.subcommand() {
+        ("testnet", Some(sub_cli_args)) => {
+            if cli_args.is_present("eth2-config") && sub_cli_args.is_present("bootstrap") {
+                return Err(
+                    "Cannot specify --eth2-config and --bootstrap as it may result \
+                     in ambiguity."
+                        .into(),
+                );
+            }
+            process_testnet_subcommand(sub_cli_args, client_config, log)
+        }
+        _ => {
+            unimplemented!("Resuming (not starting a testnet)");
+        }
+    }
+}
+
+fn process_testnet_subcommand(
+    cli_args: &ArgMatches,
+    mut client_config: ClientConfig,
+    log: &Logger,
+) -> Result<(ClientConfig, Eth2Config)> {
+    let eth2_config = if cli_args.is_present("bootstrap") {
+        let bootstrapper = Bootstrapper::from_server_string(format!(
+            "http://{}:{}",
+            client_config.server, client_config.server_http_port
+        ))?;
+
+        let eth2_config = bootstrapper.eth2_config()?;
+
+        info!(
+            log,
+            "Bootstrapped eth2 config via HTTP";
+            "slot_time_millis" => eth2_config.spec.milliseconds_per_slot,
+            "spec" => &eth2_config.spec_constants,
+        );
+
+        eth2_config
+    } else {
+        return Err("Starting without bootstrap is not implemented".into());
+    };
+
+    client_config.key_source = match cli_args.subcommand() {
+        ("range", Some(sub_cli_args)) => {
+            let first = sub_cli_args
+                .value_of("first_validator")
+                .ok_or_else(|| "No first validator supplied")?
+                .parse::<usize>()
+                .map_err(|e| format!("Unable to parse first validator: {:?}", e))?;
+            let count = sub_cli_args
+                .value_of("validator_count")
+                .ok_or_else(|| "No validator count supplied")?
+                .parse::<usize>()
+                .map_err(|e| format!("Unable to parse validator count: {:?}", e))?;
+
+            info!(
+                log,
+                "Generating unsafe testing keys";
+                "first_validator" => first,
+                "count" => count
+            );
+
+            KeySource::TestingKeypairRange(first..first + count)
+        }
+        _ => KeySource::Disk,
+    };
+
+    Ok((client_config, eth2_config))
+}

validator_client/src/service.rs

@@ -73,12 +73,15 @@ impl<B: BeaconNodeDuties + 'static, S: Signer + 'static, E: EthSpec> Service<B,
         eth2_config: Eth2Config,
         log: slog::Logger,
     ) -> error_chain::Result<Service<ValidatorServiceClient, Keypair, E>> {
-        // initialise the beacon node client to check for a connection
+        let server_url = format!(
+            "{}:{}",
+            client_config.server, client_config.server_grpc_port
+        );
 
         let env = Arc::new(EnvBuilder::new().build());
         // Beacon node gRPC beacon node endpoints.
         let beacon_node_client = {
-            let ch = ChannelBuilder::new(env.clone()).connect(&client_config.server);
+            let ch = ChannelBuilder::new(env.clone()).connect(&server_url);
             BeaconNodeServiceClient::new(ch)
         };
 
@@ -86,9 +89,14 @@ impl<B: BeaconNodeDuties + 'static, S: Signer + 'static, E: EthSpec> Service<B,
         let node_info = loop {
             match beacon_node_client.info(&Empty::new()) {
                 Err(e) => {
-                    warn!(log, "Could not connect to node. Error: {}", e);
+                    let retry_seconds = 5;
-                    info!(log, "Retrying in 5 seconds...");
+                    warn!(
-                    std::thread::sleep(Duration::from_secs(5));
+                        log,
+                        "Could not connect to beacon node";
+                        "error" => format!("{:?}", e),
+                        "retry_in" => format!("{} seconds", retry_seconds),
+                    );
+                    std::thread::sleep(Duration::from_secs(retry_seconds));
                     continue;
                 }
                 Ok(info) => {
@@ -122,7 +130,13 @@ impl<B: BeaconNodeDuties + 'static, S: Signer + 'static, E: EthSpec> Service<B,
         let genesis_time = node_info.get_genesis_time();
         let genesis_slot = Slot::from(node_info.get_genesis_slot());
 
-        info!(log,"Beacon node connected"; "Node Version" => node_info.version.clone(), "Chain ID" => node_info.network_id, "Genesis time" => genesis_time);
+        info!(
+            log,
+            "Beacon node connected";
+            "version" => node_info.version.clone(),
+            "network_id" => node_info.network_id,
+            "genesis_time" => genesis_time
+        );
 
         let proto_fork = node_info.get_fork();
         let mut previous_version: [u8; 4] = [0; 4];
@@ -139,7 +153,7 @@ impl<B: BeaconNodeDuties + 'static, S: Signer + 'static, E: EthSpec> Service<B,
 
         // Beacon node gRPC beacon block endpoints.
         let beacon_block_client = {
-            let ch = ChannelBuilder::new(env.clone()).connect(&client_config.server);
+            let ch = ChannelBuilder::new(env.clone()).connect(&server_url);
             let beacon_block_service_client = Arc::new(BeaconBlockServiceClient::new(ch));
             // a wrapper around the service client to implement the beacon block node trait
             Arc::new(BeaconBlockGrpcClient::new(beacon_block_service_client))
@@ -147,13 +161,13 @@ impl<B: BeaconNodeDuties + 'static, S: Signer + 'static, E: EthSpec> Service<B,
 
         // Beacon node gRPC validator endpoints.
         let validator_client = {
-            let ch = ChannelBuilder::new(env.clone()).connect(&client_config.server);
+            let ch = ChannelBuilder::new(env.clone()).connect(&server_url);
             Arc::new(ValidatorServiceClient::new(ch))
         };
 
         //Beacon node gRPC attester endpoints.
         let attestation_client = {
-            let ch = ChannelBuilder::new(env.clone()).connect(&client_config.server);
+            let ch = ChannelBuilder::new(env.clone()).connect(&server_url);
             Arc::new(AttestationServiceClient::new(ch))
         };