From e942d7533be92f82f9ba4f76a30fa372e58db0d9 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Tue, 12 Mar 2019 21:56:45 +1100 Subject: [PATCH 1/7] A first go at persisting validator keys and handling configuration. Addresses issue #253. - Creates a keystore directory in the config - Fetches serialized keys from the keystore directory - If no keys, generates keys randomly, saves serialized keys to keystore dir. --- eth2/utils/bls/src/keypair.rs | 4 +++ validator_client/Cargo.toml | 1 + validator_client/src/config.rs | 47 +++++++++++++++++++++++++++++++--- validator_client/src/main.rs | 30 ++++++++++++++++++---- 4 files changed, 73 insertions(+), 9 deletions(-) diff --git a/eth2/utils/bls/src/keypair.rs b/eth2/utils/bls/src/keypair.rs index d60a2fc25..6feb2a585 100644 --- a/eth2/utils/bls/src/keypair.rs +++ b/eth2/utils/bls/src/keypair.rs @@ -14,4 +14,8 @@ impl Keypair { let pk = PublicKey::from_secret_key(&sk); Keypair { sk, pk } } + + pub fn identifier(&self) -> String { + self.pk.concatenated_hex_id() + } } diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index f76772f28..e9d66f6d3 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -18,3 +18,4 @@ slog = "^2.2.3" slog-term = "^2.4.0" slog-async = "^2.3.0" ssz = { path = "../eth2/utils/ssz" } +bincode = "^1.1.2" diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index 68405ed2f..e400f228c 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -1,4 +1,7 @@ +use bls::Keypair; use std::fs; +use std::fs::File; +use std::io::{Error, ErrorKind}; use std::path::PathBuf; use types::ChainSpec; @@ -6,27 +9,63 @@ use types::ChainSpec; #[derive(Clone)] pub struct ClientConfig { pub data_dir: PathBuf, + pub key_dir: PathBuf, pub server: String, pub spec: ChainSpec, } const DEFAULT_LIGHTHOUSE_DIR: &str = ".lighthouse-validators"; +const DEFAULT_KEYSTORE_SUBDIR: &str = "keystore"; impl ClientConfig { /// Build a new configuration from defaults. - pub fn default() -> Self { + pub fn default() -> Result { let data_dir = { let home = dirs::home_dir().expect("Unable to determine home dir."); home.join(DEFAULT_LIGHTHOUSE_DIR) }; - fs::create_dir_all(&data_dir) - .unwrap_or_else(|_| panic!("Unable to create {:?}", &data_dir)); + fs::create_dir_all(&data_dir)?; + + let key_dir = data_dir.join(DEFAULT_KEYSTORE_SUBDIR); + fs::create_dir_all(&key_dir)?; + let server = "localhost:50051".to_string(); let spec = ChainSpec::foundation(); - Self { + Ok(Self { data_dir, + key_dir, server, spec, + }) + } + + // Try to load keys from datadir, or fail + pub fn fetch_keys(&self) -> Result>, Error> { + let mut key_files = fs::read_dir(&self.key_dir)?.peekable(); + + if key_files.peek().is_none() { + return Ok(None); } + + let mut key_pairs: Vec = Vec::new(); + + for key_filename in key_files { + let mut key_file = File::open(key_filename?.path())?; + + let key: Keypair = bincode::deserialize_from(&mut key_file) + .map_err(|e| Error::new(ErrorKind::InvalidData, e))?; + + key_pairs.push(key); + } + + Ok(Some(key_pairs)) + } + + pub fn save_key(&self, key: &Keypair) -> Result<(), Error> { + let key_path = self.key_dir.join(key.identifier() + ".key"); + let mut key_file = File::create(&key_path)?; + bincode::serialize_into(&mut key_file, &key) + .map_err(|e| Error::new(ErrorKind::InvalidData, e))?; + Ok(()) } } diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index ebab8538c..2f2903707 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -6,7 +6,7 @@ use bls::Keypair; use clap::{App, Arg}; use grpcio::{ChannelBuilder, EnvBuilder}; use protos::services_grpc::{BeaconBlockServiceClient, ValidatorServiceClient}; -use slog::{error, info, o, Drain}; +use slog::{debug, error, info, o, Drain}; use slot_clock::SystemTimeSlotClock; use std::path::PathBuf; use std::sync::Arc; @@ -17,6 +17,8 @@ mod block_producer_service; mod config; mod duties; +const NUMBER_OF_VALIDATOR_TEST_KEYS: u16 = 3; + fn main() { // Logging let decorator = slog_term::TermDecorator::new().build(); @@ -55,7 +57,7 @@ fn main() { ) .get_matches(); - let mut config = ClientConfig::default(); + let mut config = ClientConfig::default().expect("Unable to create a default configuration."); // Custom datadir if let Some(dir) = matches.value_of("datadir") { @@ -123,9 +125,27 @@ fn main() { * Start threads. */ let mut threads = vec![]; - // TODO: keypairs are randomly generated; they should be loaded from a file or generated. - // https://github.com/sigp/lighthouse/issues/160 - let keypairs = vec![Keypair::random()]; + + let keypairs = config + .fetch_keys() + .expect("Encountered an error while fetching saved keys.") + .unwrap_or_else(|| { + // TODO: Key generation should occur in a separate binary + let mut k = Vec::new(); + info!( + log, + "No key pairs found, generating and saving 3 random key pairs." + ); + for _n in 0..NUMBER_OF_VALIDATOR_TEST_KEYS { + let keypair = Keypair::random(); + config + .save_key(&keypair) + .expect("Unable to save newly generated private key."); + debug!(log, "Keypair generated {:?}", keypair.identifier()); + k.push(keypair); + } + k + }); for keypair in keypairs { info!(log, "Starting validator services"; "validator" => keypair.pk.concatenated_hex_id()); From b000a0972e39d9d48f4b5f649f2baf933ef53b93 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Wed, 13 Mar 2019 13:38:28 +1100 Subject: [PATCH 2/7] Minor updates to Paul's comments. - Bubbled up home_dir not found error - Made comment show up in docs --- validator_client/src/config.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index e400f228c..d9d30c64d 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -21,7 +21,8 @@ impl ClientConfig { /// Build a new configuration from defaults. pub fn default() -> Result { let data_dir = { - let home = dirs::home_dir().expect("Unable to determine home dir."); + let home = dirs::home_dir() + .ok_or(Error::new(ErrorKind::NotFound, "Unable to determine home directory."))?; home.join(DEFAULT_LIGHTHOUSE_DIR) }; fs::create_dir_all(&data_dir)?; @@ -39,7 +40,7 @@ impl ClientConfig { }) } - // Try to load keys from datadir, or fail + /// Try to load keys from key_dir, returning None if none are found or an error. pub fn fetch_keys(&self) -> Result>, Error> { let mut key_files = fs::read_dir(&self.key_dir)?.peekable(); From 49f6e7ac659f7daa70a6939f99332976e9bac2af Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Wed, 20 Mar 2019 16:23:33 +1100 Subject: [PATCH 3/7] Moved configuration around for validator client. - Custom datadir/server argument logic moved into configuration, out of main. - Updated the validator config directory structure, as per issue #253 suggestions - Removed the 'generate 3 random keys' function - Updated the README to reflect new structure - Just exit if there are no keys, don't generate any (this is for accounts_manager, in a separate commit). - Created a lib.rs file, so that the validator client configuration can be included by external crates. --- validator_client/Cargo.toml | 9 +++ validator_client/README.md | 29 +++++++-- validator_client/src/config.rs | 107 +++++++++++++++++++++++++-------- validator_client/src/lib.rs | 3 + validator_client/src/main.rs | 72 +++++----------------- 5 files changed, 136 insertions(+), 84 deletions(-) create mode 100644 validator_client/src/lib.rs diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index e9d66f6d3..327fab22b 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -4,6 +4,15 @@ version = "0.1.0" authors = ["Paul Hauner "] edition = "2018" +[[bin]] +name = "validator_client" +path = "src/main.rs" + +[lib] +name = "validator_client" +path = "src/lib.rs" + + [dependencies] block_proposer = { path = "../eth2/block_proposer" } bls = { path = "../eth2/utils/bls" } diff --git a/validator_client/README.md b/validator_client/README.md index aa84fe013..109d9f317 100644 --- a/validator_client/README.md +++ b/validator_client/README.md @@ -57,10 +57,31 @@ complete and return a block from the BN. ### Configuration -Presently the validator specifics (pubkey, etc.) are randomly generated and the -chain specification (slot length, BLS domain, etc.) are fixed to foundation -parameters. This is temporary and will be upgrade so these parameters can be -read from file (or initialized on first-boot). +Validator configurations are stored in a separate data directory from the main Beacon Node +binary. The validator data directory defaults to: +`$HOME/.lighthouse-validator`, however an alternative can be specified on the command line +with `--datadir`. + +The configuration directory structure looks like: +``` +~/.lighthouse-validator +└── validators + ├── 3cf4210d58ec + │   └── private.key + ├── 9b5d8b5be4e7 + │   └── private.key + └── cf6e07188f48 + └── private.key +``` + +Where the hex value of the directory is a portion of the validator public key. + +Validator keys must be generated using the separate `accounts_manager` binary, which will +place the keys into this directory structure in a format compatible with the validator client. + +The chain specification (slot length, BLS domain, etc.) defaults to foundation +parameters, however is temporary and an upgrade will allow these parameters to be +read from a file (or initialized on first-boot). ## BN Communication diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index d9d30c64d..2e109c69d 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -1,72 +1,131 @@ use bls::Keypair; +use clap::ArgMatches; use std::fs; use std::fs::File; use std::io::{Error, ErrorKind}; use std::path::PathBuf; use types::ChainSpec; +use bincode; /// Stores the core configuration for this validator instance. #[derive(Clone)] -pub struct ClientConfig { +pub struct ValidatorClientConfig { + /// The data directory, which stores all validator databases pub data_dir: PathBuf, - pub key_dir: PathBuf, + /// The directory where the individual validator configuration directories are stored. + pub validator_dir: PathBuf, + /// The server at which the Beacon Node can be contacted pub server: String, + /// The chain specification that we are connecting to pub spec: ChainSpec, } -const DEFAULT_LIGHTHOUSE_DIR: &str = ".lighthouse-validators"; -const DEFAULT_KEYSTORE_SUBDIR: &str = "keystore"; +const DEFAULT_VALIDATOR_DATADIR: &str = ".lighthouse-validator"; +const DEFAULT_VALIDATORS_SUBDIR: &str = "validators"; +const DEFAULT_PRIVATE_KEY_FILENAME: &str = "private.key"; -impl ClientConfig { - /// Build a new configuration from defaults. - pub fn default() -> Result { - let data_dir = { - let home = dirs::home_dir() - .ok_or(Error::new(ErrorKind::NotFound, "Unable to determine home directory."))?; - home.join(DEFAULT_LIGHTHOUSE_DIR) +impl ValidatorClientConfig { + /// Build a new configuration from defaults, which are overrided by arguments provided. + pub fn build_config(arguments: &ArgMatches) -> Result { + // Use the specified datadir, or default in the home directory + let data_dir: PathBuf = match arguments.value_of("datadir") { + Some(path) => PathBuf::from(path.to_string()), + None => { + let home = dirs::home_dir().ok_or_else(|| Error::new( + ErrorKind::NotFound, + "Unable to determine home directory.", + ))?; + home.join(DEFAULT_VALIDATOR_DATADIR) + } }; fs::create_dir_all(&data_dir)?; - let key_dir = data_dir.join(DEFAULT_KEYSTORE_SUBDIR); - fs::create_dir_all(&key_dir)?; + let validator_dir = data_dir.join(DEFAULT_VALIDATORS_SUBDIR); + fs::create_dir_all(&validator_dir)?; + + let server: String = match arguments.value_of("server") { + Some(srv) => { + //TODO: I don't think this parses correctly a server & port combo + srv.parse::() + .map_err(|e| Error::new(ErrorKind::InvalidInput, e))? + .to_string() + } + None => "localhost:50051".to_string(), + }; + + // TODO: Permit loading a custom spec from file. + let spec: ChainSpec = match arguments.value_of("spec") { + Some(spec_str) => { + match spec_str { + "foundation" => ChainSpec::foundation(), + "few_validators" => ChainSpec::few_validators(), + // Should be impossible due to clap's `possible_values(..)` function. + _ => unreachable!(), + } + } + None => ChainSpec::foundation(), + }; - let server = "localhost:50051".to_string(); - let spec = ChainSpec::foundation(); Ok(Self { data_dir, - key_dir, + validator_dir, server, spec, }) } - /// Try to load keys from key_dir, returning None if none are found or an error. + /// Try to load keys from validator_dir, returning None if none are found or an error. pub fn fetch_keys(&self) -> Result>, Error> { - let mut key_files = fs::read_dir(&self.key_dir)?.peekable(); + let mut validator_dirs = fs::read_dir(&self.validator_dir)?.peekable(); - if key_files.peek().is_none() { + // There are no validator directories. + if validator_dirs.peek().is_none() { return Ok(None); } let mut key_pairs: Vec = Vec::new(); - for key_filename in key_files { - let mut key_file = File::open(key_filename?.path())?; + for validator_dir_result in validator_dirs { + let validator_dir = validator_dir_result?; + + // Try to open the key file directly + // TODO skip keyfiles that are not found, and log the error instead of returning it. + let mut key_file = File::open(validator_dir.path().join(DEFAULT_PRIVATE_KEY_FILENAME))?; let key: Keypair = bincode::deserialize_from(&mut key_file) .map_err(|e| Error::new(ErrorKind::InvalidData, e))?; + // TODO skip keyfile if it's not matched, and log the error instead of returning it. + let validator_directory_name = validator_dir.file_name().into_string().map_err(|_| { + Error::new( + ErrorKind::InvalidData, + "The filename cannot be parsed to a string.", + ) + })?; + if key.identifier() != validator_directory_name { + return Err(Error::new( + ErrorKind::InvalidData, + "The validator directory ID did not match the key found inside.", + )); + } + key_pairs.push(key); } Ok(Some(key_pairs)) } - pub fn save_key(&self, key: &Keypair) -> Result<(), Error> { - let key_path = self.key_dir.join(key.identifier() + ".key"); + /// Saves a keypair to a file inside the appropriate validator directory. Returns the saved path filename. + pub fn save_key(&self, key: &Keypair) -> Result { + let validator_config_path = self.validator_dir.join(key.identifier()); + let key_path = validator_config_path.join(DEFAULT_PRIVATE_KEY_FILENAME); + + fs::create_dir_all(&validator_config_path)?; + let mut key_file = File::create(&key_path)?; + bincode::serialize_into(&mut key_file, &key) .map_err(|e| Error::new(ErrorKind::InvalidData, e))?; - Ok(()) + Ok(key_path) } } diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs new file mode 100644 index 000000000..133ccd932 --- /dev/null +++ b/validator_client/src/lib.rs @@ -0,0 +1,3 @@ +pub mod config; + +pub use crate::config::ValidatorClientConfig; \ No newline at end of file diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index 2f2903707..99523ef9f 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -1,24 +1,19 @@ use self::block_producer_service::{BeaconBlockGrpcClient, BlockProducerService}; use self::duties::{DutiesManager, DutiesManagerService, EpochDutiesMap}; -use crate::config::ClientConfig; +use crate::config::ValidatorClientConfig; use block_proposer::{test_utils::LocalSigner, BlockProducer}; -use bls::Keypair; use clap::{App, Arg}; use grpcio::{ChannelBuilder, EnvBuilder}; use protos::services_grpc::{BeaconBlockServiceClient, ValidatorServiceClient}; -use slog::{debug, error, info, o, Drain}; +use slog::{error, info, o, Drain}; use slot_clock::SystemTimeSlotClock; -use std::path::PathBuf; use std::sync::Arc; -use std::thread; -use types::ChainSpec; +use std::{thread, process, time}; mod block_producer_service; mod config; mod duties; -const NUMBER_OF_VALIDATOR_TEST_KEYS: u16 = 3; - fn main() { // Logging let decorator = slog_term::TermDecorator::new().build(); @@ -57,36 +52,11 @@ fn main() { ) .get_matches(); - let mut config = ClientConfig::default().expect("Unable to create a default configuration."); - - // Custom datadir - if let Some(dir) = matches.value_of("datadir") { - config.data_dir = PathBuf::from(dir.to_string()); - } - - // Custom server port - if let Some(server_str) = matches.value_of("server") { - if let Ok(addr) = server_str.parse::() { - config.server = addr.to_string(); - } else { - error!(log, "Invalid address"; "server" => server_str); - return; - } - } - - // TODO: Permit loading a custom spec from file. - // Custom spec - if let Some(spec_str) = matches.value_of("spec") { - match spec_str { - "foundation" => config.spec = ChainSpec::foundation(), - "few_validators" => config.spec = ChainSpec::few_validators(), - // Should be impossible due to clap's `possible_values(..)` function. - _ => unreachable!(), - }; - } + let config = ValidatorClientConfig::build_config(&matches) + .expect("Unable to build a configuration for the validator client."); // Log configuration - info!(log, ""; + info!(log, "Configuration parameters:"; "data_dir" => &config.data_dir.to_str(), "server" => &config.server); @@ -121,31 +91,21 @@ fn main() { let poll_interval_millis = spec.seconds_per_slot * 1000 / 10; // 10% epoch time precision. info!(log, "Starting block producer service"; "polls_per_epoch" => spec.seconds_per_slot * 1000 / poll_interval_millis); + let keypairs = config + .fetch_keys() + .expect("Encountered an error while fetching saved keys.") + .unwrap_or_else(|| { + error!(log, "No key pairs found in configuration, they must first be generated with: account_manager generate."); + // give the logger a chance to flush the error before exiting. + thread::sleep(time::Duration::from_millis(500)); + process::exit(1) + }); + /* * Start threads. */ let mut threads = vec![]; - let keypairs = config - .fetch_keys() - .expect("Encountered an error while fetching saved keys.") - .unwrap_or_else(|| { - // TODO: Key generation should occur in a separate binary - let mut k = Vec::new(); - info!( - log, - "No key pairs found, generating and saving 3 random key pairs." - ); - for _n in 0..NUMBER_OF_VALIDATOR_TEST_KEYS { - let keypair = Keypair::random(); - config - .save_key(&keypair) - .expect("Unable to save newly generated private key."); - debug!(log, "Keypair generated {:?}", keypair.identifier()); - k.push(keypair); - } - k - }); for keypair in keypairs { info!(log, "Starting validator services"; "validator" => keypair.pk.concatenated_hex_id()); From dc2fc7a2505a7c4835fb4dd8076780cc4850ed6c Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Wed, 20 Mar 2019 16:24:28 +1100 Subject: [PATCH 4/7] Added a new binary, accounts_manager. - Updated main Cargo.toml file - Created a new readme & binary - Includes configuration options from the validator, which now has it's config as a library. --- Cargo.toml | 1 + account_manager/Cargo.toml | 13 +++++++++ account_manager/README.md | 24 +++++++++++++++ account_manager/src/main.rs | 58 +++++++++++++++++++++++++++++++++++++ 4 files changed, 96 insertions(+) create mode 100644 account_manager/Cargo.toml create mode 100644 account_manager/README.md create mode 100644 account_manager/src/main.rs diff --git a/Cargo.toml b/Cargo.toml index c5aae7f43..a43b1052e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,4 +23,5 @@ members = [ "beacon_node/beacon_chain/test_harness", "protos", "validator_client", + "account_manager", ] diff --git a/account_manager/Cargo.toml b/account_manager/Cargo.toml new file mode 100644 index 000000000..c26d4b70a --- /dev/null +++ b/account_manager/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "account_manager" +version = "0.0.1" +authors = ["Luke Anderson "] +edition = "2018" + +[dependencies] +bls = { path = "../eth2/utils/bls" } +clap = "2.32.0" +slog = "^2.2.3" +slog-term = "^2.4.0" +slog-async = "^2.3.0" +validator_client = { path = "../validator_client" } diff --git a/account_manager/README.md b/account_manager/README.md new file mode 100644 index 000000000..bf8891f40 --- /dev/null +++ b/account_manager/README.md @@ -0,0 +1,24 @@ +# Lighthouse Accounts Manager + +The accounts manager (AM) is a stand-alone binary which allows +users to generate and manage the cryptographic keys necessary to +interact with Ethereum Serenity. + +## Roles + +The AM is responsible for the following tasks: +- Generation of cryptographic key pairs + - Must acquire sufficient entropy to ensure keys are generated securely (TBD) +- Secure storage of private keys + - Keys must be encrypted while at rest on the disk (TBD) + - The format is compatible with the validator client +- Produces messages and transactions necessary to initiate +staking on Ethereum 1.x (TPD) + + +## Implementation + +The AM is not a service, and does not run continuously, nor does it +interact with any running services. +It is intended to be executed separately from other Lighthouse binaries +and produce files which can be consumed by them. \ No newline at end of file diff --git a/account_manager/src/main.rs b/account_manager/src/main.rs new file mode 100644 index 000000000..03546c95b --- /dev/null +++ b/account_manager/src/main.rs @@ -0,0 +1,58 @@ +use bls::Keypair; +use clap::{App, Arg, SubCommand}; +use slog::{debug, info, o, Drain}; +use std::path::PathBuf; +use validator_client::config::ValidatorClientConfig; + +fn main() { + // Logging + let decorator = slog_term::TermDecorator::new().build(); + let drain = slog_term::CompactFormat::new(decorator).build().fuse(); + let drain = slog_async::Async::new(drain).build().fuse(); + let log = slog::Logger::root(drain, o!()); + + // CLI + let matches = App::new("Lighthouse Accounts Manager") + .version("0.0.1") + .author("Sigma Prime ") + .about("Eth 2.0 Accounts Manager") + .arg( + Arg::with_name("datadir") + .long("datadir") + .value_name("DIR") + .help("Data directory for keys and databases.") + .takes_value(true), + ) + .subcommand( + SubCommand::with_name("generate") + .about("Generates a new validator private key") + .version("0.0.1") + .author("Sigma Prime "), + ) + .get_matches(); + + let config = ValidatorClientConfig::build_config(&matches) + .expect("Unable to build a configuration for the account manager."); + + // Log configuration + info!(log, ""; + "data_dir" => &config.data_dir.to_str()); + + match matches.subcommand() { + ("generate", Some(_gen_m)) => { + let keypair = Keypair::random(); + let key_path: PathBuf = config + .save_key(&keypair) + .expect("Unable to save newly generated private key."); + debug!( + log, + "Keypair generated {:?}, saved to: {:?}", + keypair.identifier(), + key_path.to_string_lossy() + ); + } + _ => panic!( + "The account manager must be run with a subcommand. See help for more information." + ), + } +} From 9e47cb56e754f7c61d562317ff0045449ddf49c6 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Wed, 20 Mar 2019 16:27:58 +1100 Subject: [PATCH 5/7] Fixed code appearance with rustfmt. --- validator_client/src/config.rs | 14 +++++++------- validator_client/src/lib.rs | 2 +- validator_client/src/main.rs | 3 +-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index 2e109c69d..d056136c9 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -1,3 +1,4 @@ +use bincode; use bls::Keypair; use clap::ArgMatches; use std::fs; @@ -5,7 +6,6 @@ use std::fs::File; use std::io::{Error, ErrorKind}; use std::path::PathBuf; use types::ChainSpec; -use bincode; /// Stores the core configuration for this validator instance. #[derive(Clone)] @@ -31,10 +31,9 @@ impl ValidatorClientConfig { let data_dir: PathBuf = match arguments.value_of("datadir") { Some(path) => PathBuf::from(path.to_string()), None => { - let home = dirs::home_dir().ok_or_else(|| Error::new( - ErrorKind::NotFound, - "Unable to determine home directory.", - ))?; + let home = dirs::home_dir().ok_or_else(|| { + Error::new(ErrorKind::NotFound, "Unable to determine home directory.") + })?; home.join(DEFAULT_VALIDATOR_DATADIR) } }; @@ -96,13 +95,14 @@ impl ValidatorClientConfig { .map_err(|e| Error::new(ErrorKind::InvalidData, e))?; // TODO skip keyfile if it's not matched, and log the error instead of returning it. - let validator_directory_name = validator_dir.file_name().into_string().map_err(|_| { + let validator_directory_name = + validator_dir.file_name().into_string().map_err(|_| { Error::new( ErrorKind::InvalidData, "The filename cannot be parsed to a string.", ) })?; - if key.identifier() != validator_directory_name { + if key.identifier() != validator_directory_name { return Err(Error::new( ErrorKind::InvalidData, "The validator directory ID did not match the key found inside.", diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 133ccd932..60361c051 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -1,3 +1,3 @@ pub mod config; -pub use crate::config::ValidatorClientConfig; \ No newline at end of file +pub use crate::config::ValidatorClientConfig; diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index 99523ef9f..1830bd1a4 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -8,7 +8,7 @@ use protos::services_grpc::{BeaconBlockServiceClient, ValidatorServiceClient}; use slog::{error, info, o, Drain}; use slot_clock::SystemTimeSlotClock; use std::sync::Arc; -use std::{thread, process, time}; +use std::{process, thread, time}; mod block_producer_service; mod config; @@ -106,7 +106,6 @@ fn main() { */ let mut threads = vec![]; - for keypair in keypairs { info!(log, "Starting validator services"; "validator" => keypair.pk.concatenated_hex_id()); let duties_map = Arc::new(EpochDutiesMap::new(spec.slots_per_epoch)); From fba916a0d8530b491d9d13ff01fff59f5b6a8df3 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Sat, 23 Mar 2019 15:46:51 +1100 Subject: [PATCH 6/7] Updated validator config according to suggestions. - Directory structure changed slightly - Uses a filter_map instead of a for loop. - All errors reading files does not prevent others from being read. - The accounts manager needs to generate files first, with the same structure. --- account_manager/src/main.rs | 4 +- validator_client/README.md | 1 - validator_client/src/config.rs | 172 +++++++++++++++++---------------- validator_client/src/lib.rs | 2 +- validator_client/src/main.rs | 19 ++-- 5 files changed, 100 insertions(+), 98 deletions(-) diff --git a/account_manager/src/main.rs b/account_manager/src/main.rs index 03546c95b..42c78aaea 100644 --- a/account_manager/src/main.rs +++ b/account_manager/src/main.rs @@ -2,7 +2,7 @@ use bls::Keypair; use clap::{App, Arg, SubCommand}; use slog::{debug, info, o, Drain}; use std::path::PathBuf; -use validator_client::config::ValidatorClientConfig; +use validator_client::Config as ValidatorClientConfig; fn main() { // Logging @@ -31,7 +31,7 @@ fn main() { ) .get_matches(); - let config = ValidatorClientConfig::build_config(&matches) + let config = ValidatorClientConfig::parse_args(&matches, &log) .expect("Unable to build a configuration for the account manager."); // Log configuration diff --git a/validator_client/README.md b/validator_client/README.md index 109d9f317..03979fbb8 100644 --- a/validator_client/README.md +++ b/validator_client/README.md @@ -65,7 +65,6 @@ with `--datadir`. The configuration directory structure looks like: ``` ~/.lighthouse-validator -└── validators ├── 3cf4210d58ec │   └── private.key ├── 9b5d8b5be4e7 diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index d056136c9..3ca066c89 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -5,119 +5,129 @@ use std::fs; use std::fs::File; use std::io::{Error, ErrorKind}; use std::path::PathBuf; +use slog::{debug, info, error}; use types::ChainSpec; /// Stores the core configuration for this validator instance. #[derive(Clone)] -pub struct ValidatorClientConfig { +pub struct Config { /// The data directory, which stores all validator databases pub data_dir: PathBuf, - /// The directory where the individual validator configuration directories are stored. - pub validator_dir: PathBuf, /// The server at which the Beacon Node can be contacted pub server: String, /// The chain specification that we are connecting to pub spec: ChainSpec, } -const DEFAULT_VALIDATOR_DATADIR: &str = ".lighthouse-validator"; -const DEFAULT_VALIDATORS_SUBDIR: &str = "validators"; const DEFAULT_PRIVATE_KEY_FILENAME: &str = "private.key"; -impl ValidatorClientConfig { - /// Build a new configuration from defaults, which are overrided by arguments provided. - pub fn build_config(arguments: &ArgMatches) -> Result { - // Use the specified datadir, or default in the home directory - let data_dir: PathBuf = match arguments.value_of("datadir") { - Some(path) => PathBuf::from(path.to_string()), - None => { - let home = dirs::home_dir().ok_or_else(|| { - Error::new(ErrorKind::NotFound, "Unable to determine home directory.") - })?; - home.join(DEFAULT_VALIDATOR_DATADIR) - } +impl Default for Config { + fn default() -> Self { + let data_dir = { + let home = dirs::home_dir().expect("Unable to determine home directory."); + home.join(".lighthouse-validator") }; - fs::create_dir_all(&data_dir)?; + fs::create_dir_all(&data_dir) + .unwrap_or_else(|_| panic!("Unable to create {:?}", &data_dir)); - let validator_dir = data_dir.join(DEFAULT_VALIDATORS_SUBDIR); - fs::create_dir_all(&validator_dir)?; + let server = "localhost:50051".to_string(); - let server: String = match arguments.value_of("server") { - Some(srv) => { - //TODO: I don't think this parses correctly a server & port combo - srv.parse::() - .map_err(|e| Error::new(ErrorKind::InvalidInput, e))? - .to_string() - } - None => "localhost:50051".to_string(), + let spec = ChainSpec::foundation(); + + Self { + data_dir, + server, + spec, + } + } +} + +impl Config { + /// Build a new configuration from defaults, which are overrided by arguments provided. + pub fn parse_args(args: &ArgMatches, log: &slog::Logger) -> Result { + let mut config = Config::default(); + + // Use the specified datadir, or default in the home directory + if let Some(datadir) = args.value_of("datadir") { + config.data_dir = PathBuf::from(datadir); + fs::create_dir_all(&config.data_dir) + .unwrap_or_else(|_| panic!("Unable to create {:?}", &config.data_dir)); + info!(log, "Using custom data dir: {:?}", &config.data_dir); + }; + + if let Some(srv) = args.value_of("server") { + //TODO: I don't think this parses correctly a server & port combo + config.server = srv.to_string(); + info!(log, "Using custom server: {:?}", &config.server); }; // TODO: Permit loading a custom spec from file. - let spec: ChainSpec = match arguments.value_of("spec") { - Some(spec_str) => { - match spec_str { - "foundation" => ChainSpec::foundation(), - "few_validators" => ChainSpec::few_validators(), - // Should be impossible due to clap's `possible_values(..)` function. - _ => unreachable!(), - } - } - None => ChainSpec::foundation(), + if let Some(spec_str) = args.value_of("spec") { + info!(log, "Using custom spec: {:?}", spec_str); + config.spec = match spec_str { + "foundation" => ChainSpec::foundation(), + "few_validators" => ChainSpec::few_validators(), + // Should be impossible due to clap's `possible_values(..)` function. + _ => unreachable!(), + }; }; - Ok(Self { - data_dir, - validator_dir, - server, - spec, - }) + Ok(config) } /// Try to load keys from validator_dir, returning None if none are found or an error. - pub fn fetch_keys(&self) -> Result>, Error> { - let mut validator_dirs = fs::read_dir(&self.validator_dir)?.peekable(); + pub fn fetch_keys(&self, log: &slog::Logger) -> Option> { - // There are no validator directories. - if validator_dirs.peek().is_none() { - return Ok(None); + let key_pairs: Vec = fs::read_dir(&self.data_dir) + .unwrap() + .filter_map( |validator_dir| { + + let validator_dir = validator_dir.ok()?; + + if !(validator_dir.file_type().ok()?.is_dir()) { + // Skip non-directories (i.e. no files/symlinks) + return None; + } + + let key_filename = validator_dir.path().join(DEFAULT_PRIVATE_KEY_FILENAME); + + if !(key_filename.is_file()) { + info!(log, "Private key is not a file: {:?}", key_filename.to_str()); + return None; + } + + debug!(log, "Deserializing private key from file: {:?}", key_filename.to_str()); + + let mut key_file = File::open(key_filename.clone()).ok()?; + + let key: Keypair = if let Ok(key_ok) = bincode::deserialize_from(&mut key_file) { + key_ok + } else { + error!(log, "Unable to deserialize the private key file: {:?}", key_filename); + return None; + }; + + let ki = key.identifier(); + if ki != validator_dir.file_name().into_string().ok()? { + error!(log, "The validator key ({:?}) did not match the directory filename {:?}.", ki, &validator_dir.path().to_string_lossy()); + return None; + } + Some(key) + }) + .collect(); + + // Check if it's an empty vector, and return none. + if key_pairs.is_empty() { + None + } else { + Some(key_pairs) } - let mut key_pairs: Vec = Vec::new(); - - for validator_dir_result in validator_dirs { - let validator_dir = validator_dir_result?; - - // Try to open the key file directly - // TODO skip keyfiles that are not found, and log the error instead of returning it. - let mut key_file = File::open(validator_dir.path().join(DEFAULT_PRIVATE_KEY_FILENAME))?; - - let key: Keypair = bincode::deserialize_from(&mut key_file) - .map_err(|e| Error::new(ErrorKind::InvalidData, e))?; - - // TODO skip keyfile if it's not matched, and log the error instead of returning it. - let validator_directory_name = - validator_dir.file_name().into_string().map_err(|_| { - Error::new( - ErrorKind::InvalidData, - "The filename cannot be parsed to a string.", - ) - })?; - if key.identifier() != validator_directory_name { - return Err(Error::new( - ErrorKind::InvalidData, - "The validator directory ID did not match the key found inside.", - )); - } - - key_pairs.push(key); - } - - Ok(Some(key_pairs)) } /// Saves a keypair to a file inside the appropriate validator directory. Returns the saved path filename. pub fn save_key(&self, key: &Keypair) -> Result { - let validator_config_path = self.validator_dir.join(key.identifier()); + let validator_config_path = self.data_dir.join(key.identifier()); let key_path = validator_config_path.join(DEFAULT_PRIVATE_KEY_FILENAME); fs::create_dir_all(&validator_config_path)?; diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 60361c051..470a070e8 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -1,3 +1,3 @@ pub mod config; -pub use crate::config::ValidatorClientConfig; +pub use crate::config::Config; diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index 1830bd1a4..bd0e3e0c5 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -1,14 +1,14 @@ use self::block_producer_service::{BeaconBlockGrpcClient, BlockProducerService}; use self::duties::{DutiesManager, DutiesManagerService, EpochDutiesMap}; -use crate::config::ValidatorClientConfig; +use crate::config::Config; use block_proposer::{test_utils::LocalSigner, BlockProducer}; use clap::{App, Arg}; use grpcio::{ChannelBuilder, EnvBuilder}; use protos::services_grpc::{BeaconBlockServiceClient, ValidatorServiceClient}; -use slog::{error, info, o, Drain}; +use slog::{info, o, Drain}; use slot_clock::SystemTimeSlotClock; use std::sync::Arc; -use std::{process, thread, time}; +use std::thread; mod block_producer_service; mod config; @@ -52,7 +52,7 @@ fn main() { ) .get_matches(); - let config = ValidatorClientConfig::build_config(&matches) + let config = Config::parse_args(&matches, &log) .expect("Unable to build a configuration for the validator client."); // Log configuration @@ -91,15 +91,8 @@ fn main() { let poll_interval_millis = spec.seconds_per_slot * 1000 / 10; // 10% epoch time precision. info!(log, "Starting block producer service"; "polls_per_epoch" => spec.seconds_per_slot * 1000 / poll_interval_millis); - let keypairs = config - .fetch_keys() - .expect("Encountered an error while fetching saved keys.") - .unwrap_or_else(|| { - error!(log, "No key pairs found in configuration, they must first be generated with: account_manager generate."); - // give the logger a chance to flush the error before exiting. - thread::sleep(time::Duration::from_millis(500)); - process::exit(1) - }); + let keypairs = config.fetch_keys(&log) + .expect("No key pairs found in configuration, they must first be generated with: account_manager generate."); /* * Start threads. From cc208670b2bf36cec1faa334c1eed4f89a4365a8 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Sat, 23 Mar 2019 15:52:17 +1100 Subject: [PATCH 7/7] Fixed formatting with rustfmt. --- validator_client/src/config.rs | 39 ++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index 3ca066c89..e0bdaea18 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -1,11 +1,11 @@ use bincode; use bls::Keypair; use clap::ArgMatches; +use slog::{debug, error, info}; use std::fs; use std::fs::File; use std::io::{Error, ErrorKind}; use std::path::PathBuf; -use slog::{debug, info, error}; use types::ChainSpec; /// Stores the core configuration for this validator instance. @@ -77,11 +77,9 @@ impl Config { /// Try to load keys from validator_dir, returning None if none are found or an error. pub fn fetch_keys(&self, log: &slog::Logger) -> Option> { - let key_pairs: Vec = fs::read_dir(&self.data_dir) .unwrap() - .filter_map( |validator_dir| { - + .filter_map(|validator_dir| { let validator_dir = validator_dir.ok()?; if !(validator_dir.file_type().ok()?.is_dir()) { @@ -92,24 +90,40 @@ impl Config { let key_filename = validator_dir.path().join(DEFAULT_PRIVATE_KEY_FILENAME); if !(key_filename.is_file()) { - info!(log, "Private key is not a file: {:?}", key_filename.to_str()); + info!( + log, + "Private key is not a file: {:?}", + key_filename.to_str() + ); return None; } - debug!(log, "Deserializing private key from file: {:?}", key_filename.to_str()); + debug!( + log, + "Deserializing private key from file: {:?}", + key_filename.to_str() + ); let mut key_file = File::open(key_filename.clone()).ok()?; let key: Keypair = if let Ok(key_ok) = bincode::deserialize_from(&mut key_file) { - key_ok - } else { - error!(log, "Unable to deserialize the private key file: {:?}", key_filename); - return None; - }; + key_ok + } else { + error!( + log, + "Unable to deserialize the private key file: {:?}", key_filename + ); + return None; + }; let ki = key.identifier(); if ki != validator_dir.file_name().into_string().ok()? { - error!(log, "The validator key ({:?}) did not match the directory filename {:?}.", ki, &validator_dir.path().to_string_lossy()); + error!( + log, + "The validator key ({:?}) did not match the directory filename {:?}.", + ki, + &validator_dir.path().to_string_lossy() + ); return None; } Some(key) @@ -122,7 +136,6 @@ impl Config { } else { Some(key_pairs) } - } /// Saves a keypair to a file inside the appropriate validator directory. Returns the saved path filename.