From 8e201763376d56f487f11393fefa93d626d3742b Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Tue, 29 Sep 2020 00:02:44 +0000 Subject: [PATCH] Directory restructure (#1532) Closes #1487 Closes #1427 Directory restructure in accordance with #1487. Also has temporary migration code to move the old directories into new structure. Also extracts all default directory names and utility functions into a `directory` crate to avoid repetitio. ~Since `validator_definition.yaml` stores absolute paths, users will have to manually change the keystore paths or delete the file to get the validators picked up by the vc.~. `validator_definition.yaml` is migrated as well from the default directories. Co-authored-by: realbigsean Co-authored-by: Paul Hauner --- Cargo.lock | 18 ++++- Cargo.toml | 1 + account_manager/Cargo.toml | 1 + account_manager/src/common.rs | 24 +----- account_manager/src/lib.rs | 2 +- account_manager/src/validator/create.rs | 52 +++++++------ account_manager/src/validator/deposit.rs | 18 +---- account_manager/src/validator/import.rs | 20 +---- account_manager/src/validator/list.rs | 27 ++----- account_manager/src/validator/mod.rs | 37 +++++++--- account_manager/src/validator/recover.rs | 36 +++------ account_manager/src/wallet/create.rs | 12 +-- account_manager/src/wallet/list.rs | 8 +- account_manager/src/wallet/mod.rs | 36 +++++---- beacon_node/Cargo.toml | 1 + beacon_node/client/Cargo.toml | 1 + beacon_node/client/src/config.rs | 5 +- beacon_node/eth2_libp2p/Cargo.toml | 1 + beacon_node/eth2_libp2p/src/config.rs | 14 +++- beacon_node/src/config.rs | 21 ++++-- book/src/key-management.md | 14 ++-- book/src/validator-create.md | 10 ++- book/src/validator-management.md | 6 +- common/directory/Cargo.toml | 13 ++++ common/directory/src/lib.rs | 60 +++++++++++++++ consensus/types/Cargo.toml | 1 - .../builders/testing_beacon_state_builder.rs | 12 --- lcli/Cargo.toml | 1 + lcli/src/eth1_genesis.rs | 2 +- lcli/src/interop_genesis.rs | 2 +- lcli/src/new_testnet.rs | 2 +- lighthouse/Cargo.toml | 3 +- lighthouse/src/main.rs | 6 +- lighthouse/tests/account_manager.rs | 28 +++---- testing/node_test_rig/src/lib.rs | 8 +- validator_client/Cargo.toml | 1 + validator_client/src/cli.rs | 29 +++++++- validator_client/src/config.rs | 73 +++++++++++++------ validator_client/src/lib.rs | 10 +-- validator_client/src/validator_store.rs | 16 +++- 40 files changed, 367 insertions(+), 265 deletions(-) create mode 100644 common/directory/Cargo.toml create mode 100644 common/directory/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 6049d2e7f..73c7d707a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9,6 +9,7 @@ dependencies = [ "clap", "clap_utils", "deposit_contract", + "directory", "dirs", "environment", "eth2_keystore", @@ -377,6 +378,7 @@ dependencies = [ "clap_utils", "client", "ctrlc", + "directory", "dirs", "environment", "eth2_config", @@ -758,6 +760,7 @@ version = "0.2.0" dependencies = [ "beacon_chain", "bus", + "directory", "dirs", "environment", "error-chain", @@ -1216,6 +1219,16 @@ dependencies = [ "generic-array 0.14.4", ] +[[package]] +name = "directory" +version = "0.1.0" +dependencies = [ + "clap", + "clap_utils", + "dirs", + "eth2_testnet_config", +] + [[package]] name = "dirs" version = "2.0.2" @@ -1522,6 +1535,7 @@ name = "eth2_libp2p" version = "0.2.0" dependencies = [ "base64 0.12.3", + "directory", "dirs", "discv5", "environment", @@ -2567,6 +2581,7 @@ dependencies = [ "clap", "clap_utils", "deposit_contract", + "directory", "dirs", "environment", "eth2_keystore", @@ -2929,6 +2944,7 @@ dependencies = [ "boot_node", "clap", "clap_utils", + "directory", "env_logger", "environment", "eth2_testnet_config", @@ -5829,7 +5845,6 @@ dependencies = [ "compare_fields_derive", "criterion", "derivative", - "dirs", "eth2_hashing", "eth2_interop_keypairs", "eth2_ssz", @@ -6022,6 +6037,7 @@ dependencies = [ "clap", "clap_utils", "deposit_contract", + "directory", "dirs", "environment", "eth2_config", diff --git a/Cargo.toml b/Cargo.toml index 92fb5bccf..82922f5a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ members = [ "common/compare_fields", "common/compare_fields_derive", "common/deposit_contract", + "common/directory", "common/eth2_config", "common/eth2_interop_keypairs", "common/eth2_testnet_config", diff --git a/account_manager/Cargo.toml b/account_manager/Cargo.toml index 9a533aea2..7127a2ddf 100644 --- a/account_manager/Cargo.toml +++ b/account_manager/Cargo.toml @@ -24,6 +24,7 @@ eth2_testnet_config = { path = "../common/eth2_testnet_config" } web3 = "0.11.0" futures = { version = "0.3.5", features = ["compat"] } clap_utils = { path = "../common/clap_utils" } +directory = { path = "../common/directory" } eth2_wallet = { path = "../crypto/eth2_wallet" } eth2_wallet_manager = { path = "../common/eth2_wallet_manager" } rand = "0.7.2" diff --git a/account_manager/src/common.rs b/account_manager/src/common.rs index 030092036..2b9c93fb1 100644 --- a/account_manager/src/common.rs +++ b/account_manager/src/common.rs @@ -1,10 +1,8 @@ use account_utils::PlainText; use account_utils::{read_input_from_user, strip_off_newlines}; -use clap::ArgMatches; use eth2_wallet::bip39::{Language, Mnemonic}; use std::fs; -use std::fs::create_dir_all; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::str::from_utf8; use std::thread::sleep; use std::time::Duration; @@ -12,26 +10,6 @@ use std::time::Duration; pub const MNEMONIC_PROMPT: &str = "Enter the mnemonic phrase:"; pub const WALLET_NAME_PROMPT: &str = "Enter wallet name:"; -pub fn ensure_dir_exists>(path: P) -> Result<(), String> { - let path = path.as_ref(); - - if !path.exists() { - create_dir_all(path).map_err(|e| format!("Unable to create {:?}: {:?}", path, e))?; - } - - Ok(()) -} - -pub fn base_wallet_dir(matches: &ArgMatches, arg: &'static str) -> Result { - clap_utils::parse_path_with_default_in_home_dir( - matches, - arg, - PathBuf::new().join(".lighthouse").join("wallets"), - ) -} - -/// Reads in a mnemonic from the user. If the file path is provided, read from it. Otherwise, read -/// from an interactive prompt using tty, unless the `--stdin-inputs` flag is provided. pub fn read_mnemonic_from_cli( mnemonic_path: Option, stdin_inputs: bool, diff --git a/account_manager/src/lib.rs b/account_manager/src/lib.rs index 5300693dc..829756778 100644 --- a/account_manager/src/lib.rs +++ b/account_manager/src/lib.rs @@ -10,7 +10,7 @@ use types::EthSpec; pub const CMD: &str = "account_manager"; pub const SECRETS_DIR_FLAG: &str = "secrets-dir"; pub const VALIDATOR_DIR_FLAG: &str = "validator-dir"; -pub const BASE_DIR_FLAG: &str = "base-dir"; +pub const WALLETS_DIR_FLAG: &str = "wallets-dir"; pub fn cli_app<'a, 'b>() -> App<'a, 'b> { App::new(CMD) diff --git a/account_manager/src/validator/create.rs b/account_manager/src/validator/create.rs index 948942978..0d4566e46 100644 --- a/account_manager/src/validator/create.rs +++ b/account_manager/src/validator/create.rs @@ -1,10 +1,13 @@ use crate::common::read_wallet_name_from_cli; use crate::wallet::create::STDIN_INPUTS_FLAG; -use crate::{common::ensure_dir_exists, SECRETS_DIR_FLAG, VALIDATOR_DIR_FLAG}; +use crate::{SECRETS_DIR_FLAG, WALLETS_DIR_FLAG}; use account_utils::{ random_password, read_password_from_user, strip_off_newlines, validator_definitions, PlainText, }; use clap::{App, Arg, ArgMatches}; +use directory::{ + ensure_dir_exists, parse_path_or_default_with_flag, DEFAULT_SECRET_DIR, DEFAULT_WALLET_DIR, +}; use environment::Environment; use eth2_wallet_manager::WalletManager; use std::ffi::OsStr; @@ -14,7 +17,6 @@ use types::EthSpec; use validator_dir::Builder as ValidatorDirBuilder; pub const CMD: &str = "create"; -pub const BASE_DIR_FLAG: &str = "base-dir"; pub const WALLET_NAME_FLAG: &str = "wallet-name"; pub const WALLET_PASSWORD_FLAG: &str = "wallet-password"; pub const DEPOSIT_GWEI_FLAG: &str = "deposit-gwei"; @@ -44,14 +46,12 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .takes_value(true), ) .arg( - Arg::with_name(VALIDATOR_DIR_FLAG) - .long(VALIDATOR_DIR_FLAG) - .value_name("VALIDATOR_DIRECTORY") - .help( - "The path where the validator directories will be created. \ - Defaults to ~/.lighthouse/validators", - ) - .takes_value(true), + Arg::with_name(WALLETS_DIR_FLAG) + .long(WALLETS_DIR_FLAG) + .value_name(WALLETS_DIR_FLAG) + .help("A path containing Eth2 EIP-2386 wallets. Defaults to ~/.lighthouse/{testnet}/wallets") + .takes_value(true) + .conflicts_with("datadir"), ) .arg( Arg::with_name(SECRETS_DIR_FLAG) @@ -59,8 +59,9 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .value_name("SECRETS_DIR") .help( "The path where the validator keystore passwords will be stored. \ - Defaults to ~/.lighthouse/secrets", + Defaults to ~/.lighthouse/{testnet}/secrets", ) + .conflicts_with("datadir") .takes_value(true), ) .arg( @@ -111,23 +112,25 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { pub fn cli_run( matches: &ArgMatches, mut env: Environment, - wallet_base_dir: PathBuf, + validator_dir: PathBuf, ) -> Result<(), String> { let spec = env.core_context().eth2_config.spec; let name: Option = clap_utils::parse_optional(matches, WALLET_NAME_FLAG)?; let stdin_inputs = matches.is_present(STDIN_INPUTS_FLAG); + let wallet_base_dir = if matches.value_of("datadir").is_some() { + let path: PathBuf = clap_utils::parse_required(matches, "datadir")?; + path.join(DEFAULT_WALLET_DIR) + } else { + parse_path_or_default_with_flag(matches, WALLETS_DIR_FLAG, DEFAULT_WALLET_DIR)? + }; + let secrets_dir = if matches.value_of("datadir").is_some() { + let path: PathBuf = clap_utils::parse_required(matches, "datadir")?; + path.join(DEFAULT_SECRET_DIR) + } else { + parse_path_or_default_with_flag(matches, SECRETS_DIR_FLAG, DEFAULT_SECRET_DIR)? + }; - let validator_dir = clap_utils::parse_path_with_default_in_home_dir( - matches, - VALIDATOR_DIR_FLAG, - PathBuf::new().join(".lighthouse").join("validators"), - )?; - let secrets_dir = clap_utils::parse_path_with_default_in_home_dir( - matches, - SECRETS_DIR_FLAG, - PathBuf::new().join(".lighthouse").join("secrets"), - )?; let deposit_gwei = clap_utils::parse_optional(matches, DEPOSIT_GWEI_FLAG)? .unwrap_or_else(|| spec.max_effective_balance); let count: Option = clap_utils::parse_optional(matches, COUNT_FLAG)?; @@ -136,6 +139,9 @@ pub fn cli_run( ensure_dir_exists(&validator_dir)?; ensure_dir_exists(&secrets_dir)?; + eprintln!("secrets-dir path {:?}", secrets_dir); + eprintln!("wallets-dir path {:?}", wallet_base_dir); + let starting_validator_count = existing_validator_count(&validator_dir)?; let n = match (count, at_most) { @@ -166,7 +172,7 @@ pub fn cli_run( let wallet_password = read_wallet_password_from_cli(wallet_password_path, stdin_inputs)?; let mgr = WalletManager::open(&wallet_base_dir) - .map_err(|e| format!("Unable to open --{}: {:?}", BASE_DIR_FLAG, e))?; + .map_err(|e| format!("Unable to open --{}: {:?}", WALLETS_DIR_FLAG, e))?; let mut wallet = mgr .wallet_by_name(&wallet_name) diff --git a/account_manager/src/validator/deposit.rs b/account_manager/src/validator/deposit.rs index 0e508cfd2..233e7634e 100644 --- a/account_manager/src/validator/deposit.rs +++ b/account_manager/src/validator/deposit.rs @@ -46,16 +46,6 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { The deposit contract address will be determined by the --testnet-dir flag on the \ primary Lighthouse binary.", ) - .arg( - Arg::with_name(VALIDATOR_DIR_FLAG) - .long(VALIDATOR_DIR_FLAG) - .value_name("VALIDATOR_DIRECTORY") - .help( - "The path to the validator client data directory. \ - Defaults to ~/.lighthouse/validators", - ) - .takes_value(true), - ) .arg( Arg::with_name(VALIDATOR_FLAG) .long(VALIDATOR_FLAG) @@ -209,14 +199,10 @@ where pub fn cli_run( matches: &ArgMatches<'_>, mut env: Environment, + validator_dir: PathBuf, ) -> Result<(), String> { let log = env.core_context().log().clone(); - let data_dir = clap_utils::parse_path_with_default_in_home_dir( - matches, - VALIDATOR_DIR_FLAG, - PathBuf::new().join(".lighthouse").join("validators"), - )?; let validator: String = clap_utils::parse_required(matches, VALIDATOR_FLAG)?; let eth1_ipc_path: Option = clap_utils::parse_optional(matches, ETH1_IPC_FLAG)?; let eth1_http_url: Option = clap_utils::parse_optional(matches, ETH1_HTTP_FLAG)?; @@ -225,7 +211,7 @@ pub fn cli_run( let confirmation_batch_size: usize = clap_utils::parse_required(matches, CONFIRMATION_BATCH_SIZE_FLAG)?; - let manager = ValidatorManager::open(&data_dir) + let manager = ValidatorManager::open(&validator_dir) .map_err(|e| format!("Unable to read --{}: {:?}", VALIDATOR_DIR_FLAG, e))?; let validators = match validator.as_ref() { diff --git a/account_manager/src/validator/import.rs b/account_manager/src/validator/import.rs index 5216b3d9c..1998709d2 100644 --- a/account_manager/src/validator/import.rs +++ b/account_manager/src/validator/import.rs @@ -1,5 +1,4 @@ use crate::wallet::create::STDIN_INPUTS_FLAG; -use crate::{common::ensure_dir_exists, VALIDATOR_DIR_FLAG}; use account_utils::{ eth2_keystore::Keystore, read_password_from_user, @@ -55,16 +54,6 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .required_unless(KEYSTORE_FLAG) .takes_value(true), ) - .arg( - Arg::with_name(VALIDATOR_DIR_FLAG) - .long(VALIDATOR_DIR_FLAG) - .value_name("VALIDATOR_DIRECTORY") - .help( - "The path where the validator directories will be created. \ - Defaults to ~/.lighthouse/validators", - ) - .takes_value(true), - ) .arg( Arg::with_name(STDIN_INPUTS_FLAG) .long(STDIN_INPUTS_FLAG) @@ -77,19 +66,12 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { ) } -pub fn cli_run(matches: &ArgMatches) -> Result<(), String> { +pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), String> { let keystore: Option = clap_utils::parse_optional(matches, KEYSTORE_FLAG)?; let keystores_dir: Option = clap_utils::parse_optional(matches, DIR_FLAG)?; - let validator_dir = clap_utils::parse_path_with_default_in_home_dir( - matches, - VALIDATOR_DIR_FLAG, - PathBuf::new().join(".lighthouse").join("validators"), - )?; let stdin_inputs = matches.is_present(STDIN_INPUTS_FLAG); let reuse_password = matches.is_present(REUSE_PASSWORD_FLAG); - ensure_dir_exists(&validator_dir)?; - let mut defs = ValidatorDefinitions::open_or_create(&validator_dir) .map_err(|e| format!("Unable to open {}: {:?}", CONFIG_FILENAME, e))?; diff --git a/account_manager/src/validator/list.rs b/account_manager/src/validator/list.rs index 148564303..dd97de156 100644 --- a/account_manager/src/validator/list.rs +++ b/account_manager/src/validator/list.rs @@ -1,38 +1,21 @@ use crate::VALIDATOR_DIR_FLAG; -use clap::{App, Arg, ArgMatches}; +use clap::App; use std::path::PathBuf; use validator_dir::Manager as ValidatorManager; pub const CMD: &str = "list"; pub fn cli_app<'a, 'b>() -> App<'a, 'b> { - App::new(CMD) - .arg( - Arg::with_name(VALIDATOR_DIR_FLAG) - .long(VALIDATOR_DIR_FLAG) - .value_name("VALIDATOR_DIRECTORY") - .help( - "The path to search for validator directories. \ - Defaults to ~/.lighthouse/validators", - ) - .takes_value(true), - ) - .about("Lists the names of all validators.") + App::new(CMD).about("Lists the names of all validators.") } -pub fn cli_run(matches: &ArgMatches<'_>) -> Result<(), String> { - let data_dir = clap_utils::parse_path_with_default_in_home_dir( - matches, - VALIDATOR_DIR_FLAG, - PathBuf::new().join(".lighthouse").join("validators"), - )?; - - let mgr = ValidatorManager::open(&data_dir) +pub fn cli_run(validator_dir: PathBuf) -> Result<(), String> { + let mgr = ValidatorManager::open(&validator_dir) .map_err(|e| format!("Unable to read --{}: {:?}", VALIDATOR_DIR_FLAG, e))?; for (name, _path) in mgr .directory_names() - .map_err(|e| format!("Unable to list wallets: {:?}", e))? + .map_err(|e| format!("Unable to list validators: {:?}", e))? { println!("{}", name) } diff --git a/account_manager/src/validator/mod.rs b/account_manager/src/validator/mod.rs index 84ad6df39..4c650dad0 100644 --- a/account_manager/src/validator/mod.rs +++ b/account_manager/src/validator/mod.rs @@ -4,9 +4,11 @@ pub mod import; pub mod list; pub mod recover; -use crate::common::base_wallet_dir; +use crate::VALIDATOR_DIR_FLAG; use clap::{App, Arg, ArgMatches}; +use directory::{parse_path_or_default_with_flag, DEFAULT_VALIDATOR_DIR}; use environment::Environment; +use std::path::PathBuf; use types::EthSpec; pub const CMD: &str = "validator"; @@ -15,11 +17,16 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { App::new(CMD) .about("Provides commands for managing Eth2 validators.") .arg( - Arg::with_name("base-dir") - .long("base-dir") - .value_name("BASE_DIRECTORY") - .help("A path containing Eth2 EIP-2386 wallets. Defaults to ~/.lighthouse/wallets") - .takes_value(true), + Arg::with_name(VALIDATOR_DIR_FLAG) + .long(VALIDATOR_DIR_FLAG) + .value_name("VALIDATOR_DIRECTORY") + .help( + "The path to search for validator directories. \ + Defaults to ~/.lighthouse/{testnet}/validators", + ) + .takes_value(true) + .global(true) + .conflicts_with("datadir"), ) .subcommand(create::cli_app()) .subcommand(deposit::cli_app()) @@ -29,14 +36,20 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { } pub fn cli_run(matches: &ArgMatches, env: Environment) -> Result<(), String> { - let base_wallet_dir = base_wallet_dir(matches, "base-dir")?; + let validator_base_dir = if matches.value_of("datadir").is_some() { + let path: PathBuf = clap_utils::parse_required(matches, "datadir")?; + path.join(DEFAULT_VALIDATOR_DIR) + } else { + parse_path_or_default_with_flag(matches, VALIDATOR_DIR_FLAG, DEFAULT_VALIDATOR_DIR)? + }; + eprintln!("validator-dir path: {:?}", validator_base_dir); match matches.subcommand() { - (create::CMD, Some(matches)) => create::cli_run::(matches, env, base_wallet_dir), - (deposit::CMD, Some(matches)) => deposit::cli_run::(matches, env), - (import::CMD, Some(matches)) => import::cli_run(matches), - (list::CMD, Some(matches)) => list::cli_run(matches), - (recover::CMD, Some(matches)) => recover::cli_run(matches), + (create::CMD, Some(matches)) => create::cli_run::(matches, env, validator_base_dir), + (deposit::CMD, Some(matches)) => deposit::cli_run::(matches, env, validator_base_dir), + (import::CMD, Some(matches)) => import::cli_run(matches, validator_base_dir), + (list::CMD, Some(_)) => list::cli_run(validator_base_dir), + (recover::CMD, Some(matches)) => recover::cli_run(matches, validator_base_dir), (unknown, _) => Err(format!( "{} does not have a {} command. See --help", CMD, unknown diff --git a/account_manager/src/validator/recover.rs b/account_manager/src/validator/recover.rs index 376c21645..e3844d500 100644 --- a/account_manager/src/validator/recover.rs +++ b/account_manager/src/validator/recover.rs @@ -1,11 +1,13 @@ use super::create::STORE_WITHDRAW_FLAG; -use crate::common::{ensure_dir_exists, read_mnemonic_from_cli}; +use crate::common::read_mnemonic_from_cli; use crate::validator::create::COUNT_FLAG; use crate::wallet::create::STDIN_INPUTS_FLAG; -use crate::{SECRETS_DIR_FLAG, VALIDATOR_DIR_FLAG}; +use crate::SECRETS_DIR_FLAG; use account_utils::eth2_keystore::{keypair_from_secret, Keystore, KeystoreBuilder}; use account_utils::random_password; use clap::{App, Arg, ArgMatches}; +use directory::ensure_dir_exists; +use directory::{parse_path_or_default_with_flag, DEFAULT_SECRET_DIR}; use eth2_wallet::bip39::Seed; use eth2_wallet::{recover_validator_secret_from_mnemonic, KeyType, ValidatorKeystores}; use std::path::PathBuf; @@ -48,23 +50,13 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { ) .takes_value(true) ) - .arg( - Arg::with_name(VALIDATOR_DIR_FLAG) - .long(VALIDATOR_DIR_FLAG) - .value_name("VALIDATOR_DIRECTORY") - .help( - "The path where the validator directories will be created. \ - Defaults to ~/.lighthouse/validators", - ) - .takes_value(true), - ) .arg( Arg::with_name(SECRETS_DIR_FLAG) .long(SECRETS_DIR_FLAG) .value_name("SECRETS_DIR") .help( "The path where the validator keystore passwords will be stored. \ - Defaults to ~/.lighthouse/secrets", + Defaults to ~/.lighthouse/{testnet}/secrets", ) .takes_value(true), ) @@ -84,17 +76,13 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { ) } -pub fn cli_run(matches: &ArgMatches) -> Result<(), String> { - let validator_dir = clap_utils::parse_path_with_default_in_home_dir( - matches, - VALIDATOR_DIR_FLAG, - PathBuf::new().join(".lighthouse").join("validators"), - )?; - let secrets_dir = clap_utils::parse_path_with_default_in_home_dir( - matches, - SECRETS_DIR_FLAG, - PathBuf::new().join(".lighthouse").join("secrets"), - )?; +pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), String> { + let secrets_dir = if matches.value_of("datadir").is_some() { + let path: PathBuf = clap_utils::parse_required(matches, "datadir")?; + path.join(DEFAULT_SECRET_DIR) + } else { + parse_path_or_default_with_flag(matches, SECRETS_DIR_FLAG, DEFAULT_SECRET_DIR)? + }; let first_index: u32 = clap_utils::parse_required(matches, FIRST_INDEX_FLAG)?; let count: u32 = clap_utils::parse_required(matches, COUNT_FLAG)?; let mnemonic_path: Option = clap_utils::parse_optional(matches, MNEMONIC_FLAG)?; diff --git a/account_manager/src/wallet/create.rs b/account_manager/src/wallet/create.rs index 04d141b48..a769cc019 100644 --- a/account_manager/src/wallet/create.rs +++ b/account_manager/src/wallet/create.rs @@ -1,5 +1,5 @@ use crate::common::read_wallet_name_from_cli; -use crate::BASE_DIR_FLAG; +use crate::WALLETS_DIR_FLAG; use account_utils::{ is_password_sufficiently_complex, random_password, read_password_from_user, strip_off_newlines, }; @@ -80,7 +80,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { ) } -pub fn cli_run(matches: &ArgMatches, base_dir: PathBuf) -> Result<(), String> { +pub fn cli_run(matches: &ArgMatches, wallet_base_dir: PathBuf) -> Result<(), String> { let mnemonic_output_path: Option = clap_utils::parse_optional(matches, MNEMONIC_FLAG)?; // Create a new random mnemonic. @@ -88,7 +88,7 @@ pub fn cli_run(matches: &ArgMatches, base_dir: PathBuf) -> Result<(), String> { // The `tiny-bip39` crate uses `thread_rng()` for this entropy. let mnemonic = Mnemonic::new(MnemonicType::Words12, Language::English); - let wallet = create_wallet_from_mnemonic(matches, &base_dir.as_path(), &mnemonic)?; + let wallet = create_wallet_from_mnemonic(matches, &wallet_base_dir.as_path(), &mnemonic)?; if let Some(path) = mnemonic_output_path { create_with_600_perms(&path, mnemonic.phrase().as_bytes()) @@ -121,7 +121,7 @@ pub fn cli_run(matches: &ArgMatches, base_dir: PathBuf) -> Result<(), String> { pub fn create_wallet_from_mnemonic( matches: &ArgMatches, - base_dir: &Path, + wallet_base_dir: &Path, mnemonic: &Mnemonic, ) -> Result { let name: Option = clap_utils::parse_optional(matches, NAME_FLAG)?; @@ -134,8 +134,8 @@ pub fn create_wallet_from_mnemonic( unknown => return Err(format!("--{} {} is not supported", TYPE_FLAG, unknown)), }; - let mgr = WalletManager::open(&base_dir) - .map_err(|e| format!("Unable to open --{}: {:?}", BASE_DIR_FLAG, e))?; + let mgr = WalletManager::open(&wallet_base_dir) + .map_err(|e| format!("Unable to open --{}: {:?}", WALLETS_DIR_FLAG, e))?; let wallet_password: PlainText = match wallet_password_path { Some(path) => { diff --git a/account_manager/src/wallet/list.rs b/account_manager/src/wallet/list.rs index 85096dc5f..5b671b1dc 100644 --- a/account_manager/src/wallet/list.rs +++ b/account_manager/src/wallet/list.rs @@ -1,4 +1,4 @@ -use crate::BASE_DIR_FLAG; +use crate::WALLETS_DIR_FLAG; use clap::App; use eth2_wallet_manager::WalletManager; use std::path::PathBuf; @@ -9,9 +9,9 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { App::new(CMD).about("Lists the names of all wallets.") } -pub fn cli_run(base_dir: PathBuf) -> Result<(), String> { - let mgr = WalletManager::open(&base_dir) - .map_err(|e| format!("Unable to open --{}: {:?}", BASE_DIR_FLAG, e))?; +pub fn cli_run(wallet_base_dir: PathBuf) -> Result<(), String> { + let mgr = WalletManager::open(&wallet_base_dir) + .map_err(|e| format!("Unable to open --{}: {:?}", WALLETS_DIR_FLAG, e))?; for (name, _uuid) in mgr .wallets() diff --git a/account_manager/src/wallet/mod.rs b/account_manager/src/wallet/mod.rs index e8315b77a..d745cbcd2 100644 --- a/account_manager/src/wallet/mod.rs +++ b/account_manager/src/wallet/mod.rs @@ -2,11 +2,10 @@ pub mod create; pub mod list; pub mod recover; -use crate::{ - common::{base_wallet_dir, ensure_dir_exists}, - BASE_DIR_FLAG, -}; +use crate::WALLETS_DIR_FLAG; use clap::{App, Arg, ArgMatches}; +use directory::{ensure_dir_exists, parse_path_or_default_with_flag, DEFAULT_WALLET_DIR}; +use std::path::PathBuf; pub const CMD: &str = "wallet"; @@ -14,11 +13,13 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { App::new(CMD) .about("Manage wallets, from which validator keys can be derived.") .arg( - Arg::with_name(BASE_DIR_FLAG) - .long(BASE_DIR_FLAG) - .value_name("BASE_DIRECTORY") - .help("A path containing Eth2 EIP-2386 wallets. Defaults to ~/.lighthouse/wallets") - .takes_value(true), + Arg::with_name(WALLETS_DIR_FLAG) + .long(WALLETS_DIR_FLAG) + .value_name("WALLETS_DIRECTORY") + .help("A path containing Eth2 EIP-2386 wallets. Defaults to ~/.lighthouse/{testnet}/wallets") + .takes_value(true) + .global(true) + .conflicts_with("datadir"), ) .subcommand(create::cli_app()) .subcommand(list::cli_app()) @@ -26,13 +27,20 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { } pub fn cli_run(matches: &ArgMatches) -> Result<(), String> { - let base_dir = base_wallet_dir(matches, BASE_DIR_FLAG)?; - ensure_dir_exists(&base_dir)?; + let wallet_base_dir = if matches.value_of("datadir").is_some() { + let path: PathBuf = clap_utils::parse_required(matches, "datadir")?; + path.join(DEFAULT_WALLET_DIR) + } else { + parse_path_or_default_with_flag(matches, WALLETS_DIR_FLAG, DEFAULT_WALLET_DIR)? + }; + ensure_dir_exists(&wallet_base_dir)?; + + eprintln!("wallet-dir path: {:?}", wallet_base_dir); match matches.subcommand() { - (create::CMD, Some(matches)) => create::cli_run(matches, base_dir), - (list::CMD, Some(_)) => list::cli_run(base_dir), - (recover::CMD, Some(matches)) => recover::cli_run(matches, base_dir), + (create::CMD, Some(matches)) => create::cli_run(matches, wallet_base_dir), + (list::CMD, Some(_)) => list::cli_run(wallet_base_dir), + (recover::CMD, Some(matches)) => recover::cli_run(matches, wallet_base_dir), (unknown, _) => Err(format!( "{} does not have a {} command. See --help", CMD, unknown diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 0351b1cb4..deb965af1 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -30,6 +30,7 @@ tokio = { version = "0.2.21", features = ["time"] } exit-future = "0.2.0" dirs = "2.0.2" logging = { path = "../common/logging" } +directory = {path = "../common/directory"} futures = "0.3.5" environment = { path = "../lighthouse/environment" } genesis = { path = "genesis" } diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index de6f7e59d..ba98eb946 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -41,3 +41,4 @@ lazy_static = "1.4.0" lighthouse_metrics = { path = "../../common/lighthouse_metrics" } time = "0.2.16" bus = "2.2.3" +directory = {path = "../../common/directory"} diff --git a/beacon_node/client/src/config.rs b/beacon_node/client/src/config.rs index 19088e785..fdcd3d6e8 100644 --- a/beacon_node/client/src/config.rs +++ b/beacon_node/client/src/config.rs @@ -1,11 +1,10 @@ +use directory::DEFAULT_ROOT_DIR; use network::NetworkConfig; use serde_derive::{Deserialize, Serialize}; use std::fs; use std::path::PathBuf; use types::Graffiti; -pub const DEFAULT_DATADIR: &str = ".lighthouse"; - /// The number initial validators when starting the `Minimal`. const TESTNET_SPEC_CONSTANTS: &str = "minimal"; @@ -72,7 +71,7 @@ pub struct Config { impl Default for Config { fn default() -> Self { Self { - data_dir: PathBuf::from(DEFAULT_DATADIR), + data_dir: PathBuf::from(DEFAULT_ROOT_DIR), db_name: "chain_db".to_string(), freezer_db_path: None, log_file: PathBuf::from(""), diff --git a/beacon_node/eth2_libp2p/Cargo.toml b/beacon_node/eth2_libp2p/Cargo.toml index 2df5123b5..de916f8fa 100644 --- a/beacon_node/eth2_libp2p/Cargo.toml +++ b/beacon_node/eth2_libp2p/Cargo.toml @@ -36,6 +36,7 @@ discv5 = { version = "0.1.0-alpha.12", features = ["libp2p"] } tiny-keccak = "2.0.2" environment = { path = "../../lighthouse/environment" } rand = "0.7.3" +directory = { path = "../../common/directory" } regex = "1.3.9" [dependencies.libp2p] diff --git a/beacon_node/eth2_libp2p/src/config.rs b/beacon_node/eth2_libp2p/src/config.rs index 73094642d..11bb0d362 100644 --- a/beacon_node/eth2_libp2p/src/config.rs +++ b/beacon_node/eth2_libp2p/src/config.rs @@ -1,5 +1,8 @@ use crate::types::GossipKind; use crate::{Enr, PeerIdSerialized}; +use directory::{ + DEFAULT_BEACON_NODE_DIR, DEFAULT_HARDCODED_TESTNET, DEFAULT_NETWORK_DIR, DEFAULT_ROOT_DIR, +}; use discv5::{Discv5Config, Discv5ConfigBuilder}; use libp2p::gossipsub::{ GossipsubConfig, GossipsubConfigBuilder, GossipsubMessage, MessageId, ValidationMode, @@ -74,9 +77,14 @@ pub struct Config { impl Default for Config { /// Generate a default network configuration. fn default() -> Self { - let mut network_dir = dirs::home_dir().unwrap_or_else(|| PathBuf::from(".")); - network_dir.push(".lighthouse"); - network_dir.push("network"); + // WARNING: this directory default should be always overrided with parameters + // from cli for specific networks. + let network_dir = dirs::home_dir() + .unwrap_or_else(|| PathBuf::from(".")) + .join(DEFAULT_ROOT_DIR) + .join(DEFAULT_HARDCODED_TESTNET) + .join(DEFAULT_BEACON_NODE_DIR) + .join(DEFAULT_NETWORK_DIR); // The function used to generate a gossipsub message id // We use the first 8 bytes of SHA256(data) for content addressing diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 42b3b8277..aabdbb35c 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -1,7 +1,8 @@ use beacon_chain::builder::PUBKEY_CACHE_FILENAME; use clap::ArgMatches; use clap_utils::BAD_TESTNET_DIR_MESSAGE; -use client::{config::DEFAULT_DATADIR, ClientConfig, ClientGenesis}; +use client::{ClientConfig, ClientGenesis}; +use directory::{DEFAULT_BEACON_NODE_DIR, DEFAULT_NETWORK_DIR, DEFAULT_ROOT_DIR}; use eth2_libp2p::{multiaddr::Protocol, Enr, Multiaddr, NetworkConfig, PeerIdSerialized}; use eth2_testnet_config::Eth2TestnetConfig; use slog::{crit, info, warn, Logger}; @@ -13,9 +14,6 @@ use std::net::{TcpListener, UdpSocket}; use std::path::PathBuf; use types::{ChainSpec, EthSpec, GRAFFITI_BYTES_LEN}; -pub const BEACON_NODE_DIR: &str = "beacon"; -pub const NETWORK_DIR: &str = "network"; - /// Gets the fully-initialized global client. /// /// The top-level `clap` arguments should be provided as `cli_args`. @@ -295,7 +293,7 @@ pub fn set_network_config( if let Some(dir) = cli_args.value_of("network-dir") { config.network_dir = PathBuf::from(dir); } else { - config.network_dir = data_dir.join(NETWORK_DIR); + config.network_dir = data_dir.join(DEFAULT_NETWORK_DIR); }; if let Some(listen_address_str) = cli_args.value_of("listen-address") { @@ -456,11 +454,18 @@ pub fn get_data_dir(cli_args: &ArgMatches) -> PathBuf { // Read the `--datadir` flag. // // If it's not present, try and find the home directory (`~`) and push the default data - // directory onto it. + // directory and the testnet name onto it. + cli_args .value_of("datadir") - .map(|path| PathBuf::from(path).join(BEACON_NODE_DIR)) - .or_else(|| dirs::home_dir().map(|home| home.join(DEFAULT_DATADIR).join(BEACON_NODE_DIR))) + .map(|path| PathBuf::from(path).join(DEFAULT_BEACON_NODE_DIR)) + .or_else(|| { + dirs::home_dir().map(|home| { + home.join(DEFAULT_ROOT_DIR) + .join(directory::get_testnet_name(cli_args)) + .join(DEFAULT_BEACON_NODE_DIR) + }) + }) .unwrap_or_else(|| PathBuf::from(".")) } diff --git a/book/src/key-management.md b/book/src/key-management.md index 53edec221..4b03bec0e 100644 --- a/book/src/key-management.md +++ b/book/src/key-management.md @@ -40,12 +40,12 @@ keypairs. Creating a single validator looks like this: - `lighthouse account validator create --wallet-name wally --wallet-password wally.pass --count 1` -In step (1), we created a wallet in `~/.lighthouse/wallets` with the name +In step (1), we created a wallet in `~/.lighthouse/{testnet}/wallets` with the name `wally`. We encrypted this using a pre-defined password in the `wally.pass` file. Then, in step (2), we created one new validator in the -`~/.lighthouse/validators` directory using `wally` (unlocking it with +`~/.lighthouse/{testnet}/validators` directory using `wally` (unlocking it with `wally.pass`) and storing the passwords to the validators voting key in -`~/.lighthouse/secrets`. +`~/.lighthouse/{testnet}/secrets`. Thanks to the hierarchical key derivation scheme, we can delete all of the aforementioned directories and then regenerate them as long as we remembered @@ -63,14 +63,16 @@ There are three important directories in Lighthouse validator key management: - `wallets/`: contains encrypted wallets which are used for hierarchical key derivation. - - Defaults to `~/.lighthouse/wallets` + - Defaults to `~/.lighthouse/{testnet}/wallets` - `validators/`: contains a directory for each validator containing encrypted keystores and other validator-specific data. - - Defaults to `~/.lighthouse/validators` + - Defaults to `~/.lighthouse/{testnet}/validators` - `secrets/`: since the validator signing keys are "hot", the validator process needs access to the passwords to decrypt the keystores in the validators dir. These passwords are stored here. - - Defaults to `~/.lighthouse/secrets` + - Defaults to `~/.lighthouse/{testnet}/secrets` + +where `testnet` is the name of the testnet passed in the `--testnet` parameter (default is `medalla`). When the validator client boots, it searches the `validators/` for directories containing voting keystores. When it discovers a keystore, it searches the diff --git a/book/src/validator-create.md b/book/src/validator-create.md index 25112e748..9d73cdf80 100644 --- a/book/src/validator-create.md +++ b/book/src/validator-create.md @@ -41,7 +41,7 @@ OPTIONS: The GWEI value of the deposit amount. Defaults to the minimum amount required for an active validator (MAX_EFFECTIVE_BALANCE) --secrets-dir - The path where the validator keystore passwords will be stored. Defaults to ~/.lighthouse/secrets + The path where the validator keystore passwords will be stored. Defaults to ~/.lighthouse/{testnet}/secrets -s, --spec Specifies the default eth2 spec type. [default: mainnet] [possible values: mainnet, minimal, interop] @@ -53,7 +53,7 @@ OPTIONS: Path to directory containing eth2_testnet specs. Defaults to a hard-coded Lighthouse testnet. Only effective if there is no existing database. --validator-dir <VALIDATOR_DIRECTORY> - The path where the validator directories will be created. Defaults to ~/.lighthouse/validators + The path where the validator directories will be created. Defaults to ~/.lighthouse/{testnet}/validators --wallet-name <WALLET_NAME> Use the wallet identified by this name --wallet-password <WALLET_PASSWORD_PATH> @@ -73,10 +73,12 @@ This command will: - Derive a single new BLS keypair from `wally`, updating it so that it generates a new key next time. -- Create a new directory in `~/.lighthouse/validators` containing: +- Create a new directory in `~/.lighthouse/{testnet}/validators` containing: - An encrypted keystore containing the validators voting keypair. - An `eth1_deposit_data.rlp` assuming the default deposit amount (`32 ETH` for most testnets and mainnet) which can be submitted to the deposit contract for the medalla testnet. Other testnets can be set via the `--testnet` CLI param. -- Store a password to the validators voting keypair in `~/.lighthouse/secrets`. +- Store a password to the validators voting keypair in `~/.lighthouse/{testnet}/secrets`. + +where `testnet` is the name of the testnet passed in the `--testnet` parameter (default is `medalla`). \ No newline at end of file diff --git a/book/src/validator-management.md b/book/src/validator-management.md index fbb76c9b4..df0e7243d 100644 --- a/book/src/validator-management.md +++ b/book/src/validator-management.md @@ -16,7 +16,7 @@ useful. ## Introducing the `validator_definitions.yml` file The `validator_definitions.yml` file is located in the `validator-dir`, which -defaults to `~/.lighthouse/validators`. It is a +defaults to `~/.lighthouse/{testnet}/validators`. It is a [YAML](https://en.wikipedia.org/wiki/YAML) encoded file defining exactly which validators the validator client will (and won't) act for. @@ -92,7 +92,7 @@ name identical to the `voting_public_key` value. Lets assume the following directory structure: ``` -~/.lighthouse/validators +~/.lighthouse/{testnet}/validators ├── john │   └── voting-keystore.json ├── sally @@ -135,7 +135,7 @@ In order for the validator client to decrypt the validators, they will need to ensure their `secrets-dir` is organised as below: ``` -~/.lighthouse/secrets +~/.lighthouse/{testnet}/secrets ├── 0xa5566f9ec3c6e1fdf362634ebec9ef7aceb0e460e5079714808388e5d48f4ae1e12897fed1bea951c17fa389d511e477 ├── 0xaa440c566fcf34dedf233baf56cf5fb05bb420d9663b4208272545608c27c13d5b08174518c758ecd814f158f2b4a337 └── 0x87a580d31d7bc69069b55f5a01995a610dd391a26dc9e36e81057a17211983a79266800ab8531f21f1083d7d84085007 diff --git a/common/directory/Cargo.toml b/common/directory/Cargo.toml new file mode 100644 index 000000000..ebea5f3dc --- /dev/null +++ b/common/directory/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "directory" +version = "0.1.0" +authors = ["pawan <pawandhananjay@gmail.com>"] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +clap = "2.33.0" +clap_utils = {path = "../clap_utils"} +dirs = "2.0.2" +eth2_testnet_config = { path = "../eth2_testnet_config" } diff --git a/common/directory/src/lib.rs b/common/directory/src/lib.rs new file mode 100644 index 000000000..765fdabd6 --- /dev/null +++ b/common/directory/src/lib.rs @@ -0,0 +1,60 @@ +use clap::ArgMatches; +pub use eth2_testnet_config::DEFAULT_HARDCODED_TESTNET; +use std::fs::create_dir_all; +use std::path::{Path, PathBuf}; + +/// Names for the default directories. +pub const DEFAULT_ROOT_DIR: &str = ".lighthouse"; +pub const DEFAULT_BEACON_NODE_DIR: &str = "beacon"; +pub const DEFAULT_NETWORK_DIR: &str = "network"; +pub const DEFAULT_VALIDATOR_DIR: &str = "validators"; +pub const DEFAULT_SECRET_DIR: &str = "secrets"; +pub const DEFAULT_WALLET_DIR: &str = "wallets"; + +/// Base directory name for unnamed testnets passed through the --testnet-dir flag +pub const CUSTOM_TESTNET_DIR: &str = "custom"; + +/// Gets the testnet directory name +/// +/// Tries to get the name first from the "testnet" flag, +/// if not present, then checks the "testnet-dir" flag and returns a custom name +/// If neither flags are present, returns the default hardcoded network name. +pub fn get_testnet_name(matches: &ArgMatches) -> String { + if let Some(testnet_name) = matches.value_of("testnet") { + testnet_name.to_string() + } else if matches.value_of("testnet-dir").is_some() { + CUSTOM_TESTNET_DIR.to_string() + } else { + eth2_testnet_config::DEFAULT_HARDCODED_TESTNET.to_string() + } +} + +/// Checks if a directory exists in the given path and creates a directory if it does not exist. +pub fn ensure_dir_exists<P: AsRef<Path>>(path: P) -> Result<(), String> { + let path = path.as_ref(); + + if !path.exists() { + create_dir_all(path).map_err(|e| format!("Unable to create {:?}: {:?}", path, e))?; + } + + Ok(()) +} + +/// If `arg` is in `matches`, parses the value as a path. +/// +/// Otherwise, attempts to find the default directory for the `testnet` from the `matches` +/// and appends `flag` to it. +pub fn parse_path_or_default_with_flag( + matches: &ArgMatches, + arg: &'static str, + flag: &str, +) -> Result<PathBuf, String> { + clap_utils::parse_path_with_default_in_home_dir( + matches, + arg, + PathBuf::new() + .join(DEFAULT_ROOT_DIR) + .join(get_testnet_name(matches)) + .join(flag), + ) +} diff --git a/consensus/types/Cargo.toml b/consensus/types/Cargo.toml index 8f6fed4b4..80b4007b9 100644 --- a/consensus/types/Cargo.toml +++ b/consensus/types/Cargo.toml @@ -12,7 +12,6 @@ harness = false bls = { path = "../../crypto/bls" } compare_fields = { path = "../../common/compare_fields" } compare_fields_derive = { path = "../../common/compare_fields_derive" } -dirs = "2.0.2" eth2_interop_keypairs = { path = "../../common/eth2_interop_keypairs" } ethereum-types = "0.9.1" eth2_hashing = "0.1.0" diff --git a/consensus/types/src/test_utils/builders/testing_beacon_state_builder.rs b/consensus/types/src/test_utils/builders/testing_beacon_state_builder.rs index 67a3dae26..922d4017f 100644 --- a/consensus/types/src/test_utils/builders/testing_beacon_state_builder.rs +++ b/consensus/types/src/test_utils/builders/testing_beacon_state_builder.rs @@ -4,21 +4,9 @@ use crate::*; use bls::get_withdrawal_credentials; use log::debug; use rayon::prelude::*; -use std::path::PathBuf; pub const KEYPAIRS_FILE: &str = "keypairs.raw_keypairs"; -/// Returns the directory where the generated keypairs should be stored. -/// -/// It is either `$HOME/.lighthouse/keypairs.raw_keypairs` or, if `$HOME` is not available, -/// `./keypairs.raw_keypairs`. -pub fn keypairs_path() -> PathBuf { - let dir = dirs::home_dir() - .map(|home| (home.join(".lighthouse"))) - .unwrap_or_else(|| PathBuf::from("")); - dir.join(KEYPAIRS_FILE) -} - /// Builds a beacon state to be used for testing purposes. /// /// This struct should **never be used for production purposes.** diff --git a/lcli/Cargo.toml b/lcli/Cargo.toml index c2c2c09b8..8872922bb 100644 --- a/lcli/Cargo.toml +++ b/lcli/Cargo.toml @@ -35,3 +35,4 @@ validator_dir = { path = "../common/validator_dir", features = ["insecure_keys"] rand = "0.7.2" eth2_keystore = { path = "../crypto/eth2_keystore" } lighthouse_version = { path = "../common/lighthouse_version" } +directory = { path = "../common/directory" } diff --git a/lcli/src/eth1_genesis.rs b/lcli/src/eth1_genesis.rs index 2c6f7d8cf..9fd0757d8 100644 --- a/lcli/src/eth1_genesis.rs +++ b/lcli/src/eth1_genesis.rs @@ -20,7 +20,7 @@ pub fn run<T: EthSpec>(mut env: Environment<T>, matches: &ArgMatches<'_>) -> Res .and_then(|dir| dir.parse::<PathBuf>().map_err(|_| ())) .unwrap_or_else(|_| { dirs::home_dir() - .map(|home| home.join(".lighthouse").join("testnet")) + .map(|home| home.join(directory::DEFAULT_ROOT_DIR).join("testnet")) .expect("should locate home directory") }); diff --git a/lcli/src/interop_genesis.rs b/lcli/src/interop_genesis.rs index 9c8609b5c..28cd2625b 100644 --- a/lcli/src/interop_genesis.rs +++ b/lcli/src/interop_genesis.rs @@ -31,7 +31,7 @@ pub fn run<T: EthSpec>(mut env: Environment<T>, matches: &ArgMatches) -> Result< .and_then(|dir| dir.parse::<PathBuf>().map_err(|_| ())) .unwrap_or_else(|_| { dirs::home_dir() - .map(|home| home.join(".lighthouse").join("testnet")) + .map(|home| home.join(directory::DEFAULT_ROOT_DIR).join("testnet")) .expect("should locate home directory") }); diff --git a/lcli/src/new_testnet.rs b/lcli/src/new_testnet.rs index 918426e74..fc60e8c98 100644 --- a/lcli/src/new_testnet.rs +++ b/lcli/src/new_testnet.rs @@ -10,7 +10,7 @@ pub fn run<T: EthSpec>(matches: &ArgMatches) -> Result<(), String> { let testnet_dir_path = parse_path_with_default_in_home_dir( matches, "testnet-dir", - PathBuf::from(".lighthouse/testnet"), + PathBuf::from(directory::DEFAULT_ROOT_DIR).join("testnet"), )?; let deposit_contract_address: Address = parse_required(matches, "deposit-contract-address")?; let deposit_contract_deploy_block = parse_required(matches, "deposit-contract-deploy-block")?; diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index 3bc232d9f..1daf5f97c 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -31,9 +31,10 @@ validator_client = { "path" = "../validator_client" } account_manager = { "path" = "../account_manager" } clap_utils = { path = "../common/clap_utils" } eth2_testnet_config = { path = "../common/eth2_testnet_config" } +directory = { path = "../common/directory" } lighthouse_version = { path = "../common/lighthouse_version" } +account_utils = { path = "../common/account_utils" } [dev-dependencies] tempfile = "3.1.0" validator_dir = { path = "../common/validator_dir" } -account_utils = { path = "../common/account_utils" } diff --git a/lighthouse/src/main.rs b/lighthouse/src/main.rs index c174992e0..9d13706a1 100644 --- a/lighthouse/src/main.rs +++ b/lighthouse/src/main.rs @@ -10,7 +10,6 @@ use std::process::exit; use types::EthSpec; use validator_client::ProductionValidatorClient; -pub const DEFAULT_DATA_DIR: &str = ".lighthouse"; pub const ETH2_CONFIG_FILENAME: &str = "eth2-spec.toml"; fn bls_library_name() -> &'static str { @@ -91,7 +90,10 @@ fn main() { .short("d") .value_name("DIR") .global(true) - .help("Data directory for lighthouse keys and databases.") + .help( + "Root data directory for lighthouse keys and databases. \ + Defaults to $HOME/.lighthouse/{default-testnet}, \ + currently, $HOME/.lighthouse/medalla") .takes_value(true), ) .arg( diff --git a/lighthouse/tests/account_manager.rs b/lighthouse/tests/account_manager.rs index f5c473034..30f885b4e 100644 --- a/lighthouse/tests/account_manager.rs +++ b/lighthouse/tests/account_manager.rs @@ -11,7 +11,7 @@ use account_manager::{ list::CMD as LIST_CMD, CMD as WALLET_CMD, }, - BASE_DIR_FLAG, CMD as ACCOUNT_CMD, *, + CMD as ACCOUNT_CMD, WALLETS_DIR_FLAG, *, }; use account_utils::{ eth2_keystore::KeystoreBuilder, @@ -73,7 +73,7 @@ fn dir_child_count<P: AsRef<Path>>(dir: P) -> usize { fn list_wallets<P: AsRef<Path>>(base_dir: P) -> Vec<String> { let output = output_result( wallet_cmd() - .arg(format!("--{}", BASE_DIR_FLAG)) + .arg(format!("--{}", WALLETS_DIR_FLAG)) .arg(base_dir.as_ref().as_os_str()) .arg(LIST_CMD), ) @@ -97,7 +97,7 @@ fn create_wallet<P: AsRef<Path>>( ) -> Result<Output, String> { output_result( wallet_cmd() - .arg(format!("--{}", BASE_DIR_FLAG)) + .arg(format!("--{}", WALLETS_DIR_FLAG)) .arg(base_dir.as_ref().as_os_str()) .arg(CREATE_CMD) .arg(format!("--{}", NAME_FLAG)) @@ -233,15 +233,15 @@ impl TestValidator { store_withdrawal_key: bool, ) -> Result<Vec<String>, String> { let mut cmd = validator_cmd(); - cmd.arg(format!("--{}", BASE_DIR_FLAG)) - .arg(self.wallet.base_dir().into_os_string()) + cmd.arg(format!("--{}", VALIDATOR_DIR_FLAG)) + .arg(self.validator_dir.clone().into_os_string()) .arg(CREATE_CMD) + .arg(format!("--{}", WALLETS_DIR_FLAG)) + .arg(self.wallet.base_dir().into_os_string()) .arg(format!("--{}", WALLET_NAME_FLAG)) .arg(&self.wallet.name) .arg(format!("--{}", WALLET_PASSWORD_FLAG)) .arg(self.wallet.password_path().into_os_string()) - .arg(format!("--{}", VALIDATOR_DIR_FLAG)) - .arg(self.validator_dir.clone().into_os_string()) .arg(format!("--{}", SECRETS_DIR_FLAG)) .arg(self.secrets_dir.clone().into_os_string()) .arg(format!("--{}", DEPOSIT_GWEI_FLAG)) @@ -375,13 +375,6 @@ fn validator_create() { assert_eq!(dir_child_count(validator_dir.path()), 6); } -/// Returns the `lighthouse account validator import` command. -fn validator_import_cmd() -> Command { - let mut cmd = validator_cmd(); - cmd.arg(IMPORT_CMD); - cmd -} - #[test] fn validator_import_launchpad() { const PASSWORD: &str = "cats"; @@ -407,12 +400,13 @@ fn validator_import_launchpad() { // Create a not-keystore file in the src dir. File::create(src_dir.path().join(NOT_KEYSTORE_NAME)).unwrap(); - let mut child = validator_import_cmd() + let mut child = validator_cmd() + .arg(format!("--{}", VALIDATOR_DIR_FLAG)) + .arg(dst_dir.path().as_os_str()) + .arg(IMPORT_CMD) .arg(format!("--{}", STDIN_INPUTS_FLAG)) // Using tty does not work well with tests. .arg(format!("--{}", import::DIR_FLAG)) .arg(src_dir.path().as_os_str()) - .arg(format!("--{}", VALIDATOR_DIR_FLAG)) - .arg(dst_dir.path().as_os_str()) .stderr(Stdio::piped()) .stdin(Stdio::piped()) .spawn() diff --git a/testing/node_test_rig/src/lib.rs b/testing/node_test_rig/src/lib.rs index 9459a07b5..b1a74b64a 100644 --- a/testing/node_test_rig/src/lib.rs +++ b/testing/node_test_rig/src/lib.rs @@ -96,7 +96,7 @@ pub fn testing_client_config() -> ClientConfig { /// This struct is separate to `LocalValidatorClient` to allow for pre-computation of validator /// keypairs since the task is quite resource intensive. pub struct ValidatorFiles { - pub datadir: TempDir, + pub validator_dir: TempDir, pub secrets_dir: TempDir, } @@ -110,7 +110,7 @@ impl ValidatorFiles { .map_err(|e| format!("Unable to create VC secrets dir: {:?}", e))?; Ok(Self { - datadir, + validator_dir: datadir, secrets_dir, }) } @@ -120,7 +120,7 @@ impl ValidatorFiles { let this = Self::new()?; build_deterministic_validator_dirs( - this.datadir.path().into(), + this.validator_dir.path().into(), this.secrets_dir.path().into(), keypair_indices, ) @@ -170,7 +170,7 @@ impl<E: EthSpec> LocalValidatorClient<E> { mut config: ValidatorConfig, files: ValidatorFiles, ) -> Result<Self, String> { - config.data_dir = files.datadir.path().into(); + config.validator_dir = files.validator_dir.path().into(); config.secrets_dir = files.secrets_dir.path().into(); ProductionValidatorClient::new(context, config) diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index 1bbde0c5a..77a6e5ce9 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -31,6 +31,7 @@ slog-term = "2.5.0" tokio = { version = "0.2.21", features = ["time"] } futures = { version = "0.3.5", features = ["compat"] } dirs = "2.0.2" +directory = {path = "../common/directory"} logging = { path = "../common/logging" } environment = { path = "../lighthouse/environment" } parking_lot = "0.11.0" diff --git a/validator_client/src/cli.rs b/validator_client/src/cli.rs index 7ac483439..9ad0c3faa 100644 --- a/validator_client/src/cli.rs +++ b/validator_client/src/cli.rs @@ -16,6 +16,19 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .default_value(&DEFAULT_HTTP_SERVER) .takes_value(true), ) + .arg( + Arg::with_name("validators-dir") + .long("validators-dir") + .value_name("VALIDATORS_DIR") + .help( + "The directory which contains the validator keystores, deposit data for \ + each validator along with the common slashing protection database \ + and the validator_definitions.yml" + ) + .takes_value(true) + .conflicts_with("datadir") + .requires("secrets-dir") + ) .arg( Arg::with_name("secrets-dir") .long("secrets-dir") @@ -24,9 +37,11 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { "The directory which contains the password to unlock the validator \ voting keypairs. Each password should be contained in a file where the \ name is the 0x-prefixed hex representation of the validators voting public \ - key. Defaults to ~/.lighthouse/secrets.", + key. Defaults to ~/.lighthouse/{testnet}/secrets.", ) - .takes_value(true), + .takes_value(true) + .conflicts_with("datadir") + .requires("validators-dir"), ) .arg(Arg::with_name("auto-register").long("auto-register").help( "If present, the validator client will register any new signing keys with \ @@ -48,6 +63,16 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { that might also be using the same keystores." ) ) + .arg( + Arg::with_name("strict-slashing-protection") + .long("strict-slashing-protection") + .help( + "If present, do not create a new slashing database. This is to ensure that users \ + do not accidentally get slashed in case their slashing protection db ends up in the \ + wrong directory during directory restructure and vc creates a new empty db and \ + re-registers all validators." + ) + ) .arg( Arg::with_name("disable-auto-discover") .long("disable-auto-discover") diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index 4a11c5aec..991b55162 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -1,12 +1,14 @@ use clap::ArgMatches; -use clap_utils::{parse_optional, parse_path_with_default_in_home_dir}; +use clap_utils::{parse_optional, parse_required}; +use directory::{ + get_testnet_name, DEFAULT_HARDCODED_TESTNET, DEFAULT_ROOT_DIR, DEFAULT_SECRET_DIR, + DEFAULT_VALIDATOR_DIR, +}; use serde_derive::{Deserialize, Serialize}; use std::path::PathBuf; use types::{Graffiti, GRAFFITI_BYTES_LEN}; pub const DEFAULT_HTTP_SERVER: &str = "http://localhost:5052/"; -pub const DEFAULT_DATA_DIR: &str = ".lighthouse/validators"; -pub const DEFAULT_SECRETS_DIR: &str = ".lighthouse/secrets"; /// Path to the slashing protection database within the datadir. pub const SLASHING_PROTECTION_FILENAME: &str = "slashing_protection.sqlite"; @@ -14,7 +16,7 @@ pub const SLASHING_PROTECTION_FILENAME: &str = "slashing_protection.sqlite"; #[derive(Clone, Serialize, Deserialize)] pub struct Config { /// The data directory, which stores all validator databases - pub data_dir: PathBuf, + pub validator_dir: PathBuf, /// The directory containing the passwords to unlock validator keystores. pub secrets_dir: PathBuf, /// The http endpoint of the beacon node API. @@ -28,6 +30,8 @@ pub struct Config { pub delete_lockfiles: bool, /// If true, don't scan the validators dir for new keystores. pub disable_auto_discover: bool, + /// If true, don't re-register existing validators in definitions.yml for slashing protection. + pub strict_slashing_protection: bool, /// Graffiti to be inserted everytime we create a block. pub graffiti: Option<Graffiti>, } @@ -35,19 +39,22 @@ pub struct Config { impl Default for Config { /// Build a new configuration from defaults. fn default() -> Self { - let data_dir = dirs::home_dir() - .map(|home| home.join(DEFAULT_DATA_DIR)) - .unwrap_or_else(|| PathBuf::from(".")); - let secrets_dir = dirs::home_dir() - .map(|home| home.join(DEFAULT_SECRETS_DIR)) - .unwrap_or_else(|| PathBuf::from(".")); + // WARNING: these directory defaults should be always overrided with parameters + // from cli for specific networks. + let base_dir = dirs::home_dir() + .unwrap_or_else(|| PathBuf::from(".")) + .join(DEFAULT_ROOT_DIR) + .join(DEFAULT_HARDCODED_TESTNET); + let validator_dir = base_dir.join(DEFAULT_VALIDATOR_DIR); + let secrets_dir = base_dir.join(DEFAULT_SECRET_DIR); Self { - data_dir, + validator_dir, secrets_dir, http_server: DEFAULT_HTTP_SERVER.to_string(), allow_unsynced_beacon_node: false, delete_lockfiles: false, disable_auto_discover: false, + strict_slashing_protection: false, graffiti: None, } } @@ -59,16 +66,39 @@ impl Config { pub fn from_cli(cli_args: &ArgMatches) -> Result<Config, String> { let mut config = Config::default(); - config.data_dir = parse_path_with_default_in_home_dir( - cli_args, - "datadir", - PathBuf::from(".lighthouse").join("validators"), - )?; + let default_root_dir = dirs::home_dir() + .map(|home| home.join(DEFAULT_ROOT_DIR)) + .unwrap_or_else(|| PathBuf::from(".")); - if !config.data_dir.exists() { + let (mut validator_dir, mut secrets_dir) = (None, None); + if cli_args.value_of("datadir").is_some() { + let base_dir: PathBuf = parse_required(cli_args, "datadir")?; + validator_dir = Some(base_dir.join(DEFAULT_VALIDATOR_DIR)); + secrets_dir = Some(base_dir.join(DEFAULT_SECRET_DIR)); + } + if cli_args.value_of("validators-dir").is_some() + && cli_args.value_of("secrets-dir").is_some() + { + validator_dir = Some(parse_required(cli_args, "validators-dir")?); + secrets_dir = Some(parse_required(cli_args, "secrets-dir")?); + } + + config.validator_dir = validator_dir.unwrap_or_else(|| { + default_root_dir + .join(get_testnet_name(cli_args)) + .join(DEFAULT_VALIDATOR_DIR) + }); + + config.secrets_dir = secrets_dir.unwrap_or_else(|| { + default_root_dir + .join(get_testnet_name(cli_args)) + .join(DEFAULT_SECRET_DIR) + }); + + if !config.validator_dir.exists() { return Err(format!( - "The directory for validator data (--datadir) does not exist: {:?}", - config.data_dir + "The directory for validator data does not exist: {:?}", + config.validator_dir )); } @@ -79,10 +109,7 @@ impl Config { config.allow_unsynced_beacon_node = cli_args.is_present("allow-unsynced"); config.delete_lockfiles = cli_args.is_present("delete-lockfiles"); config.disable_auto_discover = cli_args.is_present("disable-auto-discover"); - - if let Some(secrets_dir) = parse_optional(cli_args, "secrets-dir")? { - config.secrets_dir = secrets_dir; - } + config.strict_slashing_protection = cli_args.is_present("strict-slashing-protection"); if let Some(input_graffiti) = cli_args.value_of("graffiti") { let graffiti_bytes = input_graffiti.as_bytes(); diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 6b709023f..6d82baa6b 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -68,18 +68,18 @@ impl<T: EthSpec> ProductionValidatorClient<T> { log, "Starting validator client"; "beacon_node" => &config.http_server, - "datadir" => format!("{:?}", config.data_dir), + "validator_dir" => format!("{:?}", config.validator_dir), ); - let mut validator_defs = ValidatorDefinitions::open_or_create(&config.data_dir) + let mut validator_defs = ValidatorDefinitions::open_or_create(&config.validator_dir) .map_err(|e| format!("Unable to open or create validator definitions: {:?}", e))?; if !config.disable_auto_discover { let new_validators = validator_defs - .discover_local_keystores(&config.data_dir, &config.secrets_dir, &log) + .discover_local_keystores(&config.validator_dir, &config.secrets_dir, &log) .map_err(|e| format!("Unable to discover local validator keystores: {:?}", e))?; validator_defs - .save(&config.data_dir) + .save(&config.validator_dir) .map_err(|e| format!("Unable to update validator definitions: {:?}", e))?; info!( log, @@ -90,7 +90,7 @@ impl<T: EthSpec> ProductionValidatorClient<T> { let validators = InitializedValidators::from_definitions( validator_defs, - config.data_dir.clone(), + config.validator_dir.clone(), config.delete_lockfiles, log.clone(), ) diff --git a/validator_client/src/validator_store.rs b/validator_client/src/validator_store.rs index f7d0442d3..66a616ff3 100644 --- a/validator_client/src/validator_store.rs +++ b/validator_client/src/validator_store.rs @@ -62,14 +62,24 @@ impl<T: SlotClock + 'static, E: EthSpec> ValidatorStore<T, E> { fork_service: ForkService<T, E>, log: Logger, ) -> Result<Self, String> { - let slashing_db_path = config.data_dir.join(SLASHING_PROTECTION_FILENAME); - let slashing_protection = + let slashing_db_path = config.validator_dir.join(SLASHING_PROTECTION_FILENAME); + let slashing_protection = if config.strict_slashing_protection { + // Don't create a new slashing database if `strict_slashing_protection` is turned on. + SlashingDatabase::open(&slashing_db_path).map_err(|e| { + format!( + "Failed to open slashing protection database: {:?}. + Ensure that `slashing_protection.sqlite` is in {:?} folder", + e, config.validator_dir + ) + })? + } else { SlashingDatabase::open_or_create(&slashing_db_path).map_err(|e| { format!( "Failed to open or create slashing protection database: {:?}", e ) - })?; + })? + }; Ok(Self { validators: Arc::new(RwLock::new(validators)),