Directory restructure (#1532)

Closes #1487
Closes #1427

Directory restructure in accordance with #1487. Also has temporary migration code to move the old directories into new structure.
Also extracts all default directory names and utility functions into a `directory` crate to avoid repetitio.

~Since `validator_definition.yaml` stores absolute paths, users will have to manually change the keystore paths or delete the file to get the validators picked up by the vc.~. `validator_definition.yaml` is migrated as well from the default directories.

Co-authored-by: realbigsean <seananderson33@gmail.com>
Co-authored-by: Paul Hauner <paul@paulhauner.com>

Cargo.lock

@@ -9,6 +9,7 @@ dependencies = [
  "clap",
  "clap_utils",
  "deposit_contract",
+ "directory",
  "dirs",
  "environment",
  "eth2_keystore",
@@ -377,6 +378,7 @@ dependencies = [
  "clap_utils",
  "client",
  "ctrlc",
+ "directory",
  "dirs",
  "environment",
  "eth2_config",
@@ -758,6 +760,7 @@ version = "0.2.0"
 dependencies = [
  "beacon_chain",
  "bus",
+ "directory",
  "dirs",
  "environment",
  "error-chain",
@@ -1216,6 +1219,16 @@ dependencies = [
  "generic-array 0.14.4",
 ]
 
+[[package]]
+name = "directory"
+version = "0.1.0"
+dependencies = [
+ "clap",
+ "clap_utils",
+ "dirs",
+ "eth2_testnet_config",
+]
+
 [[package]]
 name = "dirs"
 version = "2.0.2"
@@ -1522,6 +1535,7 @@ name = "eth2_libp2p"
 version = "0.2.0"
 dependencies = [
  "base64 0.12.3",
+ "directory",
  "dirs",
  "discv5",
  "environment",
@@ -2567,6 +2581,7 @@ dependencies = [
  "clap",
  "clap_utils",
  "deposit_contract",
+ "directory",
  "dirs",
  "environment",
  "eth2_keystore",
@@ -2929,6 +2944,7 @@ dependencies = [
  "boot_node",
  "clap",
  "clap_utils",
+ "directory",
  "env_logger",
  "environment",
  "eth2_testnet_config",
@@ -5829,7 +5845,6 @@ dependencies = [
  "compare_fields_derive",
  "criterion",
  "derivative",
- "dirs",
  "eth2_hashing",
  "eth2_interop_keypairs",
  "eth2_ssz",
@@ -6022,6 +6037,7 @@ dependencies = [
  "clap",
  "clap_utils",
  "deposit_contract",
+ "directory",
  "dirs",
  "environment",
  "eth2_config",

Cargo.toml

@@ -20,6 +20,7 @@ members = [
     "common/compare_fields",
     "common/compare_fields_derive",
     "common/deposit_contract",
+    "common/directory",
     "common/eth2_config",
     "common/eth2_interop_keypairs",
     "common/eth2_testnet_config",

account_manager/Cargo.toml

@@ -24,6 +24,7 @@ eth2_testnet_config = { path = "../common/eth2_testnet_config" }
 web3 = "0.11.0"
 futures = { version = "0.3.5", features = ["compat"] }
 clap_utils = { path = "../common/clap_utils" }
+directory = { path = "../common/directory" }
 eth2_wallet = { path = "../crypto/eth2_wallet" }
 eth2_wallet_manager = { path = "../common/eth2_wallet_manager" }
 rand = "0.7.2"

account_manager/src/common.rs

@@ -1,10 +1,8 @@
 use account_utils::PlainText;
 use account_utils::{read_input_from_user, strip_off_newlines};
-use clap::ArgMatches;
 use eth2_wallet::bip39::{Language, Mnemonic};
 use std::fs;
-use std::fs::create_dir_all;
+use std::path::PathBuf;
-use std::path::{Path, PathBuf};
 use std::str::from_utf8;
 use std::thread::sleep;
 use std::time::Duration;
@@ -12,26 +10,6 @@ use std::time::Duration;
 pub const MNEMONIC_PROMPT: &str = "Enter the mnemonic phrase:";
 pub const WALLET_NAME_PROMPT: &str = "Enter wallet name:";
 
-pub fn ensure_dir_exists<P: AsRef<Path>>(path: P) -> Result<(), String> {
-    let path = path.as_ref();
-
-    if !path.exists() {
-        create_dir_all(path).map_err(|e| format!("Unable to create {:?}: {:?}", path, e))?;
-    }
-
-    Ok(())
-}
-
-pub fn base_wallet_dir(matches: &ArgMatches, arg: &'static str) -> Result<PathBuf, String> {
-    clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        arg,
-        PathBuf::new().join(".lighthouse").join("wallets"),
-    )
-}
-
-/// Reads in a mnemonic from the user. If the file path is provided, read from it. Otherwise, read
-/// from an interactive prompt using tty, unless the `--stdin-inputs` flag is provided.
 pub fn read_mnemonic_from_cli(
     mnemonic_path: Option<PathBuf>,
     stdin_inputs: bool,

account_manager/src/lib.rs

@@ -10,7 +10,7 @@ use types::EthSpec;
 pub const CMD: &str = "account_manager";
 pub const SECRETS_DIR_FLAG: &str = "secrets-dir";
 pub const VALIDATOR_DIR_FLAG: &str = "validator-dir";
-pub const BASE_DIR_FLAG: &str = "base-dir";
+pub const WALLETS_DIR_FLAG: &str = "wallets-dir";
 
 pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD)

account_manager/src/validator/create.rs

@@ -1,10 +1,13 @@
 use crate::common::read_wallet_name_from_cli;
 use crate::wallet::create::STDIN_INPUTS_FLAG;
-use crate::{common::ensure_dir_exists, SECRETS_DIR_FLAG, VALIDATOR_DIR_FLAG};
+use crate::{SECRETS_DIR_FLAG, WALLETS_DIR_FLAG};
 use account_utils::{
     random_password, read_password_from_user, strip_off_newlines, validator_definitions, PlainText,
 };
 use clap::{App, Arg, ArgMatches};
+use directory::{
+    ensure_dir_exists, parse_path_or_default_with_flag, DEFAULT_SECRET_DIR, DEFAULT_WALLET_DIR,
+};
 use environment::Environment;
 use eth2_wallet_manager::WalletManager;
 use std::ffi::OsStr;
@@ -14,7 +17,6 @@ use types::EthSpec;
 use validator_dir::Builder as ValidatorDirBuilder;
 
 pub const CMD: &str = "create";
-pub const BASE_DIR_FLAG: &str = "base-dir";
 pub const WALLET_NAME_FLAG: &str = "wallet-name";
 pub const WALLET_PASSWORD_FLAG: &str = "wallet-password";
 pub const DEPOSIT_GWEI_FLAG: &str = "deposit-gwei";
@@ -44,14 +46,12 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .takes_value(true),
         )
         .arg(
-            Arg::with_name(VALIDATOR_DIR_FLAG)
+            Arg::with_name(WALLETS_DIR_FLAG)
-                .long(VALIDATOR_DIR_FLAG)
+                .long(WALLETS_DIR_FLAG)
-                .value_name("VALIDATOR_DIRECTORY")
+                .value_name(WALLETS_DIR_FLAG)
-                .help(
+                .help("A path containing Eth2 EIP-2386 wallets. Defaults to ~/.lighthouse/{testnet}/wallets")
-                    "The path where the validator directories will be created. \
+                .takes_value(true)
-                    Defaults to ~/.lighthouse/validators",
+                .conflicts_with("datadir"),
-                )
-                .takes_value(true),
         )
         .arg(
             Arg::with_name(SECRETS_DIR_FLAG)
@@ -59,8 +59,9 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .value_name("SECRETS_DIR")
                 .help(
                     "The path where the validator keystore passwords will be stored. \
-                    Defaults to ~/.lighthouse/secrets",
+                    Defaults to ~/.lighthouse/{testnet}/secrets",
                 )
+                .conflicts_with("datadir")
                 .takes_value(true),
         )
         .arg(
@@ -111,23 +112,25 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
 pub fn cli_run<T: EthSpec>(
     matches: &ArgMatches,
     mut env: Environment<T>,
-    wallet_base_dir: PathBuf,
+    validator_dir: PathBuf,
 ) -> Result<(), String> {
     let spec = env.core_context().eth2_config.spec;
 
     let name: Option<String> = clap_utils::parse_optional(matches, WALLET_NAME_FLAG)?;
     let stdin_inputs = matches.is_present(STDIN_INPUTS_FLAG);
+    let wallet_base_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_WALLET_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, WALLETS_DIR_FLAG, DEFAULT_WALLET_DIR)?
+    };
+    let secrets_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_SECRET_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, SECRETS_DIR_FLAG, DEFAULT_SECRET_DIR)?
+    };
 
-    let validator_dir = clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        VALIDATOR_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("validators"),
-    )?;
-    let secrets_dir = clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        SECRETS_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("secrets"),
-    )?;
     let deposit_gwei = clap_utils::parse_optional(matches, DEPOSIT_GWEI_FLAG)?
         .unwrap_or_else(|| spec.max_effective_balance);
     let count: Option<usize> = clap_utils::parse_optional(matches, COUNT_FLAG)?;
@@ -136,6 +139,9 @@ pub fn cli_run<T: EthSpec>(
     ensure_dir_exists(&validator_dir)?;
     ensure_dir_exists(&secrets_dir)?;
 
+    eprintln!("secrets-dir path {:?}", secrets_dir);
+    eprintln!("wallets-dir path {:?}", wallet_base_dir);
+
     let starting_validator_count = existing_validator_count(&validator_dir)?;
 
     let n = match (count, at_most) {
@@ -166,7 +172,7 @@ pub fn cli_run<T: EthSpec>(
     let wallet_password = read_wallet_password_from_cli(wallet_password_path, stdin_inputs)?;
 
     let mgr = WalletManager::open(&wallet_base_dir)
-        .map_err(|e| format!("Unable to open --{}: {:?}", BASE_DIR_FLAG, e))?;
+        .map_err(|e| format!("Unable to open --{}: {:?}", WALLETS_DIR_FLAG, e))?;
 
     let mut wallet = mgr
         .wallet_by_name(&wallet_name)

account_manager/src/validator/deposit.rs

@@ -46,16 +46,6 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
             The deposit contract address will be determined by the --testnet-dir flag on the \
             primary Lighthouse binary.",
         )
-        .arg(
-            Arg::with_name(VALIDATOR_DIR_FLAG)
-                .long(VALIDATOR_DIR_FLAG)
-                .value_name("VALIDATOR_DIRECTORY")
-                .help(
-                    "The path to the validator client data directory. \
-                    Defaults to ~/.lighthouse/validators",
-                )
-                .takes_value(true),
-        )
         .arg(
             Arg::with_name(VALIDATOR_FLAG)
                 .long(VALIDATOR_FLAG)
@@ -209,14 +199,10 @@ where
 pub fn cli_run<T: EthSpec>(
     matches: &ArgMatches<'_>,
     mut env: Environment<T>,
+    validator_dir: PathBuf,
 ) -> Result<(), String> {
     let log = env.core_context().log().clone();
 
-    let data_dir = clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        VALIDATOR_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("validators"),
-    )?;
     let validator: String = clap_utils::parse_required(matches, VALIDATOR_FLAG)?;
     let eth1_ipc_path: Option<PathBuf> = clap_utils::parse_optional(matches, ETH1_IPC_FLAG)?;
     let eth1_http_url: Option<String> = clap_utils::parse_optional(matches, ETH1_HTTP_FLAG)?;
@@ -225,7 +211,7 @@ pub fn cli_run<T: EthSpec>(
     let confirmation_batch_size: usize =
         clap_utils::parse_required(matches, CONFIRMATION_BATCH_SIZE_FLAG)?;
 
-    let manager = ValidatorManager::open(&data_dir)
+    let manager = ValidatorManager::open(&validator_dir)
         .map_err(|e| format!("Unable to read --{}: {:?}", VALIDATOR_DIR_FLAG, e))?;
 
     let validators = match validator.as_ref() {

account_manager/src/validator/import.rs

@@ -1,5 +1,4 @@
 use crate::wallet::create::STDIN_INPUTS_FLAG;
-use crate::{common::ensure_dir_exists, VALIDATOR_DIR_FLAG};
 use account_utils::{
     eth2_keystore::Keystore,
     read_password_from_user,
@@ -55,16 +54,6 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .required_unless(KEYSTORE_FLAG)
                 .takes_value(true),
         )
-        .arg(
-            Arg::with_name(VALIDATOR_DIR_FLAG)
-                .long(VALIDATOR_DIR_FLAG)
-                .value_name("VALIDATOR_DIRECTORY")
-                .help(
-                    "The path where the validator directories will be created. \
-                    Defaults to ~/.lighthouse/validators",
-                )
-                .takes_value(true),
-        )
         .arg(
             Arg::with_name(STDIN_INPUTS_FLAG)
                 .long(STDIN_INPUTS_FLAG)
@@ -77,19 +66,12 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
         )
 }
 
-pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
+pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), String> {
     let keystore: Option<PathBuf> = clap_utils::parse_optional(matches, KEYSTORE_FLAG)?;
     let keystores_dir: Option<PathBuf> = clap_utils::parse_optional(matches, DIR_FLAG)?;
-    let validator_dir = clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        VALIDATOR_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("validators"),
-    )?;
     let stdin_inputs = matches.is_present(STDIN_INPUTS_FLAG);
     let reuse_password = matches.is_present(REUSE_PASSWORD_FLAG);
 
-    ensure_dir_exists(&validator_dir)?;
-
     let mut defs = ValidatorDefinitions::open_or_create(&validator_dir)
         .map_err(|e| format!("Unable to open {}: {:?}", CONFIG_FILENAME, e))?;
 

account_manager/src/validator/list.rs

@@ -1,38 +1,21 @@
 use crate::VALIDATOR_DIR_FLAG;
-use clap::{App, Arg, ArgMatches};
+use clap::App;
 use std::path::PathBuf;
 use validator_dir::Manager as ValidatorManager;
 
 pub const CMD: &str = "list";
 
 pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
-    App::new(CMD)
+    App::new(CMD).about("Lists the names of all validators.")
-        .arg(
-            Arg::with_name(VALIDATOR_DIR_FLAG)
-                .long(VALIDATOR_DIR_FLAG)
-                .value_name("VALIDATOR_DIRECTORY")
-                .help(
-                    "The path to search for validator directories. \
-                    Defaults to ~/.lighthouse/validators",
-                )
-                .takes_value(true),
-        )
-        .about("Lists the names of all validators.")
 }
 
-pub fn cli_run(matches: &ArgMatches<'_>) -> Result<(), String> {
+pub fn cli_run(validator_dir: PathBuf) -> Result<(), String> {
-    let data_dir = clap_utils::parse_path_with_default_in_home_dir(
+    let mgr = ValidatorManager::open(&validator_dir)
-        matches,
-        VALIDATOR_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("validators"),
-    )?;
-
-    let mgr = ValidatorManager::open(&data_dir)
         .map_err(|e| format!("Unable to read --{}: {:?}", VALIDATOR_DIR_FLAG, e))?;
 
     for (name, _path) in mgr
         .directory_names()
-        .map_err(|e| format!("Unable to list wallets: {:?}", e))?
+        .map_err(|e| format!("Unable to list validators: {:?}", e))?
     {
         println!("{}", name)
     }

account_manager/src/validator/mod.rs

@@ -4,9 +4,11 @@ pub mod import;
 pub mod list;
 pub mod recover;
 
-use crate::common::base_wallet_dir;
+use crate::VALIDATOR_DIR_FLAG;
 use clap::{App, Arg, ArgMatches};
+use directory::{parse_path_or_default_with_flag, DEFAULT_VALIDATOR_DIR};
 use environment::Environment;
+use std::path::PathBuf;
 use types::EthSpec;
 
 pub const CMD: &str = "validator";
@@ -15,11 +17,16 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD)
         .about("Provides commands for managing Eth2 validators.")
         .arg(
-            Arg::with_name("base-dir")
+            Arg::with_name(VALIDATOR_DIR_FLAG)
-                .long("base-dir")
+                .long(VALIDATOR_DIR_FLAG)
-                .value_name("BASE_DIRECTORY")
+                .value_name("VALIDATOR_DIRECTORY")
-                .help("A path containing Eth2 EIP-2386 wallets. Defaults to ~/.lighthouse/wallets")
+                .help(
-                .takes_value(true),
+                    "The path to search for validator directories. \
+                    Defaults to ~/.lighthouse/{testnet}/validators",
+                )
+                .takes_value(true)
+                .global(true)
+                .conflicts_with("datadir"),
         )
         .subcommand(create::cli_app())
         .subcommand(deposit::cli_app())
@@ -29,14 +36,20 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
 }
 
 pub fn cli_run<T: EthSpec>(matches: &ArgMatches, env: Environment<T>) -> Result<(), String> {
-    let base_wallet_dir = base_wallet_dir(matches, "base-dir")?;
+    let validator_base_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_VALIDATOR_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, VALIDATOR_DIR_FLAG, DEFAULT_VALIDATOR_DIR)?
+    };
+    eprintln!("validator-dir path: {:?}", validator_base_dir);
 
     match matches.subcommand() {
-        (create::CMD, Some(matches)) => create::cli_run::<T>(matches, env, base_wallet_dir),
+        (create::CMD, Some(matches)) => create::cli_run::<T>(matches, env, validator_base_dir),
-        (deposit::CMD, Some(matches)) => deposit::cli_run::<T>(matches, env),
+        (deposit::CMD, Some(matches)) => deposit::cli_run::<T>(matches, env, validator_base_dir),
-        (import::CMD, Some(matches)) => import::cli_run(matches),
+        (import::CMD, Some(matches)) => import::cli_run(matches, validator_base_dir),
-        (list::CMD, Some(matches)) => list::cli_run(matches),
+        (list::CMD, Some(_)) => list::cli_run(validator_base_dir),
-        (recover::CMD, Some(matches)) => recover::cli_run(matches),
+        (recover::CMD, Some(matches)) => recover::cli_run(matches, validator_base_dir),
         (unknown, _) => Err(format!(
             "{} does not have a {} command. See --help",
             CMD, unknown

account_manager/src/validator/recover.rs

@@ -1,11 +1,13 @@
 use super::create::STORE_WITHDRAW_FLAG;
-use crate::common::{ensure_dir_exists, read_mnemonic_from_cli};
+use crate::common::read_mnemonic_from_cli;
 use crate::validator::create::COUNT_FLAG;
 use crate::wallet::create::STDIN_INPUTS_FLAG;
-use crate::{SECRETS_DIR_FLAG, VALIDATOR_DIR_FLAG};
+use crate::SECRETS_DIR_FLAG;
 use account_utils::eth2_keystore::{keypair_from_secret, Keystore, KeystoreBuilder};
 use account_utils::random_password;
 use clap::{App, Arg, ArgMatches};
+use directory::ensure_dir_exists;
+use directory::{parse_path_or_default_with_flag, DEFAULT_SECRET_DIR};
 use eth2_wallet::bip39::Seed;
 use eth2_wallet::{recover_validator_secret_from_mnemonic, KeyType, ValidatorKeystores};
 use std::path::PathBuf;
@@ -48,23 +50,13 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 )
                 .takes_value(true)
         )
-        .arg(
-            Arg::with_name(VALIDATOR_DIR_FLAG)
-                .long(VALIDATOR_DIR_FLAG)
-                .value_name("VALIDATOR_DIRECTORY")
-                .help(
-                    "The path where the validator directories will be created. \
-                    Defaults to ~/.lighthouse/validators",
-                )
-                .takes_value(true),
-        )
         .arg(
             Arg::with_name(SECRETS_DIR_FLAG)
                 .long(SECRETS_DIR_FLAG)
                 .value_name("SECRETS_DIR")
                 .help(
                     "The path where the validator keystore passwords will be stored. \
-                    Defaults to ~/.lighthouse/secrets",
+                    Defaults to ~/.lighthouse/{testnet}/secrets",
                 )
                 .takes_value(true),
         )
@@ -84,17 +76,13 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
         )
 }
 
-pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
+pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), String> {
-    let validator_dir = clap_utils::parse_path_with_default_in_home_dir(
+    let secrets_dir = if matches.value_of("datadir").is_some() {
-        matches,
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
-        VALIDATOR_DIR_FLAG,
+        path.join(DEFAULT_SECRET_DIR)
-        PathBuf::new().join(".lighthouse").join("validators"),
+    } else {
-    )?;
+        parse_path_or_default_with_flag(matches, SECRETS_DIR_FLAG, DEFAULT_SECRET_DIR)?
-    let secrets_dir = clap_utils::parse_path_with_default_in_home_dir(
+    };
-        matches,
-        SECRETS_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("secrets"),
-    )?;
     let first_index: u32 = clap_utils::parse_required(matches, FIRST_INDEX_FLAG)?;
     let count: u32 = clap_utils::parse_required(matches, COUNT_FLAG)?;
     let mnemonic_path: Option<PathBuf> = clap_utils::parse_optional(matches, MNEMONIC_FLAG)?;

account_manager/src/wallet/create.rs

@@ -1,5 +1,5 @@
 use crate::common::read_wallet_name_from_cli;
-use crate::BASE_DIR_FLAG;
+use crate::WALLETS_DIR_FLAG;
 use account_utils::{
     is_password_sufficiently_complex, random_password, read_password_from_user, strip_off_newlines,
 };
@@ -80,7 +80,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
         )
 }
 
-pub fn cli_run(matches: &ArgMatches, base_dir: PathBuf) -> Result<(), String> {
+pub fn cli_run(matches: &ArgMatches, wallet_base_dir: PathBuf) -> Result<(), String> {
     let mnemonic_output_path: Option<PathBuf> = clap_utils::parse_optional(matches, MNEMONIC_FLAG)?;
 
     // Create a new random mnemonic.
@@ -88,7 +88,7 @@ pub fn cli_run(matches: &ArgMatches, base_dir: PathBuf) -> Result<(), String> {
     // The `tiny-bip39` crate uses `thread_rng()` for this entropy.
     let mnemonic = Mnemonic::new(MnemonicType::Words12, Language::English);
 
-    let wallet = create_wallet_from_mnemonic(matches, &base_dir.as_path(), &mnemonic)?;
+    let wallet = create_wallet_from_mnemonic(matches, &wallet_base_dir.as_path(), &mnemonic)?;
 
     if let Some(path) = mnemonic_output_path {
         create_with_600_perms(&path, mnemonic.phrase().as_bytes())
@@ -121,7 +121,7 @@ pub fn cli_run(matches: &ArgMatches, base_dir: PathBuf) -> Result<(), String> {
 
 pub fn create_wallet_from_mnemonic(
     matches: &ArgMatches,
-    base_dir: &Path,
+    wallet_base_dir: &Path,
     mnemonic: &Mnemonic,
 ) -> Result<LockedWallet, String> {
     let name: Option<String> = clap_utils::parse_optional(matches, NAME_FLAG)?;
@@ -134,8 +134,8 @@ pub fn create_wallet_from_mnemonic(
         unknown => return Err(format!("--{} {} is not supported", TYPE_FLAG, unknown)),
     };
 
-    let mgr = WalletManager::open(&base_dir)
+    let mgr = WalletManager::open(&wallet_base_dir)
-        .map_err(|e| format!("Unable to open --{}: {:?}", BASE_DIR_FLAG, e))?;
+        .map_err(|e| format!("Unable to open --{}: {:?}", WALLETS_DIR_FLAG, e))?;
 
     let wallet_password: PlainText = match wallet_password_path {
         Some(path) => {

account_manager/src/wallet/list.rs

@@ -1,4 +1,4 @@
-use crate::BASE_DIR_FLAG;
+use crate::WALLETS_DIR_FLAG;
 use clap::App;
 use eth2_wallet_manager::WalletManager;
 use std::path::PathBuf;
@@ -9,9 +9,9 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD).about("Lists the names of all wallets.")
 }
 
-pub fn cli_run(base_dir: PathBuf) -> Result<(), String> {
+pub fn cli_run(wallet_base_dir: PathBuf) -> Result<(), String> {
-    let mgr = WalletManager::open(&base_dir)
+    let mgr = WalletManager::open(&wallet_base_dir)
-        .map_err(|e| format!("Unable to open --{}: {:?}", BASE_DIR_FLAG, e))?;
+        .map_err(|e| format!("Unable to open --{}: {:?}", WALLETS_DIR_FLAG, e))?;
 
     for (name, _uuid) in mgr
         .wallets()

account_manager/src/wallet/mod.rs

@@ -2,11 +2,10 @@ pub mod create;
 pub mod list;
 pub mod recover;
 
-use crate::{
+use crate::WALLETS_DIR_FLAG;
-    common::{base_wallet_dir, ensure_dir_exists},
-    BASE_DIR_FLAG,
-};
 use clap::{App, Arg, ArgMatches};
+use directory::{ensure_dir_exists, parse_path_or_default_with_flag, DEFAULT_WALLET_DIR};
+use std::path::PathBuf;
 
 pub const CMD: &str = "wallet";
 
@@ -14,11 +13,13 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD)
         .about("Manage wallets, from which validator keys can be derived.")
         .arg(
-            Arg::with_name(BASE_DIR_FLAG)
+            Arg::with_name(WALLETS_DIR_FLAG)
-                .long(BASE_DIR_FLAG)
+                .long(WALLETS_DIR_FLAG)
-                .value_name("BASE_DIRECTORY")
+                .value_name("WALLETS_DIRECTORY")
-                .help("A path containing Eth2 EIP-2386 wallets. Defaults to ~/.lighthouse/wallets")
+                .help("A path containing Eth2 EIP-2386 wallets. Defaults to ~/.lighthouse/{testnet}/wallets")
-                .takes_value(true),
+                .takes_value(true)
+                .global(true)
+                .conflicts_with("datadir"),
         )
         .subcommand(create::cli_app())
         .subcommand(list::cli_app())
@@ -26,13 +27,20 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
 }
 
 pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
-    let base_dir = base_wallet_dir(matches, BASE_DIR_FLAG)?;
+    let wallet_base_dir = if matches.value_of("datadir").is_some() {
-    ensure_dir_exists(&base_dir)?;
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_WALLET_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, WALLETS_DIR_FLAG, DEFAULT_WALLET_DIR)?
+    };
+    ensure_dir_exists(&wallet_base_dir)?;
+
+    eprintln!("wallet-dir path: {:?}", wallet_base_dir);
 
     match matches.subcommand() {
-        (create::CMD, Some(matches)) => create::cli_run(matches, base_dir),
+        (create::CMD, Some(matches)) => create::cli_run(matches, wallet_base_dir),
-        (list::CMD, Some(_)) => list::cli_run(base_dir),
+        (list::CMD, Some(_)) => list::cli_run(wallet_base_dir),
-        (recover::CMD, Some(matches)) => recover::cli_run(matches, base_dir),
+        (recover::CMD, Some(matches)) => recover::cli_run(matches, wallet_base_dir),
         (unknown, _) => Err(format!(
             "{} does not have a {} command. See --help",
             CMD, unknown

beacon_node/Cargo.toml

@@ -30,6 +30,7 @@ tokio = { version = "0.2.21", features = ["time"] }
 exit-future = "0.2.0"
 dirs = "2.0.2"
 logging = { path = "../common/logging" }
+directory = {path = "../common/directory"}
 futures = "0.3.5"
 environment = { path = "../lighthouse/environment" }
 genesis = { path = "genesis" }

beacon_node/client/Cargo.toml

@@ -41,3 +41,4 @@ lazy_static = "1.4.0"
 lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
 time = "0.2.16"
 bus = "2.2.3"
+directory = {path = "../../common/directory"}

beacon_node/client/src/config.rs

@@ -1,11 +1,10 @@
+use directory::DEFAULT_ROOT_DIR;
 use network::NetworkConfig;
 use serde_derive::{Deserialize, Serialize};
 use std::fs;
 use std::path::PathBuf;
 use types::Graffiti;
 
-pub const DEFAULT_DATADIR: &str = ".lighthouse";
-
 /// The number initial validators when starting the `Minimal`.
 const TESTNET_SPEC_CONSTANTS: &str = "minimal";
 
@@ -72,7 +71,7 @@ pub struct Config {
 impl Default for Config {
     fn default() -> Self {
         Self {
-            data_dir: PathBuf::from(DEFAULT_DATADIR),
+            data_dir: PathBuf::from(DEFAULT_ROOT_DIR),
             db_name: "chain_db".to_string(),
             freezer_db_path: None,
             log_file: PathBuf::from(""),

beacon_node/eth2_libp2p/Cargo.toml

@@ -36,6 +36,7 @@ discv5 = { version = "0.1.0-alpha.12", features = ["libp2p"] }
 tiny-keccak = "2.0.2"
 environment = { path = "../../lighthouse/environment" }
 rand = "0.7.3"
+directory = { path = "../../common/directory" }
 regex = "1.3.9"
 
 [dependencies.libp2p]

beacon_node/eth2_libp2p/src/config.rs

@@ -1,5 +1,8 @@
 use crate::types::GossipKind;
 use crate::{Enr, PeerIdSerialized};
+use directory::{
+    DEFAULT_BEACON_NODE_DIR, DEFAULT_HARDCODED_TESTNET, DEFAULT_NETWORK_DIR, DEFAULT_ROOT_DIR,
+};
 use discv5::{Discv5Config, Discv5ConfigBuilder};
 use libp2p::gossipsub::{
     GossipsubConfig, GossipsubConfigBuilder, GossipsubMessage, MessageId, ValidationMode,
@@ -74,9 +77,14 @@ pub struct Config {
 impl Default for Config {
     /// Generate a default network configuration.
     fn default() -> Self {
-        let mut network_dir = dirs::home_dir().unwrap_or_else(|| PathBuf::from("."));
+        // WARNING: this directory default should be always overrided with parameters
-        network_dir.push(".lighthouse");
+        // from cli for specific networks.
-        network_dir.push("network");
+        let network_dir = dirs::home_dir()
+            .unwrap_or_else(|| PathBuf::from("."))
+            .join(DEFAULT_ROOT_DIR)
+            .join(DEFAULT_HARDCODED_TESTNET)
+            .join(DEFAULT_BEACON_NODE_DIR)
+            .join(DEFAULT_NETWORK_DIR);
 
         // The function used to generate a gossipsub message id
         // We use the first 8 bytes of SHA256(data) for content addressing

beacon_node/src/config.rs

@@ -1,7 +1,8 @@
 use beacon_chain::builder::PUBKEY_CACHE_FILENAME;
 use clap::ArgMatches;
 use clap_utils::BAD_TESTNET_DIR_MESSAGE;
-use client::{config::DEFAULT_DATADIR, ClientConfig, ClientGenesis};
+use client::{ClientConfig, ClientGenesis};
+use directory::{DEFAULT_BEACON_NODE_DIR, DEFAULT_NETWORK_DIR, DEFAULT_ROOT_DIR};
 use eth2_libp2p::{multiaddr::Protocol, Enr, Multiaddr, NetworkConfig, PeerIdSerialized};
 use eth2_testnet_config::Eth2TestnetConfig;
 use slog::{crit, info, warn, Logger};
@@ -13,9 +14,6 @@ use std::net::{TcpListener, UdpSocket};
 use std::path::PathBuf;
 use types::{ChainSpec, EthSpec, GRAFFITI_BYTES_LEN};
 
-pub const BEACON_NODE_DIR: &str = "beacon";
-pub const NETWORK_DIR: &str = "network";
-
 /// Gets the fully-initialized global client.
 ///
 /// The top-level `clap` arguments should be provided as `cli_args`.
@@ -295,7 +293,7 @@ pub fn set_network_config(
     if let Some(dir) = cli_args.value_of("network-dir") {
         config.network_dir = PathBuf::from(dir);
     } else {
-        config.network_dir = data_dir.join(NETWORK_DIR);
+        config.network_dir = data_dir.join(DEFAULT_NETWORK_DIR);
     };
 
     if let Some(listen_address_str) = cli_args.value_of("listen-address") {
@@ -456,11 +454,18 @@ pub fn get_data_dir(cli_args: &ArgMatches) -> PathBuf {
     // Read the `--datadir` flag.
     //
     // If it's not present, try and find the home directory (`~`) and push the default data
-    // directory onto it.
+    // directory and the testnet name onto it.
+
     cli_args
         .value_of("datadir")
-        .map(|path| PathBuf::from(path).join(BEACON_NODE_DIR))
+        .map(|path| PathBuf::from(path).join(DEFAULT_BEACON_NODE_DIR))
-        .or_else(|| dirs::home_dir().map(|home| home.join(DEFAULT_DATADIR).join(BEACON_NODE_DIR)))
+        .or_else(|| {
+            dirs::home_dir().map(|home| {
+                home.join(DEFAULT_ROOT_DIR)
+                    .join(directory::get_testnet_name(cli_args))
+                    .join(DEFAULT_BEACON_NODE_DIR)
+            })
+        })
         .unwrap_or_else(|| PathBuf::from("."))
 }
 

book/src/key-management.md

@@ -40,12 +40,12 @@ keypairs. Creating a single validator looks like this:
 	- `lighthouse account validator create --wallet-name wally --wallet-password wally.pass --count 1`
 
 
-In step (1), we created a wallet in `~/.lighthouse/wallets` with the name
+In step (1), we created a wallet in `~/.lighthouse/{testnet}/wallets` with the name
 `wally`. We encrypted this using a pre-defined password in the
 `wally.pass` file. Then, in step (2), we created one new validator in the
-`~/.lighthouse/validators` directory using `wally` (unlocking it with
+`~/.lighthouse/{testnet}/validators` directory using `wally` (unlocking it with
 `wally.pass`) and storing the passwords to the validators voting key in
-`~/.lighthouse/secrets`.
+`~/.lighthouse/{testnet}/secrets`.
 
 Thanks to the hierarchical key derivation scheme, we can delete all of the
 aforementioned directories and then regenerate them as long as we remembered
@@ -63,14 +63,16 @@ There are three important directories in Lighthouse validator key management:
 
 - `wallets/`: contains encrypted wallets which are used for hierarchical
 	key derivation.
-	- Defaults to `~/.lighthouse/wallets`
+	- Defaults to `~/.lighthouse/{testnet}/wallets`
 - `validators/`: contains a directory for each validator containing
 	encrypted keystores and other validator-specific data.
-	- Defaults to `~/.lighthouse/validators`
+	- Defaults to `~/.lighthouse/{testnet}/validators`
 - `secrets/`: since the validator signing keys are "hot", the validator process
 	needs access to the passwords to decrypt the keystores in the validators
 	dir. These passwords are stored here.
-	- Defaults to `~/.lighthouse/secrets`
+	- Defaults to `~/.lighthouse/{testnet}/secrets`
+
+where `testnet` is the name of the testnet passed in the `--testnet` parameter (default is `medalla`).
 
 When the validator client boots, it searches the `validators/` for directories
 containing voting keystores. When it discovers a keystore, it searches the

book/src/validator-create.md

@@ -41,7 +41,7 @@ OPTIONS:
             The GWEI value of the deposit amount. Defaults to the minimum amount required for an active validator
             (MAX_EFFECTIVE_BALANCE)
         --secrets-dir <SECRETS_DIR>
-            The path where the validator keystore passwords will be stored. Defaults to ~/.lighthouse/secrets
+            The path where the validator keystore passwords will be stored. Defaults to ~/.lighthouse/{testnet}/secrets
 
     -s, --spec <TITLE>
             Specifies the default eth2 spec type. [default: mainnet]  [possible values: mainnet, minimal, interop]
@@ -53,7 +53,7 @@ OPTIONS:
             Path to directory containing eth2_testnet specs. Defaults to a hard-coded Lighthouse testnet. Only effective
             if there is no existing database.
         --validator-dir <VALIDATOR_DIRECTORY>
-            The path where the validator directories will be created. Defaults to ~/.lighthouse/validators
+            The path where the validator directories will be created. Defaults to ~/.lighthouse/{testnet}/validators
 
         --wallet-name <WALLET_NAME>                   Use the wallet identified by this name
         --wallet-password <WALLET_PASSWORD_PATH>
@@ -73,10 +73,12 @@ This command will:
 
 - Derive a single new BLS keypair from `wally`, updating it so that it generates a
     new key next time.
-- Create a new directory in `~/.lighthouse/validators` containing:
+- Create a new directory in `~/.lighthouse/{testnet}/validators` containing:
     - An encrypted keystore containing the validators voting keypair.
 	- An `eth1_deposit_data.rlp` assuming the default deposit amount (`32 ETH`
 		for most testnets and mainnet) which can be submitted to the deposit
 		contract for the medalla testnet. Other testnets can be set via the
 		`--testnet` CLI param.
-- Store a password to the validators voting keypair in `~/.lighthouse/secrets`.
+- Store a password to the validators voting keypair in `~/.lighthouse/{testnet}/secrets`.
+
+where `testnet` is the name of the testnet passed in the `--testnet` parameter (default is `medalla`).

book/src/validator-management.md

@@ -16,7 +16,7 @@ useful.
 ## Introducing the `validator_definitions.yml` file
 
 The `validator_definitions.yml` file is located in the `validator-dir`, which
-defaults to `~/.lighthouse/validators`. It is a
+defaults to `~/.lighthouse/{testnet}/validators`. It is a
 [YAML](https://en.wikipedia.org/wiki/YAML) encoded file defining exactly which
 validators the validator client will (and won't) act for.
 
@@ -92,7 +92,7 @@ name identical to the `voting_public_key` value.
 Lets assume the following directory structure:
 
 ```
-~/.lighthouse/validators
+~/.lighthouse/{testnet}/validators
 ├── john
 │   └── voting-keystore.json
 ├── sally
@@ -135,7 +135,7 @@ In order for the validator client to decrypt the validators, they will need to
 ensure their `secrets-dir` is organised as below:
 
 ```
-~/.lighthouse/secrets
+~/.lighthouse/{testnet}/secrets
 ├── 0xa5566f9ec3c6e1fdf362634ebec9ef7aceb0e460e5079714808388e5d48f4ae1e12897fed1bea951c17fa389d511e477
 ├── 0xaa440c566fcf34dedf233baf56cf5fb05bb420d9663b4208272545608c27c13d5b08174518c758ecd814f158f2b4a337
 └── 0x87a580d31d7bc69069b55f5a01995a610dd391a26dc9e36e81057a17211983a79266800ab8531f21f1083d7d84085007

common/directory/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "directory"
+version = "0.1.0"
+authors = ["pawan <pawandhananjay@gmail.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+clap = "2.33.0"
+clap_utils = {path = "../clap_utils"}
+dirs = "2.0.2"
+eth2_testnet_config = { path = "../eth2_testnet_config" }

common/directory/src/lib.rs

@@ -0,0 +1,60 @@
+use clap::ArgMatches;
+pub use eth2_testnet_config::DEFAULT_HARDCODED_TESTNET;
+use std::fs::create_dir_all;
+use std::path::{Path, PathBuf};
+
+/// Names for the default directories.
+pub const DEFAULT_ROOT_DIR: &str = ".lighthouse";
+pub const DEFAULT_BEACON_NODE_DIR: &str = "beacon";
+pub const DEFAULT_NETWORK_DIR: &str = "network";
+pub const DEFAULT_VALIDATOR_DIR: &str = "validators";
+pub const DEFAULT_SECRET_DIR: &str = "secrets";
+pub const DEFAULT_WALLET_DIR: &str = "wallets";
+
+/// Base directory name for unnamed testnets passed through the --testnet-dir flag
+pub const CUSTOM_TESTNET_DIR: &str = "custom";
+
+/// Gets the testnet directory name
+///
+/// Tries to get the name first from the "testnet" flag,
+/// if not present, then checks the "testnet-dir" flag and returns a custom name
+/// If neither flags are present, returns the default hardcoded network name.
+pub fn get_testnet_name(matches: &ArgMatches) -> String {
+    if let Some(testnet_name) = matches.value_of("testnet") {
+        testnet_name.to_string()
+    } else if matches.value_of("testnet-dir").is_some() {
+        CUSTOM_TESTNET_DIR.to_string()
+    } else {
+        eth2_testnet_config::DEFAULT_HARDCODED_TESTNET.to_string()
+    }
+}
+
+/// Checks if a directory exists in the given path and creates a directory if it does not exist.
+pub fn ensure_dir_exists<P: AsRef<Path>>(path: P) -> Result<(), String> {
+    let path = path.as_ref();
+
+    if !path.exists() {
+        create_dir_all(path).map_err(|e| format!("Unable to create {:?}: {:?}", path, e))?;
+    }
+
+    Ok(())
+}
+
+/// If `arg` is in `matches`, parses the value as a path.
+///
+/// Otherwise, attempts to find the default directory for the `testnet` from the `matches`
+/// and appends `flag` to it.
+pub fn parse_path_or_default_with_flag(
+    matches: &ArgMatches,
+    arg: &'static str,
+    flag: &str,
+) -> Result<PathBuf, String> {
+    clap_utils::parse_path_with_default_in_home_dir(
+        matches,
+        arg,
+        PathBuf::new()
+            .join(DEFAULT_ROOT_DIR)
+            .join(get_testnet_name(matches))
+            .join(flag),
+    )
+}

consensus/types/Cargo.toml

@@ -12,7 +12,6 @@ harness = false
 bls = { path = "../../crypto/bls" }
 compare_fields = { path = "../../common/compare_fields" }
 compare_fields_derive = { path = "../../common/compare_fields_derive" }
-dirs = "2.0.2"
 eth2_interop_keypairs = { path = "../../common/eth2_interop_keypairs" }
 ethereum-types = "0.9.1"
 eth2_hashing = "0.1.0"

consensus/types/src/test_utils/builders/testing_beacon_state_builder.rs

@@ -4,21 +4,9 @@ use crate::*;
 use bls::get_withdrawal_credentials;
 use log::debug;
 use rayon::prelude::*;
-use std::path::PathBuf;
 
 pub const KEYPAIRS_FILE: &str = "keypairs.raw_keypairs";
 
-/// Returns the directory where the generated keypairs should be stored.
-///
-/// It is either `$HOME/.lighthouse/keypairs.raw_keypairs` or, if `$HOME` is not available,
-/// `./keypairs.raw_keypairs`.
-pub fn keypairs_path() -> PathBuf {
-    let dir = dirs::home_dir()
-        .map(|home| (home.join(".lighthouse")))
-        .unwrap_or_else(|| PathBuf::from(""));
-    dir.join(KEYPAIRS_FILE)
-}
-
 /// Builds a beacon state to be used for testing purposes.
 ///
 /// This struct should **never be used for production purposes.**

lcli/Cargo.toml

@@ -35,3 +35,4 @@ validator_dir = { path = "../common/validator_dir", features = ["insecure_keys"]
 rand = "0.7.2"
 eth2_keystore = { path = "../crypto/eth2_keystore" }
 lighthouse_version = { path = "../common/lighthouse_version" }
+directory = { path = "../common/directory" }

lcli/src/eth1_genesis.rs

@@ -20,7 +20,7 @@ pub fn run<T: EthSpec>(mut env: Environment<T>, matches: &ArgMatches<'_>) -> Res
         .and_then(|dir| dir.parse::<PathBuf>().map_err(|_| ()))
         .unwrap_or_else(|_| {
             dirs::home_dir()
-                .map(|home| home.join(".lighthouse").join("testnet"))
+                .map(|home| home.join(directory::DEFAULT_ROOT_DIR).join("testnet"))
                 .expect("should locate home directory")
         });
 

lcli/src/interop_genesis.rs

@@ -31,7 +31,7 @@ pub fn run<T: EthSpec>(mut env: Environment<T>, matches: &ArgMatches) -> Result<
         .and_then(|dir| dir.parse::<PathBuf>().map_err(|_| ()))
         .unwrap_or_else(|_| {
             dirs::home_dir()
-                .map(|home| home.join(".lighthouse").join("testnet"))
+                .map(|home| home.join(directory::DEFAULT_ROOT_DIR).join("testnet"))
                 .expect("should locate home directory")
         });
 

lcli/src/new_testnet.rs

@@ -10,7 +10,7 @@ pub fn run<T: EthSpec>(matches: &ArgMatches) -> Result<(), String> {
     let testnet_dir_path = parse_path_with_default_in_home_dir(
         matches,
         "testnet-dir",
-        PathBuf::from(".lighthouse/testnet"),
+        PathBuf::from(directory::DEFAULT_ROOT_DIR).join("testnet"),
     )?;
     let deposit_contract_address: Address = parse_required(matches, "deposit-contract-address")?;
     let deposit_contract_deploy_block = parse_required(matches, "deposit-contract-deploy-block")?;

lighthouse/Cargo.toml

@@ -31,9 +31,10 @@ validator_client = { "path" = "../validator_client" }
 account_manager = { "path" = "../account_manager" }
 clap_utils = { path = "../common/clap_utils" }
 eth2_testnet_config = { path = "../common/eth2_testnet_config" }
+directory = { path = "../common/directory" }
 lighthouse_version = { path = "../common/lighthouse_version" }
+account_utils = { path = "../common/account_utils" }
 
 [dev-dependencies]
 tempfile = "3.1.0"
 validator_dir = { path = "../common/validator_dir" }
-account_utils = { path = "../common/account_utils" }

lighthouse/src/main.rs

@@ -10,7 +10,6 @@ use std::process::exit;
 use types::EthSpec;
 use validator_client::ProductionValidatorClient;
 
-pub const DEFAULT_DATA_DIR: &str = ".lighthouse";
 pub const ETH2_CONFIG_FILENAME: &str = "eth2-spec.toml";
 
 fn bls_library_name() -> &'static str {
@@ -91,7 +90,10 @@ fn main() {
                 .short("d")
                 .value_name("DIR")
                 .global(true)
-                .help("Data directory for lighthouse keys and databases.")
+                .help(
+                    "Root data directory for lighthouse keys and databases. \
+                    Defaults to $HOME/.lighthouse/{default-testnet}, \
+                    currently, $HOME/.lighthouse/medalla")
                 .takes_value(true),
         )
         .arg(

lighthouse/tests/account_manager.rs

@@ -11,7 +11,7 @@ use account_manager::{
         list::CMD as LIST_CMD,
         CMD as WALLET_CMD,
     },
-    BASE_DIR_FLAG, CMD as ACCOUNT_CMD, *,
+    CMD as ACCOUNT_CMD, WALLETS_DIR_FLAG, *,
 };
 use account_utils::{
     eth2_keystore::KeystoreBuilder,
@@ -73,7 +73,7 @@ fn dir_child_count<P: AsRef<Path>>(dir: P) -> usize {
 fn list_wallets<P: AsRef<Path>>(base_dir: P) -> Vec<String> {
     let output = output_result(
         wallet_cmd()
-            .arg(format!("--{}", BASE_DIR_FLAG))
+            .arg(format!("--{}", WALLETS_DIR_FLAG))
             .arg(base_dir.as_ref().as_os_str())
             .arg(LIST_CMD),
     )
@@ -97,7 +97,7 @@ fn create_wallet<P: AsRef<Path>>(
 ) -> Result<Output, String> {
     output_result(
         wallet_cmd()
-            .arg(format!("--{}", BASE_DIR_FLAG))
+            .arg(format!("--{}", WALLETS_DIR_FLAG))
             .arg(base_dir.as_ref().as_os_str())
             .arg(CREATE_CMD)
             .arg(format!("--{}", NAME_FLAG))
@@ -233,15 +233,15 @@ impl TestValidator {
         store_withdrawal_key: bool,
     ) -> Result<Vec<String>, String> {
         let mut cmd = validator_cmd();
-        cmd.arg(format!("--{}", BASE_DIR_FLAG))
+        cmd.arg(format!("--{}", VALIDATOR_DIR_FLAG))
-            .arg(self.wallet.base_dir().into_os_string())
+            .arg(self.validator_dir.clone().into_os_string())
             .arg(CREATE_CMD)
+            .arg(format!("--{}", WALLETS_DIR_FLAG))
+            .arg(self.wallet.base_dir().into_os_string())
             .arg(format!("--{}", WALLET_NAME_FLAG))
             .arg(&self.wallet.name)
             .arg(format!("--{}", WALLET_PASSWORD_FLAG))
             .arg(self.wallet.password_path().into_os_string())
-            .arg(format!("--{}", VALIDATOR_DIR_FLAG))
-            .arg(self.validator_dir.clone().into_os_string())
             .arg(format!("--{}", SECRETS_DIR_FLAG))
             .arg(self.secrets_dir.clone().into_os_string())
             .arg(format!("--{}", DEPOSIT_GWEI_FLAG))
@@ -375,13 +375,6 @@ fn validator_create() {
     assert_eq!(dir_child_count(validator_dir.path()), 6);
 }
 
-/// Returns the `lighthouse account validator import` command.
-fn validator_import_cmd() -> Command {
-    let mut cmd = validator_cmd();
-    cmd.arg(IMPORT_CMD);
-    cmd
-}
-
 #[test]
 fn validator_import_launchpad() {
     const PASSWORD: &str = "cats";
@@ -407,12 +400,13 @@ fn validator_import_launchpad() {
     // Create a not-keystore file in the src dir.
     File::create(src_dir.path().join(NOT_KEYSTORE_NAME)).unwrap();
 
-    let mut child = validator_import_cmd()
+    let mut child = validator_cmd()
+        .arg(format!("--{}", VALIDATOR_DIR_FLAG))
+        .arg(dst_dir.path().as_os_str())
+        .arg(IMPORT_CMD)
         .arg(format!("--{}", STDIN_INPUTS_FLAG)) // Using tty does not work well with tests.
         .arg(format!("--{}", import::DIR_FLAG))
         .arg(src_dir.path().as_os_str())
-        .arg(format!("--{}", VALIDATOR_DIR_FLAG))
-        .arg(dst_dir.path().as_os_str())
         .stderr(Stdio::piped())
         .stdin(Stdio::piped())
         .spawn()

testing/node_test_rig/src/lib.rs

@@ -96,7 +96,7 @@ pub fn testing_client_config() -> ClientConfig {
 /// This struct is separate to `LocalValidatorClient` to allow for pre-computation of validator
 /// keypairs since the task is quite resource intensive.
 pub struct ValidatorFiles {
-    pub datadir: TempDir,
+    pub validator_dir: TempDir,
     pub secrets_dir: TempDir,
 }
 
@@ -110,7 +110,7 @@ impl ValidatorFiles {
             .map_err(|e| format!("Unable to create VC secrets dir: {:?}", e))?;
 
         Ok(Self {
-            datadir,
+            validator_dir: datadir,
             secrets_dir,
         })
     }
@@ -120,7 +120,7 @@ impl ValidatorFiles {
         let this = Self::new()?;
 
         build_deterministic_validator_dirs(
-            this.datadir.path().into(),
+            this.validator_dir.path().into(),
             this.secrets_dir.path().into(),
             keypair_indices,
         )
@@ -170,7 +170,7 @@ impl<E: EthSpec> LocalValidatorClient<E> {
         mut config: ValidatorConfig,
         files: ValidatorFiles,
     ) -> Result<Self, String> {
-        config.data_dir = files.datadir.path().into();
+        config.validator_dir = files.validator_dir.path().into();
         config.secrets_dir = files.secrets_dir.path().into();
 
         ProductionValidatorClient::new(context, config)

validator_client/Cargo.toml

@@ -31,6 +31,7 @@ slog-term = "2.5.0"
 tokio = { version = "0.2.21", features = ["time"] }
 futures = { version = "0.3.5", features = ["compat"] }
 dirs = "2.0.2"
+directory = {path = "../common/directory"}
 logging = { path = "../common/logging" }
 environment = { path = "../lighthouse/environment" }
 parking_lot = "0.11.0"

validator_client/src/cli.rs

@@ -16,6 +16,19 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .default_value(&DEFAULT_HTTP_SERVER)
                 .takes_value(true),
         )
+        .arg(
+            Arg::with_name("validators-dir")
+                .long("validators-dir")
+                .value_name("VALIDATORS_DIR")
+                .help(
+                    "The directory which contains the validator keystores, deposit data for \
+                    each validator along with the common slashing protection database \
+                    and the validator_definitions.yml"
+                )
+                .takes_value(true)
+                .conflicts_with("datadir")
+                .requires("secrets-dir")
+        )
         .arg(
             Arg::with_name("secrets-dir")
                 .long("secrets-dir")
@@ -24,9 +37,11 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                     "The directory which contains the password to unlock the validator \
                     voting keypairs. Each password should be contained in a file where the \
                     name is the 0x-prefixed hex representation of the validators voting public \
-                    key. Defaults to ~/.lighthouse/secrets.",
+                    key. Defaults to ~/.lighthouse/{testnet}/secrets.",
                 )
-                .takes_value(true),
+                .takes_value(true)
+                .conflicts_with("datadir")
+                .requires("validators-dir"),
         )
         .arg(Arg::with_name("auto-register").long("auto-register").help(
             "If present, the validator client will register any new signing keys with \
@@ -48,6 +63,16 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 that might also be using the same keystores."
             )
         )
+        .arg(
+            Arg::with_name("strict-slashing-protection")
+            .long("strict-slashing-protection")
+            .help(
+                "If present, do not create a new slashing database. This is to ensure that users \
+                do not accidentally get slashed in case their slashing protection db ends up in the \
+                wrong directory during directory restructure and vc creates a new empty db and \
+                re-registers all validators."
+            )
+        )
         .arg(
             Arg::with_name("disable-auto-discover")
             .long("disable-auto-discover")

validator_client/src/config.rs

@@ -1,12 +1,14 @@
 use clap::ArgMatches;
-use clap_utils::{parse_optional, parse_path_with_default_in_home_dir};
+use clap_utils::{parse_optional, parse_required};
+use directory::{
+    get_testnet_name, DEFAULT_HARDCODED_TESTNET, DEFAULT_ROOT_DIR, DEFAULT_SECRET_DIR,
+    DEFAULT_VALIDATOR_DIR,
+};
 use serde_derive::{Deserialize, Serialize};
 use std::path::PathBuf;
 use types::{Graffiti, GRAFFITI_BYTES_LEN};
 
 pub const DEFAULT_HTTP_SERVER: &str = "http://localhost:5052/";
-pub const DEFAULT_DATA_DIR: &str = ".lighthouse/validators";
-pub const DEFAULT_SECRETS_DIR: &str = ".lighthouse/secrets";
 /// Path to the slashing protection database within the datadir.
 pub const SLASHING_PROTECTION_FILENAME: &str = "slashing_protection.sqlite";
 
@@ -14,7 +16,7 @@ pub const SLASHING_PROTECTION_FILENAME: &str = "slashing_protection.sqlite";
 #[derive(Clone, Serialize, Deserialize)]
 pub struct Config {
     /// The data directory, which stores all validator databases
-    pub data_dir: PathBuf,
+    pub validator_dir: PathBuf,
     /// The directory containing the passwords to unlock validator keystores.
     pub secrets_dir: PathBuf,
     /// The http endpoint of the beacon node API.
@@ -28,6 +30,8 @@ pub struct Config {
     pub delete_lockfiles: bool,
     /// If true, don't scan the validators dir for new keystores.
     pub disable_auto_discover: bool,
+    /// If true, don't re-register existing validators in definitions.yml for slashing protection.
+    pub strict_slashing_protection: bool,
     /// Graffiti to be inserted everytime we create a block.
     pub graffiti: Option<Graffiti>,
 }
@@ -35,19 +39,22 @@ pub struct Config {
 impl Default for Config {
     /// Build a new configuration from defaults.
     fn default() -> Self {
-        let data_dir = dirs::home_dir()
+        // WARNING: these directory defaults should be always overrided with parameters
-            .map(|home| home.join(DEFAULT_DATA_DIR))
+        // from cli for specific networks.
-            .unwrap_or_else(|| PathBuf::from("."));
+        let base_dir = dirs::home_dir()
-        let secrets_dir = dirs::home_dir()
+            .unwrap_or_else(|| PathBuf::from("."))
-            .map(|home| home.join(DEFAULT_SECRETS_DIR))
+            .join(DEFAULT_ROOT_DIR)
-            .unwrap_or_else(|| PathBuf::from("."));
+            .join(DEFAULT_HARDCODED_TESTNET);
+        let validator_dir = base_dir.join(DEFAULT_VALIDATOR_DIR);
+        let secrets_dir = base_dir.join(DEFAULT_SECRET_DIR);
         Self {
-            data_dir,
+            validator_dir,
             secrets_dir,
             http_server: DEFAULT_HTTP_SERVER.to_string(),
             allow_unsynced_beacon_node: false,
             delete_lockfiles: false,
             disable_auto_discover: false,
+            strict_slashing_protection: false,
             graffiti: None,
         }
     }
@@ -59,16 +66,39 @@ impl Config {
     pub fn from_cli(cli_args: &ArgMatches) -> Result<Config, String> {
         let mut config = Config::default();
 
-        config.data_dir = parse_path_with_default_in_home_dir(
+        let default_root_dir = dirs::home_dir()
-            cli_args,
+            .map(|home| home.join(DEFAULT_ROOT_DIR))
-            "datadir",
+            .unwrap_or_else(|| PathBuf::from("."));
-            PathBuf::from(".lighthouse").join("validators"),
-        )?;
 
-        if !config.data_dir.exists() {
+        let (mut validator_dir, mut secrets_dir) = (None, None);
+        if cli_args.value_of("datadir").is_some() {
+            let base_dir: PathBuf = parse_required(cli_args, "datadir")?;
+            validator_dir = Some(base_dir.join(DEFAULT_VALIDATOR_DIR));
+            secrets_dir = Some(base_dir.join(DEFAULT_SECRET_DIR));
+        }
+        if cli_args.value_of("validators-dir").is_some()
+            && cli_args.value_of("secrets-dir").is_some()
+        {
+            validator_dir = Some(parse_required(cli_args, "validators-dir")?);
+            secrets_dir = Some(parse_required(cli_args, "secrets-dir")?);
+        }
+
+        config.validator_dir = validator_dir.unwrap_or_else(|| {
+            default_root_dir
+                .join(get_testnet_name(cli_args))
+                .join(DEFAULT_VALIDATOR_DIR)
+        });
+
+        config.secrets_dir = secrets_dir.unwrap_or_else(|| {
+            default_root_dir
+                .join(get_testnet_name(cli_args))
+                .join(DEFAULT_SECRET_DIR)
+        });
+
+        if !config.validator_dir.exists() {
             return Err(format!(
-                "The directory for validator data  (--datadir) does not exist: {:?}",
+                "The directory for validator data does not exist: {:?}",
-                config.data_dir
+                config.validator_dir
             ));
         }
 
@@ -79,10 +109,7 @@ impl Config {
         config.allow_unsynced_beacon_node = cli_args.is_present("allow-unsynced");
         config.delete_lockfiles = cli_args.is_present("delete-lockfiles");
         config.disable_auto_discover = cli_args.is_present("disable-auto-discover");
-
+        config.strict_slashing_protection = cli_args.is_present("strict-slashing-protection");
-        if let Some(secrets_dir) = parse_optional(cli_args, "secrets-dir")? {
-            config.secrets_dir = secrets_dir;
-        }
 
         if let Some(input_graffiti) = cli_args.value_of("graffiti") {
             let graffiti_bytes = input_graffiti.as_bytes();

validator_client/src/lib.rs

@@ -68,18 +68,18 @@ impl<T: EthSpec> ProductionValidatorClient<T> {
             log,
             "Starting validator client";
             "beacon_node" => &config.http_server,
-            "datadir" => format!("{:?}", config.data_dir),
+            "validator_dir" => format!("{:?}", config.validator_dir),
         );
 
-        let mut validator_defs = ValidatorDefinitions::open_or_create(&config.data_dir)
+        let mut validator_defs = ValidatorDefinitions::open_or_create(&config.validator_dir)
             .map_err(|e| format!("Unable to open or create validator definitions: {:?}", e))?;
 
         if !config.disable_auto_discover {
             let new_validators = validator_defs
-                .discover_local_keystores(&config.data_dir, &config.secrets_dir, &log)
+                .discover_local_keystores(&config.validator_dir, &config.secrets_dir, &log)
                 .map_err(|e| format!("Unable to discover local validator keystores: {:?}", e))?;
             validator_defs
-                .save(&config.data_dir)
+                .save(&config.validator_dir)
                 .map_err(|e| format!("Unable to update validator definitions: {:?}", e))?;
             info!(
                 log,
@@ -90,7 +90,7 @@ impl<T: EthSpec> ProductionValidatorClient<T> {
 
         let validators = InitializedValidators::from_definitions(
             validator_defs,
-            config.data_dir.clone(),
+            config.validator_dir.clone(),
             config.delete_lockfiles,
             log.clone(),
         )

validator_client/src/validator_store.rs

@@ -62,14 +62,24 @@ impl<T: SlotClock + 'static, E: EthSpec> ValidatorStore<T, E> {
         fork_service: ForkService<T, E>,
         log: Logger,
     ) -> Result<Self, String> {
-        let slashing_db_path = config.data_dir.join(SLASHING_PROTECTION_FILENAME);
+        let slashing_db_path = config.validator_dir.join(SLASHING_PROTECTION_FILENAME);
-        let slashing_protection =
+        let slashing_protection = if config.strict_slashing_protection {
+            // Don't create a new slashing database if `strict_slashing_protection` is turned on.
+            SlashingDatabase::open(&slashing_db_path).map_err(|e| {
+                format!(
+                    "Failed to open slashing protection database: {:?}.
+                    Ensure that `slashing_protection.sqlite` is in {:?} folder",
+                    e, config.validator_dir
+                )
+            })?
+        } else {
             SlashingDatabase::open_or_create(&slashing_db_path).map_err(|e| {
                 format!(
                     "Failed to open or create slashing protection database: {:?}",
                     e
                 )
-            })?;
+            })?
+        };
 
         Ok(Self {
             validators: Arc::new(RwLock::new(validators)),