Add validator-manager (#3502)

## Issue Addressed

Addresses #2557

## Proposed Changes

Adds the `lighthouse validator-manager` command, which provides:

- `lighthouse validator-manager create`
    - Creates a `validators.json` file and a `deposits.json` (same format as https://github.com/ethereum/staking-deposit-cli)
- `lighthouse validator-manager import`
    - Imports validators from a `validators.json` file to the VC via the HTTP API.
- `lighthouse validator-manager move`
    - Moves validators from one VC to the other, utilizing only the VC API.

## Additional Info

In 98bcb947c I've reduced some VC `ERRO` and `CRIT` warnings to `WARN` or `DEBG` for the case where a pubkey is missing from the validator store. These were being triggered when we removed a validator but still had it in caches. It seems to me that `UnknownPubkey` will only happen in the case where we've removed a validator, so downgrading the logs is prudent. All the logs are `DEBG` apart from attestations and blocks which are `WARN`. I thought having *some* logging about this condition might help us down the track.

In 856cd7e37d I've made the VC delete the corresponding password file when it's deleting a keystore. This seemed like nice hygiene. Notably, it'll only delete that password file after it scans the validator definitions and finds that no other validator is also using that password file.

Cargo.lock

@@ -30,6 +30,8 @@ dependencies = [
  "filesystem",
  "safe_arith",
  "sensitive_url",
+ "serde",
+ "serde_json",
  "slashing_protection",
  "slot_clock",
  "tempfile",
@@ -4327,6 +4329,7 @@ dependencies = [
  "env_logger 0.9.3",
  "environment",
  "eth1",
+ "eth2",
  "eth2_network_config",
  "ethereum_hashing",
  "futures",
@@ -4349,6 +4352,7 @@ dependencies = [
  "unused_port",
  "validator_client",
  "validator_dir",
+ "validator_manager",
 ]
 
 [[package]]
@@ -8613,6 +8617,7 @@ dependencies = [
  "bls",
  "deposit_contract",
  "derivative",
+ "directory",
  "eth2_keystore",
  "filesystem",
  "hex",
@@ -8623,6 +8628,31 @@ dependencies = [
  "types",
 ]
 
+[[package]]
+name = "validator_manager"
+version = "0.1.0"
+dependencies = [
+ "account_utils",
+ "bls",
+ "clap",
+ "clap_utils",
+ "environment",
+ "eth2",
+ "eth2_keystore",
+ "eth2_network_config",
+ "eth2_wallet",
+ "ethereum_serde_utils",
+ "hex",
+ "regex",
+ "serde",
+ "serde_json",
+ "tempfile",
+ "tokio",
+ "tree_hash",
+ "types",
+ "validator_client",
+]
+
 [[package]]
 name = "valuable"
 version = "0.1.0"

Cargo.toml

@@ -83,6 +83,8 @@ members = [
     "validator_client",
     "validator_client/slashing_protection",
 
+    "validator_manager",
+
     "watch",
 ]
 resolver = "2"

account_manager/Cargo.toml

@@ -24,6 +24,8 @@ safe_arith = {path = "../consensus/safe_arith"}
 slot_clock = { path = "../common/slot_clock" }
 filesystem = { path = "../common/filesystem" }
 sensitive_url = { path = "../common/sensitive_url" }
+serde = { version = "1.0.116", features = ["derive"] }
+serde_json = "1.0.58"
 
 [dev-dependencies]
 tempfile = "3.1.0"

account_manager/src/common.rs

@@ -1,55 +1,7 @@
-use account_utils::PlainText;
+use account_utils::read_input_from_user;
-use account_utils::{read_input_from_user, strip_off_newlines};
-use eth2_wallet::bip39::{Language, Mnemonic};
-use std::fs;
-use std::path::PathBuf;
-use std::str::from_utf8;
-use std::thread::sleep;
-use std::time::Duration;
 
-pub const MNEMONIC_PROMPT: &str = "Enter the mnemonic phrase:";
 pub const WALLET_NAME_PROMPT: &str = "Enter wallet name:";
 
-pub fn read_mnemonic_from_cli(
-    mnemonic_path: Option<PathBuf>,
-    stdin_inputs: bool,
-) -> Result<Mnemonic, String> {
-    let mnemonic = match mnemonic_path {
-        Some(path) => fs::read(&path)
-            .map_err(|e| format!("Unable to read {:?}: {:?}", path, e))
-            .and_then(|bytes| {
-                let bytes_no_newlines: PlainText = strip_off_newlines(bytes).into();
-                let phrase = from_utf8(bytes_no_newlines.as_ref())
-                    .map_err(|e| format!("Unable to derive mnemonic: {:?}", e))?;
-                Mnemonic::from_phrase(phrase, Language::English).map_err(|e| {
-                    format!(
-                        "Unable to derive mnemonic from string {:?}: {:?}",
-                        phrase, e
-                    )
-                })
-            })?,
-        None => loop {
-            eprintln!();
-            eprintln!("{}", MNEMONIC_PROMPT);
-
-            let mnemonic = read_input_from_user(stdin_inputs)?;
-
-            match Mnemonic::from_phrase(mnemonic.as_str(), Language::English) {
-                Ok(mnemonic_m) => {
-                    eprintln!("Valid mnemonic provided.");
-                    eprintln!();
-                    sleep(Duration::from_secs(1));
-                    break mnemonic_m;
-                }
-                Err(_) => {
-                    eprintln!("Invalid mnemonic");
-                }
-            }
-        },
-    };
-    Ok(mnemonic)
-}
-
 /// Reads in a wallet name from the user. If the `--wallet-name` flag is provided, use it. Otherwise
 /// read from an interactive prompt using tty unless the `--stdin-inputs` flag is provided.
 pub fn read_wallet_name_from_cli(

account_manager/src/validator/import.rs

@@ -4,8 +4,8 @@ use account_utils::{
     eth2_keystore::Keystore,
     read_password_from_user,
     validator_definitions::{
-        recursively_find_voting_keystores, ValidatorDefinition, ValidatorDefinitions,
+        recursively_find_voting_keystores, PasswordStorage, ValidatorDefinition,
-        CONFIG_FILENAME,
+        ValidatorDefinitions, CONFIG_FILENAME,
     },
     ZeroizeString,
 };
@@ -277,7 +277,9 @@ pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), Strin
         let suggested_fee_recipient = None;
         let validator_def = ValidatorDefinition::new_keystore_with_password(
             &dest_keystore,
-            password_opt,
+            password_opt
+                .map(PasswordStorage::ValidatorDefinitions)
+                .unwrap_or(PasswordStorage::None),
             graffiti,
             suggested_fee_recipient,
             None,

account_manager/src/validator/recover.rs

@@ -1,10 +1,9 @@
 use super::create::STORE_WITHDRAW_FLAG;
-use crate::common::read_mnemonic_from_cli;
 use crate::validator::create::COUNT_FLAG;
 use crate::wallet::create::STDIN_INPUTS_FLAG;
 use crate::SECRETS_DIR_FLAG;
 use account_utils::eth2_keystore::{keypair_from_secret, Keystore, KeystoreBuilder};
-use account_utils::random_password;
+use account_utils::{random_password, read_mnemonic_from_cli};
 use clap::{App, Arg, ArgMatches};
 use directory::ensure_dir_exists;
 use directory::{parse_path_or_default_with_flag, DEFAULT_SECRET_DIR};

account_manager/src/wallet/recover.rs

@@ -1,6 +1,6 @@
-use crate::common::read_mnemonic_from_cli;
 use crate::wallet::create::{create_wallet_from_mnemonic, STDIN_INPUTS_FLAG};
 use crate::wallet::create::{HD_TYPE, NAME_FLAG, PASSWORD_FLAG, TYPE_FLAG};
+use account_utils::read_mnemonic_from_cli;
 use clap::{App, Arg, ArgMatches};
 use std::path::PathBuf;
 

book/src/SUMMARY.md

@@ -12,6 +12,9 @@
 * [Run a Node](./run_a_node.md)
 * [Become a Validator](./mainnet-validator.md)
 * [Validator Management](./validator-management.md)
+    * [The `validator-manager` Command](./validator-manager.md)
+        * [Creating validators](./validator-manager-create.md)
+        * [Moving validators](./validator-manager-move.md)
     * [Slashing Protection](./slashing-protection.md)
     * [Voluntary Exits](./voluntary-exit.md)
     * [Partial Withdrawals](./partial-withdrawal.md)
@@ -41,7 +44,7 @@
     * [Remote Signing with Web3Signer](./validator-web3signer.md)
     * [Database Configuration](./advanced_database.md)
     * [Database Migrations](./database-migrations.md)
-    * [Key Management](./key-management.md)
+    * [Key Management (Deprecated)](./key-management.md)
     * [Key Recovery](./key-recovery.md)
     * [Advanced Networking](./advanced_networking.md)
     * [Running a Slasher](./slasher.md)

book/src/key-management.md

@@ -1,9 +1,30 @@
-# Key Management
+# Key Management (Deprecated)
 
 [launchpad]: https://launchpad.ethereum.org/
 
->
+**⚠️ The information on this page refers to tooling and process that have been deprecated. Please read the "Deprecation Notice". ⚠️**
-> **Note: While Lighthouse is able to generate the validator keys and the deposit data file to submit to the deposit contract, we strongly recommend using the [staking-deposit-cli](https://github.com/ethereum/staking-deposit-cli) to create validators keys and the deposit data file. This is because the [staking-deposit-cli](https://github.com/ethereum/staking-deposit-cli) has the option to assign a withdrawal address during the key generation process, while Lighthouse wallet will always generate keys with withdrawal credentials of type 0x00. This means that users who created keys using Lighthouse will have to update their withdrawal credentials in the future to enable withdrawals. In addition, Lighthouse generates the deposit data file in the form of `*.rlp`, which cannot be uploaded to the [Staking launchpad][launchpad] that accepts only `*.json` file. This means that users have to directly interact with the deposit contract to be able to submit the deposit if they were to generate the files using Lighthouse.**
+
+## Deprecation Notice
+
+This page recommends the use of the `lighthouse account-manager` tool to create
+validators. This tool will always generate keys with the withdrawal credentials
+of type `0x00`. This means the users who created keys using `lighthouse
+account-manager` will have to update their withdrawal credentials in a
+separate step to receive staking rewards.
+
+In addition, Lighthouse generates the deposit data file in the form of `*.rlp`,
+which cannot be uploaded to the [Staking launchpad][launchpad] that accepts only
+`*.json` file. This means that users have to directly interact with the deposit
+contract to be able to submit the deposit if they were to generate the files
+using Lighthouse.
+
+Rather than continuing to read this page, we recommend users visit either:
+
+- The [Staking Launchpad][launchpad] for detailed, beginner-friendly instructions.
+- The [staking-deposit-cli](https://github.com/ethereum/staking-deposit-cli) for a CLI tool used by the [Staking Launchpad][launchpad].
+- The [validator-manager documentation](./validator-manager.md) for a Lighthouse-specific tool for streamlined validator management tools.
+
+## The `lighthouse account-manager`
 
 Lighthouse uses a _hierarchical_ key management system for producing validator
 keys. It is hierarchical because each validator key can be _derived_ from a

book/src/validator-management.md

@@ -13,6 +13,10 @@ standard directories and do not start their `lighthouse vc` with the
 this document. However, users with more complex needs may find this document
 useful.
 
+The [lighthouse validator-manager](./validator-manager.md) command can be used
+to create and import validators to a Lighthouse VC. It can also be used to move
+validators between two Lighthouse VCs.
+
 ## Introducing the `validator_definitions.yml` file
 
 The `validator_definitions.yml` file is located in the `validator-dir`, which

book/src/validator-manager-create.md

@@ -0,0 +1,206 @@
+# Creating and Importing Validators
+
+[Ethereum Staking Launchpad]: https://launchpad.ethereum.org/en/
+
+The `lighthouse validator-manager create` command derives validators from a
+mnemonic and produces two files:
+
+- `validators.json`: the keystores and passwords for the newly generated
+    validators, in JSON format.
+- `deposits.json`: a JSON file of the same format as
+    [staking-deposit-cli](https://github.com/ethereum/staking-deposit-cli) which can
+    be used for deposit submission via the [Ethereum Staking
+    Launchpad][].
+
+The `lighthouse validator-manager import` command accepts a `validators.json`
+file (from the `create` command) and submits those validators to a running
+Lighthouse Validator Client via the HTTP API.
+
+These two commands enable a workflow of:
+
+1. Creating the validators via the `create` command.
+1. Importing the validators via the `import` command.
+1. Depositing validators via the [Ethereum Staking
+    Launchpad][].
+
+The separation of the `create` and `import` commands allows for running the
+`create` command on an air-gapped host whilst performing the `import` command on
+an internet-connected host.
+
+The `create` and `import` commands are recommended for advanced users who are
+familiar with command line tools and the practicalities of managing sensitive
+cryptographic material. **We recommend that novice users follow the workflow on
+[Ethereum Staking Launchpad][] rather than using the `create` and `import`
+commands.**
+
+## Simple Example
+
+Create validators from a mnemonic with:
+
+```bash
+lighthouse \
+    validator-manager \
+    create \
+    --network mainnet \
+    --first-index 0 \
+    --count 2 \
+    --eth1-withdrawal-address <ADDRESS> \
+    --suggested-fee-recipient <ADDRESS> \
+    --output-path ./
+```
+> If the flag `--first-index` is not provided, it will default to using index 0.
+> The `--suggested-fee-recipient` flag may be omitted to use whatever default
+> value the VC uses. It does not necessarily need to be identical to
+> `--eth1-withdrawal-address`.
+> The command will create the `deposits.json` and `validators.json` in the present working directory. If you would like these files to be created in a different directory, change the value of `output-path`, for example `--output-path /desired/directory`. The directory will be created if the path does not exist.
+
+Then, import the validators to a running VC with:
+
+```bash
+lighthouse \
+    validator-manager \
+    import \
+    --validators-file validators.json \
+    --vc-token <API-TOKEN-PATH>
+```
+> This is assuming that `validators.json` is in the present working directory. If it is not, insert the directory of the file.
+> Be sure to remove `./validators.json` after the import is successful since it
+> contains unencrypted validator keystores.
+
+## Detailed Guide
+
+This guide will create two validators and import them to a VC. For simplicity,
+the same host will be used to generate the keys and run the VC. In reality,
+users may want to perform the `create` command on an air-gapped machine and then
+move the `validators.json` and `deposits.json` files to an Internet-connected
+host. This would help protect the mnemonic from being exposed to the Internet.
+
+### 1. Create the Validators
+
+Run the `create` command, substituting `<ADDRESS>` for an execution address that
+you control. This is where all the staked ETH and rewards will ultimately
+reside, so it's very important that this address is secure, accessible and
+backed-up. The `create` command:
+
+```bash
+lighthouse \
+    validator-manager \
+    create \
+    --first-index 0 \
+    --count 2 \
+    --eth1-withdrawal-address <ADDRESS> \
+    --output-path ./
+```
+
+If successful, the command output will appear like below:
+
+```bash
+Running validator manager for mainnet network
+
+Enter the mnemonic phrase:
+<REDACTED>
+Valid mnemonic provided.
+
+Starting derivation of 2 keystores. Each keystore may take several seconds.
+Completed 1/2: 0x8885c29b8f88ee9b9a37b480fd4384fed74bda33d85bc8171a904847e65688b6c9bb4362d6597fd30109fb2def6c3ae4
+Completed 2/2: 0xa262dae3dcd2b2e280af534effa16bedb27c06f2959e114d53bd2a248ca324a018dc73179899a066149471a94a1bc92f
+Keystore generation complete
+Writing "./validators.json"
+Writing "./deposits.json"
+```
+
+This command will create validators at indices `0, 1`. The exact indices created
+can be influenced with the `--first-index` and `--count` flags. Use these flags
+with caution to prevent creating the same validator twice, this may result in a
+slashing!
+
+The command will create two files:
+
+- `./deposits.json`: this file does *not* contain sensitive information and may be uploaded to the [Ethereum Staking Launchpad].
+- `./validators.json`: this file contains **sensitive unencrypted validator keys, do not share it with anyone or upload it to any website**.
+
+### 2. Import the validators
+
+The VC which will receive the validators needs to have the following flags at a minimum:
+
+- `--http`
+- `--http-port 5062`
+- `--enable-doppelganger-protection`
+
+Therefore, the VC command might look like:
+
+```bash
+lighthouse \
+    vc \
+    --http \
+    --http-port 5062 \
+    --enable-doppelganger-protection
+```
+
+In order to import the validators, the location of the VC `api-token.txt` file
+must be known. The location of the file varies, but it is located in the
+"validator directory" of your data directory. For example:
+`~/.lighthouse/mainnet/validators/api-token.txt`. We will use `<API-TOKEN-PATH>`
+to subsitute this value. If you are unsure of the `api-token.txt` path, you can run `curl http://localhost:5062/lighthouse/auth` which will show the path.
+
+
+Once the VC is running, use the `import` command to import the validators to the VC:
+
+```bash
+lighthouse \
+    validator-manager \
+    import \
+    --validators-file validators.json \
+    --vc-token <API-TOKEN-PATH>
+```
+
+If successful, the command output will appear like below:
+
+```bash
+Running validator manager for mainnet network
+Validator client is reachable at http://localhost:5062/ and reports 0 validators
+Starting to submit 2 validators to VC, each validator may take several seconds
+Uploaded keystore 1 of 2 to the VC
+Uploaded keystore 2 of 2 to the VC
+```
+
+The user should now *securely* delete the `validators.json` file (e.g., `shred -u validators.json`).
+The `validators.json` contains the unencrypted validator keys and must not be
+shared with anyone.
+At the same time, `lighthouse vc` will log:
+```bash
+INFO Importing keystores via standard HTTP API, count: 1
+WARN No slashing protection data provided with keystores
+INFO Enabled validator                       voting_pubkey: 0xab6e29f1b98fedfca878edce2b471f1b5ee58ee4c3bd216201f98254ef6f6eac40a53d74c8b7da54f51d3e85cacae92f, signing_method: local_keystore
+INFO Modified key_cache saved successfully
+```
+The WARN message means that the `validators.json` file does not contain the slashing protection data. This is normal if you are starting a new validator. The flag `--enable-doppelganger-protection` will also protect users from potential slashing risk. 
+The validators will now go through 2-3 epochs of [doppelganger
+protection](./validator-doppelganger.md) and will automatically start performing
+their duties when they are deposited and activated. 
+
+If the host VC contains the same public key as the `validators.json` file, an error will be shown and the `import` process will stop:
+
+```bash
+Duplicate validator   0xab6e29f1b98fedfca878edce2b471f1b5ee58ee4c3bd216201f98254ef6f6eac40a53d74c8b7da54f51d3e85cacae92f already exists on the destination validator client. This may indicate that some validators are running in two places at once, which can lead to slashing. If you are certain that there is no risk, add the --ignore-duplicates flag.
+Err(DuplicateValidator(0xab6e29f1b98fedfca878edce2b471f1b5ee58ee4c3bd216201f98254ef6f6eac40a53d74c8b7da54f51d3e85cacae92f))
+```
+
+If you are certain that it is safe, you can add the flag `--ignore-duplicates` in the `import` command. The command becomes:
+
+```bash
+lighthouse \
+    validator-manager \
+    import \
+    --validators-file validators.json \
+    --vc-token <API-TOKEN-PATH> \
+    --ignore-duplicates
+```
+and the output will be as follows:
+
+```bash
+Duplicate validators are ignored, ignoring 0xab6e29f1b98fedfca878edce2b471f1b5ee58ee4c3bd216201f98254ef6f6eac40a53d74c8b7da54f51d3e85cacae92f which exists on the destination validator client
+Re-uploaded keystore 1 of 6 to the VC
+```
+
+The guide is complete.

book/src/validator-manager-move.md

@@ -0,0 +1,188 @@
+# Moving Validators
+
+The `lighthouse validator-manager move` command uses the VC HTTP API to move
+validators from one VC (the "src" VC) to another VC (the "dest" VC). The move
+operation is *comprehensive*; it will:
+
+- Disable the validators on the src VC.
+- Remove the validator keystores from the src VC file system.
+- Export the slashing database records for the appropriate validators from the src VC to the dest VC.
+- Enable the validators on the dest VC.
+- Generally result in very little or no validator downtime.
+
+It is capable of moving all validators on the src VC, a count of validators or
+a list of pubkeys.
+
+The `move` command is only guaranteed to work between two Lighthouse VCs (i.e.,
+there is no guarantee that the commands will work between Lighthouse and Teku, for instance).
+
+The `move` command only supports moving validators using a keystore on the local
+file system, it does not support `Web3Signer` validators.
+
+Although all efforts are taken to avoid it, it's possible for the `move` command
+to fail in a way that removes the validator from the src VC without adding it to the
+dest VC. Therefore, it is recommended to **never use the `move` command without
+having a backup of all validator keystores (e.g. the mnemonic).**
+
+## Simple Example
+
+The following command will move all validators from the VC running at
+`http://localhost:6062` to the VC running at `http://localhost:5062`.
+
+```bash
+lighthouse \
+	validator-manager \
+	move \
+	--src-vc-url http://localhost:6062 \
+	--src-vc-token ~/src-token.txt \
+	--dest-vc-url http://localhost:5062 \
+	--dest-vc-token ~/.lighthouse/mainnet/validators/api-token.txt \
+	--validators all
+```
+
+## Detailed Guide
+
+This guide describes the steps to move validators between two validator clients (VCs) which are
+able to SSH between each other. This guide assumes experience with the Linux command line and SSH
+connections.
+
+There will be two VCs in this example:
+
+- The *source* VC which contains the validators/keystores to be moved.
+- The *destination* VC which is to take the validators/keystores from the source.
+
+There will be two hosts in this example:
+
+- Host 1 (*"source host"*): Is running the `src-vc`.
+- Host 2 (*"destination host"*): Is running the `dest-vc`.
+
+The example assumes
+that Host 1 is able to SSH to Host 2.
+
+In reality, many host configurations are possible. For example:
+
+- Both VCs on the same host.
+- Both VCs on different hosts and the `validator-manager` being used on a third host.
+
+### 1. Configure the Source VC
+
+The source VC needs to have the following flags at a minimum:
+
+- `--http`
+- `--http-port 5062`
+- `--http-allow-keystore-export`
+
+Therefore, the source VC command might look like:
+
+```bash
+lighthouse \
+    vc \
+    --http \
+    --http-port 5062 \
+    --http-allow-keystore-export
+```
+
+### 2. Configure the Destination VC
+
+The destination VC needs to have the following flags at a minimum:
+
+- `--http`
+- `--http-port 5062`
+- `--enable-doppelganger-protection`
+
+Therefore, the destination VC command might look like:
+
+```bash
+lighthouse \
+    vc \
+    --http \
+    --http-port 5062 \
+    --enable-doppelganger-protection
+```
+
+> The `--enable-doppelganger-protection` flag is not *strictly* required, however
+> it is recommended for an additional layer of safety. It will result in 2-3
+> epochs of downtime for the validator after it is moved, which is generally an
+> inconsequential cost in lost rewards or penalties.
+> 
+> Optionally, users can add the `--http-store-passwords-in-secrets-dir` flag if they'd like to have
+> the import validator keystore passwords stored in separate files rather than in the
+> `validator-definitions.yml` file. If you don't know what this means, you can safely omit the flag.
+
+### 3. Obtain the Source API Token
+
+The VC API is protected by an *API token*. This is stored in a file on each of the hosts. Since
+we'll be running our command on the destination host, it will need to have the API token for the
+source host on its file-system.
+
+On the **source host**, find the location of the `api-token.txt` file and copy the contents. The
+location of the file varies, but it is located in the "validator directory" of your data directory,
+alongside validator keystores. For example: `~/.lighthouse/mainnet/validators/api-token.txt`. If you are unsure of the `api-token.txt` path, you can run `curl http://localhost:5062/lighthouse/auth` which will show the path.
+
+Copy the contents of that file into a new file on the **destination host** at `~/src-token.txt`. The
+API token should be similar to `api-token-0x03eace4c98e8f77477bb99efb74f9af10d800bd3318f92c33b719a4644254d4123`.
+
+### 4. Create an SSH Tunnel
+
+In the **source host**, open a terminal window, SSH to the **destination host** and establish a reverse-SSH connection
+between the **destination host** and the **source host**.
+
+```bash
+ssh dest-host
+ssh -L 6062:localhost:5062 src-host
+```
+
+It's important that you leave this session open throughout the rest of this tutorial. If you close
+this terminal window then the connection between the destination and source host will be lost.
+
+### 5. Move
+
+With the SSH tunnel established between the `dest-host` and `src-host`, from the **destination
+host** run the command to move the validators:
+
+```bash
+lighthouse \
+	validator-manager \
+	move \
+	--src-vc-url http://localhost:6062 \
+	--src-vc-token ~/src-token.txt \
+	--dest-vc-url http://localhost:5062 \
+	--dest-vc-token ~/.lighthouse/mainnet/validators/api-token.txt \
+	--validators all
+```
+
+The command will provide information about the progress of the operation and
+emit `Done.` when the operation has completed successfully. For example:
+
+```bash
+Running validator manager for mainnet network
+Validator client is reachable at http://localhost:5062/ and reports 2 validators
+Validator client is reachable at http://localhost:6062/ and reports 0 validators
+Moved keystore 1 of 2
+Moved keystore 2 of 2
+Done.
+```
+At the same time, `lighthouse vc` will log:
+```bash
+INFO Importing keystores via standard HTTP API, count: 1
+INFO Enabled validator                       voting_pubkey: 0xab6e29f1b98fedfca878edce2b471f1b5ee58ee4c3bd216201f98254ef6f6eac40a53d74c8b7da54f51d3e85cacae92f, signing_method: local_keystore
+INFO Modified key_cache saved successfully
+Once the operation completes successfully, there is nothing else to be done. The
+validators have been removed from the `src-host` and enabled at the `dest-host`.
+If the `--enable-doppelganger-protection` flag was used it may take 2-3 epochs
+for the validators to start attesting and producing blocks on the `dest-host`.
+If you would only like to move some validators, you can replace the flag `--validators all` with one or more validator public keys. For example:
+
+```bash
+lighthouse \
+	validator-manager \
+	move \
+	--src-vc-url http://localhost:6062 \
+	--src-vc-token ~/src-token.txt \
+	--dest-vc-url http://localhost:5062 \
+	--dest-vc-token ~/.lighthouse/mainnet/validators/api-token.txt \
+	--validators 0x9096aab771e44da149bd7c9926d6f7bb96ef465c0eeb4918be5178cd23a1deb4aec232c61d85ff329b54ed4a3bdfff3a,0x90fc4f72d898a8f01ab71242e36f4545aaf87e3887be81632bb8ba4b2ae8fb70753a62f866344d7905e9a07f5a9cdda1
+```
+Any errors encountered during the operation should include information on how to
+proceed. Assistance is also available on our
+[Discord](https://discord.gg/cyAszAh).

book/src/validator-manager.md

@@ -0,0 +1,35 @@
+# Validator Manager
+
+[Ethereum Staking Launchpad]: https://launchpad.ethereum.org/en/
+[Import Validators]: #import-validators
+
+## Introduction
+
+The `lighthouse validator-manager` tool provides utilities for managing validators on a *running*
+Lighthouse Validator Client. The validator manager performs operations via the HTTP API of the
+validator client (VC). Due to limitations of the
+[keymanager-APIs](https://ethereum.github.io/keymanager-APIs/), only Lighthouse VCs are fully
+supported by this command.
+
+The validator manager tool is similar to the `lighthouse account-manager` tool,
+except the latter creates files that will be read by the VC next time it starts
+whilst the former makes instant changes to a live VC.
+
+The `account-manager` is ideal for importing keys created with the
+[staking-deposit-cli](https://github.com/ethereum/staking-deposit-cli). On the
+other hand, the `validator-manager` is ideal for moving existing validators
+between two VCs or for advanced users to create validators at scale with less
+downtime.
+
+The `validator-manager` boasts the following features:
+
+- One-line command to arbitrarily move validators between two VCs, maintaining the slashing protection database.
+- Generates deposit files compatible with the [Ethereum Staking Launchpad][].
+- Generally involves zero or very little downtime.
+- The "key cache" is preserved whenever a validator is added with the validator
+    manager, preventing long waits at start up when a new validator is added.
+
+## Guides
+
+- [Creating and importing validators using the `create` and `import` commands.](./validator-manager-create.md)
+- [Moving validators between two VCs using the `move` command.](./validator-manager-move.md)

common/account_utils/src/lib.rs

@@ -13,6 +13,9 @@ use std::fs::{self, File};
 use std::io;
 use std::io::prelude::*;
 use std::path::{Path, PathBuf};
+use std::str::from_utf8;
+use std::thread::sleep;
+use std::time::Duration;
 use zeroize::Zeroize;
 
 pub mod validator_definitions;
@@ -30,6 +33,8 @@ pub const MINIMUM_PASSWORD_LEN: usize = 12;
 /// array of length 32.
 const DEFAULT_PASSWORD_LEN: usize = 48;
 
+pub const MNEMONIC_PROMPT: &str = "Enter the mnemonic phrase:";
+
 /// Returns the "default" path where a wallet should store its password file.
 pub fn default_wallet_password_path<P: AsRef<Path>>(wallet_name: &str, secrets_dir: P) -> PathBuf {
     secrets_dir.as_ref().join(format!("{}.pass", wallet_name))
@@ -59,6 +64,18 @@ pub fn read_password<P: AsRef<Path>>(path: P) -> Result<PlainText, io::Error> {
     fs::read(path).map(strip_off_newlines).map(Into::into)
 }
 
+/// Reads a password file into a `ZeroizeString` struct, with new-lines removed.
+pub fn read_password_string<P: AsRef<Path>>(path: P) -> Result<ZeroizeString, String> {
+    fs::read(path)
+        .map_err(|e| format!("Error opening file: {:?}", e))
+        .map(strip_off_newlines)
+        .and_then(|bytes| {
+            String::from_utf8(bytes)
+                .map_err(|e| format!("Error decoding utf8: {:?}", e))
+                .map(Into::into)
+        })
+}
+
 /// Write a file atomically by using a temporary file as an intermediate.
 ///
 /// Care is taken to preserve the permissions of the file at `file_path` being written.
@@ -220,6 +237,46 @@ impl AsRef<[u8]> for ZeroizeString {
     }
 }
 
+pub fn read_mnemonic_from_cli(
+    mnemonic_path: Option<PathBuf>,
+    stdin_inputs: bool,
+) -> Result<Mnemonic, String> {
+    let mnemonic = match mnemonic_path {
+        Some(path) => fs::read(&path)
+            .map_err(|e| format!("Unable to read {:?}: {:?}", path, e))
+            .and_then(|bytes| {
+                let bytes_no_newlines: PlainText = strip_off_newlines(bytes).into();
+                let phrase = from_utf8(bytes_no_newlines.as_ref())
+                    .map_err(|e| format!("Unable to derive mnemonic: {:?}", e))?;
+                Mnemonic::from_phrase(phrase, Language::English).map_err(|e| {
+                    format!(
+                        "Unable to derive mnemonic from string {:?}: {:?}",
+                        phrase, e
+                    )
+                })
+            })?,
+        None => loop {
+            eprintln!();
+            eprintln!("{}", MNEMONIC_PROMPT);
+
+            let mnemonic = read_input_from_user(stdin_inputs)?;
+
+            match Mnemonic::from_phrase(mnemonic.as_str(), Language::English) {
+                Ok(mnemonic_m) => {
+                    eprintln!("Valid mnemonic provided.");
+                    eprintln!();
+                    sleep(Duration::from_secs(1));
+                    break mnemonic_m;
+                }
+                Err(_) => {
+                    eprintln!("Invalid mnemonic");
+                }
+            }
+        },
+    };
+    Ok(mnemonic)
+}
+
 #[cfg(test)]
 mod test {
     use super::*;

common/account_utils/src/validator_definitions.rs

@@ -3,7 +3,9 @@
 //! Serves as the source-of-truth of which validators this validator client should attempt (or not
 //! attempt) to load into the `crate::intialized_validators::InitializedValidators` struct.
 
-use crate::{default_keystore_password_path, write_file_via_temporary, ZeroizeString};
+use crate::{
+    default_keystore_password_path, read_password_string, write_file_via_temporary, ZeroizeString,
+};
 use directory::ensure_dir_exists;
 use eth2_keystore::Keystore;
 use regex::Regex;
@@ -43,6 +45,18 @@ pub enum Error {
     UnableToOpenKeystore(eth2_keystore::Error),
     /// The validator directory could not be created.
     UnableToCreateValidatorDir(PathBuf),
+    UnableToReadKeystorePassword(String),
+    KeystoreWithoutPassword,
+}
+
+/// Defines how a password for a validator keystore will be persisted.
+pub enum PasswordStorage {
+    /// Store the password in the `validator_definitions.yml` file.
+    ValidatorDefinitions(ZeroizeString),
+    /// Store the password in a separate, dedicated file (likely in the "secrets" directory).
+    File(PathBuf),
+    /// Don't store the password at all.
+    None,
 }
 
 #[derive(Clone, PartialEq, Serialize, Deserialize, Hash, Eq)]
@@ -92,6 +106,34 @@ impl SigningDefinition {
     pub fn is_local_keystore(&self) -> bool {
         matches!(self, SigningDefinition::LocalKeystore { .. })
     }
+
+    pub fn voting_keystore_password(&self) -> Result<Option<ZeroizeString>, Error> {
+        match self {
+            SigningDefinition::LocalKeystore {
+                voting_keystore_password: Some(password),
+                ..
+            } => Ok(Some(password.clone())),
+            SigningDefinition::LocalKeystore {
+                voting_keystore_password_path: Some(path),
+                ..
+            } => read_password_string(path)
+                .map(Into::into)
+                .map(Option::Some)
+                .map_err(Error::UnableToReadKeystorePassword),
+            SigningDefinition::LocalKeystore { .. } => Err(Error::KeystoreWithoutPassword),
+            SigningDefinition::Web3Signer(_) => Ok(None),
+        }
+    }
+
+    pub fn voting_keystore_password_path(&self) -> Option<&PathBuf> {
+        match self {
+            SigningDefinition::LocalKeystore {
+                voting_keystore_password_path: Some(path),
+                ..
+            } => Some(path),
+            _ => None,
+        }
+    }
 }
 
 /// A validator that may be initialized by this validator client.
@@ -129,7 +171,7 @@ impl ValidatorDefinition {
     /// This function does not check the password against the keystore.
     pub fn new_keystore_with_password<P: AsRef<Path>>(
         voting_keystore_path: P,
-        voting_keystore_password: Option<ZeroizeString>,
+        voting_keystore_password_storage: PasswordStorage,
         graffiti: Option<GraffitiString>,
         suggested_fee_recipient: Option<Address>,
         gas_limit: Option<u64>,
@@ -139,6 +181,12 @@ impl ValidatorDefinition {
         let keystore =
             Keystore::from_json_file(&voting_keystore_path).map_err(Error::UnableToOpenKeystore)?;
         let voting_public_key = keystore.public_key().ok_or(Error::InvalidKeystorePubkey)?;
+        let (voting_keystore_password_path, voting_keystore_password) =
+            match voting_keystore_password_storage {
+                PasswordStorage::ValidatorDefinitions(password) => (None, Some(password)),
+                PasswordStorage::File(path) => (Some(path), None),
+                PasswordStorage::None => (None, None),
+            };
 
         Ok(ValidatorDefinition {
             enabled: true,
@@ -150,7 +198,7 @@ impl ValidatorDefinition {
             builder_proposals,
             signing_definition: SigningDefinition::LocalKeystore {
                 voting_keystore_path,
-                voting_keystore_password_path: None,
+                voting_keystore_password_path,
                 voting_keystore_password,
             },
         })
@@ -346,6 +394,13 @@ impl ValidatorDefinitions {
     pub fn as_mut_slice(&mut self) -> &mut [ValidatorDefinition] {
         self.0.as_mut_slice()
     }
+
+    // Returns an iterator over all the `voting_keystore_password_paths` in self.
+    pub fn iter_voting_keystore_password_paths(&self) -> impl Iterator<Item = &PathBuf> {
+        self.0
+            .iter()
+            .filter_map(|def| def.signing_definition.voting_keystore_password_path())
+    }
 }
 
 /// Perform an exhaustive tree search of `dir`, adding any discovered voting keystore paths to

common/eth2/src/lighthouse_vc/http_client.rs

@@ -490,6 +490,21 @@ impl ValidatorClientHttpClient {
         .await
     }
 
+    /// `DELETE eth/v1/keystores`
+    pub async fn delete_lighthouse_keystores(
+        &self,
+        req: &DeleteKeystoresRequest,
+    ) -> Result<ExportKeystoresResponse, Error> {
+        let mut path = self.server.full.clone();
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("lighthouse")
+            .push("keystores");
+
+        self.delete_with_unsigned_response(path, req).await
+    }
+
     fn make_keystores_url(&self) -> Result<Url, Error> {
         let mut url = self.server.full.clone();
         url.path_segments_mut()

common/eth2/src/lighthouse_vc/std_types.rs

@@ -1,9 +1,10 @@
 use account_utils::ZeroizeString;
 use eth2_keystore::Keystore;
 use serde::{Deserialize, Serialize};
-use slashing_protection::interchange::Interchange;
 use types::{Address, PublicKeyBytes};
 
+pub use slashing_protection::interchange::Interchange;
+
 #[derive(Debug, Deserialize, Serialize, PartialEq)]
 pub struct GetFeeRecipientResponse {
     pub pubkey: PublicKeyBytes,
@@ -27,7 +28,7 @@ pub struct ListKeystoresResponse {
     pub data: Vec<SingleKeystoreResponse>,
 }
 
-#[derive(Debug, Deserialize, Serialize, PartialEq)]
+#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Hash)]
 pub struct SingleKeystoreResponse {
     pub validating_pubkey: PublicKeyBytes,
     pub derivation_path: Option<String>,

common/eth2/src/lighthouse_vc/types.rs

@@ -152,3 +152,19 @@ pub struct UpdateGasLimitRequest {
 pub struct VoluntaryExitQuery {
     pub epoch: Option<Epoch>,
 }
+
+#[derive(Deserialize, Serialize)]
+pub struct ExportKeystoresResponse {
+    pub data: Vec<SingleExportKeystoresResponse>,
+    #[serde(with = "serde_utils::json_str")]
+    pub slashing_protection: Interchange,
+}
+
+#[derive(Deserialize, Serialize)]
+pub struct SingleExportKeystoresResponse {
+    pub status: Status<DeleteKeystoreStatus>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub validating_keystore: Option<KeystoreJsonStr>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub validating_keystore_password: Option<ZeroizeString>,
+}

common/validator_dir/Cargo.toml

@@ -20,6 +20,7 @@ tree_hash = "0.5.0"
 hex = "0.4.2"
 derivative = "2.1.1"
 lockfile = { path = "../lockfile" }
+directory = { path = "../directory" }
 
 [dev-dependencies]
 tempfile = "3.1.0"

common/validator_dir/src/builder.rs

@@ -1,6 +1,7 @@
 use crate::{Error as DirError, ValidatorDir};
 use bls::get_withdrawal_credentials;
 use deposit_contract::{encode_eth1_tx_data, Error as DepositError};
+use directory::ensure_dir_exists;
 use eth2_keystore::{Error as KeystoreError, Keystore, KeystoreBuilder, PlainText};
 use filesystem::create_with_600_perms;
 use rand::{distributions::Alphanumeric, Rng};
@@ -41,6 +42,7 @@ pub enum Error {
     #[cfg(feature = "insecure_keys")]
     InsecureKeysError(String),
     MissingPasswordDir,
+    UnableToCreatePasswordDir(String),
 }
 
 impl From<KeystoreError> for Error {
@@ -78,6 +80,13 @@ impl<'a> Builder<'a> {
         self
     }
 
+    /// Optionally supply a directory in which to store the passwords for the validator keystores.
+    /// If `None` is provided, do not store the password.
+    pub fn password_dir_opt(mut self, password_dir_opt: Option<PathBuf>) -> Self {
+        self.password_dir = password_dir_opt;
+        self
+    }
+
     /// Build the `ValidatorDir` use the given `keystore` which can be unlocked with `password`.
     ///
     /// The builder will not necessarily check that `password` can unlock `keystore`.
@@ -153,6 +162,10 @@ impl<'a> Builder<'a> {
             create_dir_all(&dir).map_err(Error::UnableToCreateDir)?;
         }
 
+        if let Some(password_dir) = &self.password_dir {
+            ensure_dir_exists(password_dir).map_err(Error::UnableToCreatePasswordDir)?;
+        }
+
         // The withdrawal keystore must be initialized in order to store it or create an eth1
         // deposit.
         if (self.store_withdrawal_keystore || self.deposit_info.is_some())
@@ -234,7 +247,7 @@ impl<'a> Builder<'a> {
                 if self.store_withdrawal_keystore {
                     // Write the withdrawal password to file.
                     write_password_to_file(
-                        password_dir.join(withdrawal_keypair.pk.as_hex_string()),
+                        keystore_password_path(password_dir, &withdrawal_keystore),
                         withdrawal_password.as_bytes(),
                     )?;
 
@@ -250,7 +263,7 @@ impl<'a> Builder<'a> {
         if let Some(password_dir) = self.password_dir.as_ref() {
             // Write the voting password to file.
             write_password_to_file(
-                password_dir.join(format!("0x{}", voting_keystore.pubkey())),
+                keystore_password_path(password_dir, &voting_keystore),
                 voting_password.as_bytes(),
             )?;
         }
@@ -262,6 +275,12 @@ impl<'a> Builder<'a> {
     }
 }
 
+pub fn keystore_password_path<P: AsRef<Path>>(password_dir: P, keystore: &Keystore) -> PathBuf {
+    password_dir
+        .as_ref()
+        .join(format!("0x{}", keystore.pubkey()))
+}
+
 /// Writes a JSON keystore to file.
 fn write_keystore_to_file(path: PathBuf, keystore: &Keystore) -> Result<(), Error> {
     if path.exists() {

common/validator_dir/src/lib.rs

@@ -15,6 +15,6 @@ pub use crate::validator_dir::{
     ETH1_DEPOSIT_TX_HASH_FILE,
 };
 pub use builder::{
-    Builder, Error as BuilderError, ETH1_DEPOSIT_DATA_FILE, VOTING_KEYSTORE_FILE,
+    keystore_password_path, Builder, Error as BuilderError, ETH1_DEPOSIT_DATA_FILE,
-    WITHDRAWAL_KEYSTORE_FILE,
+    VOTING_KEYSTORE_FILE, WITHDRAWAL_KEYSTORE_FILE,
 };

common/validator_dir/src/validator_dir.rs

@@ -1,5 +1,5 @@
 use crate::builder::{
-    ETH1_DEPOSIT_AMOUNT_FILE, ETH1_DEPOSIT_DATA_FILE, VOTING_KEYSTORE_FILE,
+    keystore_password_path, ETH1_DEPOSIT_AMOUNT_FILE, ETH1_DEPOSIT_DATA_FILE, VOTING_KEYSTORE_FILE,
     WITHDRAWAL_KEYSTORE_FILE,
 };
 use deposit_contract::decode_eth1_tx_data;
@@ -219,9 +219,7 @@ pub fn unlock_keypair<P: AsRef<Path>>(
     )
     .map_err(Error::UnableToReadKeystore)?;
 
-    let password_path = password_dir
+    let password_path = keystore_password_path(password_dir, &keystore);
-        .as_ref()
-        .join(format!("0x{}", keystore.pubkey()));
     let password: PlainText = read(&password_path)
         .map_err(|_| Error::UnableToReadPassword(password_path))?
         .into();

consensus/types/src/lib.rs

@@ -71,6 +71,7 @@ pub mod sync_duty;
 pub mod validator;
 pub mod validator_subscription;
 pub mod voluntary_exit;
+pub mod withdrawal_credentials;
 #[macro_use]
 pub mod slot_epoch_macros;
 pub mod config_and_preset;
@@ -189,6 +190,7 @@ pub use crate::validator_registration_data::*;
 pub use crate::validator_subscription::ValidatorSubscription;
 pub use crate::voluntary_exit::VoluntaryExit;
 pub use crate::withdrawal::Withdrawal;
+pub use crate::withdrawal_credentials::WithdrawalCredentials;
 
 pub type CommitteeIndex = u64;
 pub type Hash256 = H256;

consensus/types/src/withdrawal_credentials.rs

@@ -0,0 +1,57 @@
+use crate::*;
+use bls::get_withdrawal_credentials;
+
+pub struct WithdrawalCredentials(Hash256);
+
+impl WithdrawalCredentials {
+    pub fn bls(withdrawal_public_key: &PublicKey, spec: &ChainSpec) -> Self {
+        let withdrawal_credentials =
+            get_withdrawal_credentials(withdrawal_public_key, spec.bls_withdrawal_prefix_byte);
+        Self(Hash256::from_slice(&withdrawal_credentials))
+    }
+
+    pub fn eth1(withdrawal_address: Address, spec: &ChainSpec) -> Self {
+        let mut withdrawal_credentials = [0; 32];
+        withdrawal_credentials[0] = spec.eth1_address_withdrawal_prefix_byte;
+        withdrawal_credentials[12..].copy_from_slice(withdrawal_address.as_bytes());
+        Self(Hash256::from_slice(&withdrawal_credentials))
+    }
+}
+
+impl From<WithdrawalCredentials> for Hash256 {
+    fn from(withdrawal_credentials: WithdrawalCredentials) -> Self {
+        withdrawal_credentials.0
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::test_utils::generate_deterministic_keypair;
+    use std::str::FromStr;
+
+    #[test]
+    fn bls_withdrawal_credentials() {
+        let spec = &MainnetEthSpec::default_spec();
+        let keypair = generate_deterministic_keypair(0);
+        let credentials = WithdrawalCredentials::bls(&keypair.pk, spec);
+        let manually_generated_credentials =
+            get_withdrawal_credentials(&keypair.pk, spec.bls_withdrawal_prefix_byte);
+        let hash: Hash256 = credentials.into();
+        assert_eq!(hash[0], spec.bls_withdrawal_prefix_byte);
+        assert_eq!(hash.as_bytes(), &manually_generated_credentials);
+    }
+
+    #[test]
+    fn eth1_withdrawal_credentials() {
+        let spec = &MainnetEthSpec::default_spec();
+        let address = Address::from_str("0x25c4a76E7d118705e7Ea2e9b7d8C59930d8aCD3b").unwrap();
+        let credentials = WithdrawalCredentials::eth1(address, spec);
+        let hash: Hash256 = credentials.into();
+        assert_eq!(
+            hash,
+            Hash256::from_str("0x01000000000000000000000025c4a76E7d118705e7Ea2e9b7d8C59930d8aCD3b")
+                .unwrap()
+        )
+    }
+}

lighthouse/Cargo.toml

@@ -56,6 +56,7 @@ directory = { path = "../common/directory" }
 unused_port = { path = "../common/unused_port" }
 database_manager = { path = "../database_manager" }
 slasher = { path = "../slasher" }
+validator_manager = { path = "../validator_manager" }
 
 [dev-dependencies]
 tempfile = "3.1.0"
@@ -64,6 +65,7 @@ slashing_protection = { path = "../validator_client/slashing_protection" }
 lighthouse_network = { path = "../beacon_node/lighthouse_network" }
 sensitive_url = { path = "../common/sensitive_url" }
 eth1 = { path = "../beacon_node/eth1" }
+eth2 = { path = "../common/eth2" }
 
 [[test]]
 name = "lighthouse_tests"

lighthouse/src/main.rs

@@ -329,6 +329,7 @@ fn main() {
         .subcommand(validator_client::cli_app())
         .subcommand(account_manager::cli_app())
         .subcommand(database_manager::cli_app())
+        .subcommand(validator_manager::cli_app())
         .get_matches();
 
     // Configure the allocator early in the process, before it has the chance to use the default values for
@@ -567,6 +568,16 @@ fn run<E: EthSpec>(
         return Ok(());
     }
 
+    if let Some(sub_matches) = matches.subcommand_matches(validator_manager::CMD) {
+        eprintln!("Running validator manager for {} network", network_name);
+
+        // Pass the entire `environment` to the account manager so it can run blocking operations.
+        validator_manager::run::<E>(sub_matches, environment)?;
+
+        // Exit as soon as account manager returns control.
+        return Ok(());
+    }
+
     if let Some(sub_matches) = matches.subcommand_matches(database_manager::CMD) {
         info!(log, "Running database manager for {} network", network_name);
         // Pass the entire `environment` to the database manager so it can run blocking operations.

lighthouse/tests/main.rs

@@ -5,3 +5,4 @@ mod beacon_node;
 mod boot_node;
 mod exec;
 mod validator_client;
+mod validator_manager;

lighthouse/tests/validator_client.rs

@@ -309,6 +309,32 @@ fn http_allow_origin_all_flag() {
         .run()
         .with_config(|config| assert_eq!(config.http_api.allow_origin, Some("*".to_string())));
 }
+#[test]
+fn http_allow_keystore_export_default() {
+    CommandLineTest::new()
+        .run()
+        .with_config(|config| assert!(!config.http_api.allow_keystore_export));
+}
+#[test]
+fn http_allow_keystore_export_present() {
+    CommandLineTest::new()
+        .flag("http-allow-keystore-export", None)
+        .run()
+        .with_config(|config| assert!(config.http_api.allow_keystore_export));
+}
+#[test]
+fn http_store_keystore_passwords_in_secrets_dir_default() {
+    CommandLineTest::new()
+        .run()
+        .with_config(|config| assert!(!config.http_api.store_passwords_in_secrets_dir));
+}
+#[test]
+fn http_store_keystore_passwords_in_secrets_dir_present() {
+    CommandLineTest::new()
+        .flag("http-store-passwords-in-secrets-dir", None)
+        .run()
+        .with_config(|config| assert!(config.http_api.store_passwords_in_secrets_dir));
+}
 
 // Tests for Metrics flags.
 #[test]

lighthouse/tests/validator_manager.rs

@@ -0,0 +1,344 @@
+use eth2::SensitiveUrl;
+use serde::de::DeserializeOwned;
+use std::fs;
+use std::marker::PhantomData;
+use std::path::PathBuf;
+use std::process::{Command, Stdio};
+use std::str::FromStr;
+use tempfile::{tempdir, TempDir};
+use types::*;
+use validator_manager::{
+    create_validators::CreateConfig,
+    import_validators::ImportConfig,
+    move_validators::{MoveConfig, PasswordSource, Validators},
+};
+
+const EXAMPLE_ETH1_ADDRESS: &str = "0x00000000219ab540356cBB839Cbe05303d7705Fa";
+
+const EXAMPLE_PUBKEY_0: &str = "0x933ad9491b62059dd065b560d256d8957a8c402cc6e8d8ee7290ae11e8f7329267a8811c397529dac52ae1342ba58c95";
+const EXAMPLE_PUBKEY_1: &str = "0xa1d1ad0714035353258038e964ae9675dc0252ee22cea896825c01458e1807bfad2f9969338798548d9858a571f7425c";
+
+struct CommandLineTest<T> {
+    cmd: Command,
+    config_path: PathBuf,
+    _dir: TempDir,
+    _phantom: PhantomData<T>,
+}
+
+impl<T> Default for CommandLineTest<T> {
+    fn default() -> Self {
+        let dir = tempdir().unwrap();
+        let config_path = dir.path().join("config.json");
+        let mut cmd = Command::new(env!("CARGO_BIN_EXE_lighthouse"));
+        cmd.arg("--dump-config")
+            .arg(config_path.as_os_str())
+            .arg("validator-manager")
+            .stdin(Stdio::null())
+            .stdout(Stdio::null())
+            .stderr(Stdio::null());
+        Self {
+            cmd,
+            config_path,
+            _dir: dir,
+            _phantom: PhantomData,
+        }
+    }
+}
+
+impl<T> CommandLineTest<T> {
+    fn flag(mut self, flag: &str, value: Option<&str>) -> Self {
+        self.cmd.arg(flag);
+        if let Some(value) = value {
+            self.cmd.arg(value);
+        }
+        self
+    }
+
+    fn run(mut cmd: Command, should_succeed: bool) {
+        let output = cmd.output().expect("process should complete");
+        if output.status.success() != should_succeed {
+            let stdout = String::from_utf8(output.stdout).unwrap();
+            let stderr = String::from_utf8(output.stderr).unwrap();
+            eprintln!("{}", stdout);
+            eprintln!("{}", stderr);
+            panic!(
+                "Command success was {} when expecting {}",
+                !should_succeed, should_succeed
+            );
+        }
+    }
+}
+
+impl<T: DeserializeOwned> CommandLineTest<T> {
+    fn assert_success<F: Fn(T)>(self, func: F) {
+        Self::run(self.cmd, true);
+        let contents = fs::read_to_string(self.config_path).unwrap();
+        let config: T = serde_json::from_str(&contents).unwrap();
+        func(config)
+    }
+
+    fn assert_failed(self) {
+        Self::run(self.cmd, false);
+    }
+}
+
+impl CommandLineTest<CreateConfig> {
+    fn validators_create() -> Self {
+        Self::default().flag("create", None)
+    }
+}
+
+impl CommandLineTest<ImportConfig> {
+    fn validators_import() -> Self {
+        Self::default().flag("import", None)
+    }
+}
+
+impl CommandLineTest<MoveConfig> {
+    fn validators_move() -> Self {
+        Self::default().flag("move", None)
+    }
+}
+
+#[test]
+pub fn validator_create_without_output_path() {
+    CommandLineTest::validators_create().assert_failed();
+}
+
+#[test]
+pub fn validator_create_defaults() {
+    CommandLineTest::validators_create()
+        .flag("--output-path", Some("./meow"))
+        .flag("--count", Some("1"))
+        .assert_success(|config| {
+            let expected = CreateConfig {
+                output_path: PathBuf::from("./meow"),
+                first_index: 0,
+                count: 1,
+                deposit_gwei: MainnetEthSpec::default_spec().max_effective_balance,
+                mnemonic_path: None,
+                stdin_inputs: cfg!(windows) || false,
+                disable_deposits: false,
+                specify_voting_keystore_password: false,
+                eth1_withdrawal_address: None,
+                builder_proposals: None,
+                fee_recipient: None,
+                gas_limit: None,
+                bn_url: None,
+                force_bls_withdrawal_credentials: false,
+            };
+            assert_eq!(expected, config);
+        });
+}
+
+#[test]
+pub fn validator_create_misc_flags() {
+    CommandLineTest::validators_create()
+        .flag("--output-path", Some("./meow"))
+        .flag("--deposit-gwei", Some("42"))
+        .flag("--first-index", Some("12"))
+        .flag("--count", Some("9"))
+        .flag("--mnemonic-path", Some("./woof"))
+        .flag("--stdin-inputs", None)
+        .flag("--specify-voting-keystore-password", None)
+        .flag("--eth1-withdrawal-address", Some(EXAMPLE_ETH1_ADDRESS))
+        .flag("--builder-proposals", Some("true"))
+        .flag("--suggested-fee-recipient", Some(EXAMPLE_ETH1_ADDRESS))
+        .flag("--gas-limit", Some("1337"))
+        .flag("--beacon-node", Some("http://localhost:1001"))
+        .flag("--force-bls-withdrawal-credentials", None)
+        .assert_success(|config| {
+            let expected = CreateConfig {
+                output_path: PathBuf::from("./meow"),
+                first_index: 12,
+                count: 9,
+                deposit_gwei: 42,
+                mnemonic_path: Some(PathBuf::from("./woof")),
+                stdin_inputs: true,
+                disable_deposits: false,
+                specify_voting_keystore_password: true,
+                eth1_withdrawal_address: Some(Address::from_str(EXAMPLE_ETH1_ADDRESS).unwrap()),
+                builder_proposals: Some(true),
+                fee_recipient: Some(Address::from_str(EXAMPLE_ETH1_ADDRESS).unwrap()),
+                gas_limit: Some(1337),
+                bn_url: Some(SensitiveUrl::parse("http://localhost:1001").unwrap()),
+                force_bls_withdrawal_credentials: true,
+            };
+            assert_eq!(expected, config);
+        });
+}
+
+#[test]
+pub fn validator_create_disable_deposits() {
+    CommandLineTest::validators_create()
+        .flag("--output-path", Some("./meow"))
+        .flag("--count", Some("1"))
+        .flag("--disable-deposits", None)
+        .flag("--builder-proposals", Some("false"))
+        .assert_success(|config| {
+            assert_eq!(config.disable_deposits, true);
+            assert_eq!(config.builder_proposals, Some(false));
+        });
+}
+
+#[test]
+pub fn validator_import_defaults() {
+    CommandLineTest::validators_import()
+        .flag("--validators-file", Some("./vals.json"))
+        .flag("--vc-token", Some("./token.json"))
+        .assert_success(|config| {
+            let expected = ImportConfig {
+                validators_file_path: PathBuf::from("./vals.json"),
+                vc_url: SensitiveUrl::parse("http://localhost:5062").unwrap(),
+                vc_token_path: PathBuf::from("./token.json"),
+                ignore_duplicates: false,
+            };
+            assert_eq!(expected, config);
+        });
+}
+
+#[test]
+pub fn validator_import_misc_flags() {
+    CommandLineTest::validators_import()
+        .flag("--validators-file", Some("./vals.json"))
+        .flag("--vc-token", Some("./token.json"))
+        .flag("--ignore-duplicates", None)
+        .assert_success(|config| {
+            let expected = ImportConfig {
+                validators_file_path: PathBuf::from("./vals.json"),
+                vc_url: SensitiveUrl::parse("http://localhost:5062").unwrap(),
+                vc_token_path: PathBuf::from("./token.json"),
+                ignore_duplicates: true,
+            };
+            assert_eq!(expected, config);
+        });
+}
+
+#[test]
+pub fn validator_import_missing_token() {
+    CommandLineTest::validators_import()
+        .flag("--validators-file", Some("./vals.json"))
+        .assert_failed();
+}
+
+#[test]
+pub fn validator_import_missing_validators_file() {
+    CommandLineTest::validators_import()
+        .flag("--vc-token", Some("./token.json"))
+        .assert_failed();
+}
+
+#[test]
+pub fn validator_move_defaults() {
+    CommandLineTest::validators_move()
+        .flag("--src-vc-url", Some("http://localhost:1"))
+        .flag("--src-vc-token", Some("./1.json"))
+        .flag("--dest-vc-url", Some("http://localhost:2"))
+        .flag("--dest-vc-token", Some("./2.json"))
+        .flag("--validators", Some("all"))
+        .assert_success(|config| {
+            let expected = MoveConfig {
+                src_vc_url: SensitiveUrl::parse("http://localhost:1").unwrap(),
+                src_vc_token_path: PathBuf::from("./1.json"),
+                dest_vc_url: SensitiveUrl::parse("http://localhost:2").unwrap(),
+                dest_vc_token_path: PathBuf::from("./2.json"),
+                validators: Validators::All,
+                builder_proposals: None,
+                fee_recipient: None,
+                gas_limit: None,
+                password_source: PasswordSource::Interactive {
+                    stdin_inputs: cfg!(windows) || false,
+                },
+            };
+            assert_eq!(expected, config);
+        });
+}
+
+#[test]
+pub fn validator_move_misc_flags_0() {
+    CommandLineTest::validators_move()
+        .flag("--src-vc-url", Some("http://localhost:1"))
+        .flag("--src-vc-token", Some("./1.json"))
+        .flag("--dest-vc-url", Some("http://localhost:2"))
+        .flag("--dest-vc-token", Some("./2.json"))
+        .flag(
+            "--validators",
+            Some(&format!("{},{}", EXAMPLE_PUBKEY_0, EXAMPLE_PUBKEY_1)),
+        )
+        .flag("--builder-proposals", Some("true"))
+        .flag("--suggested-fee-recipient", Some(EXAMPLE_ETH1_ADDRESS))
+        .flag("--gas-limit", Some("1337"))
+        .flag("--stdin-inputs", None)
+        .assert_success(|config| {
+            let expected = MoveConfig {
+                src_vc_url: SensitiveUrl::parse("http://localhost:1").unwrap(),
+                src_vc_token_path: PathBuf::from("./1.json"),
+                dest_vc_url: SensitiveUrl::parse("http://localhost:2").unwrap(),
+                dest_vc_token_path: PathBuf::from("./2.json"),
+                validators: Validators::Specific(vec![
+                    PublicKeyBytes::from_str(EXAMPLE_PUBKEY_0).unwrap(),
+                    PublicKeyBytes::from_str(EXAMPLE_PUBKEY_1).unwrap(),
+                ]),
+                builder_proposals: Some(true),
+                fee_recipient: Some(Address::from_str(EXAMPLE_ETH1_ADDRESS).unwrap()),
+                gas_limit: Some(1337),
+                password_source: PasswordSource::Interactive { stdin_inputs: true },
+            };
+            assert_eq!(expected, config);
+        });
+}
+
+#[test]
+pub fn validator_move_misc_flags_1() {
+    CommandLineTest::validators_move()
+        .flag("--src-vc-url", Some("http://localhost:1"))
+        .flag("--src-vc-token", Some("./1.json"))
+        .flag("--dest-vc-url", Some("http://localhost:2"))
+        .flag("--dest-vc-token", Some("./2.json"))
+        .flag("--validators", Some(&format!("{}", EXAMPLE_PUBKEY_0)))
+        .flag("--builder-proposals", Some("false"))
+        .assert_success(|config| {
+            let expected = MoveConfig {
+                src_vc_url: SensitiveUrl::parse("http://localhost:1").unwrap(),
+                src_vc_token_path: PathBuf::from("./1.json"),
+                dest_vc_url: SensitiveUrl::parse("http://localhost:2").unwrap(),
+                dest_vc_token_path: PathBuf::from("./2.json"),
+                validators: Validators::Specific(vec![
+                    PublicKeyBytes::from_str(EXAMPLE_PUBKEY_0).unwrap()
+                ]),
+                builder_proposals: Some(false),
+                fee_recipient: None,
+                gas_limit: None,
+                password_source: PasswordSource::Interactive {
+                    stdin_inputs: cfg!(windows) || false,
+                },
+            };
+            assert_eq!(expected, config);
+        });
+}
+
+#[test]
+pub fn validator_move_count() {
+    CommandLineTest::validators_move()
+        .flag("--src-vc-url", Some("http://localhost:1"))
+        .flag("--src-vc-token", Some("./1.json"))
+        .flag("--dest-vc-url", Some("http://localhost:2"))
+        .flag("--dest-vc-token", Some("./2.json"))
+        .flag("--count", Some("42"))
+        .assert_success(|config| {
+            let expected = MoveConfig {
+                src_vc_url: SensitiveUrl::parse("http://localhost:1").unwrap(),
+                src_vc_token_path: PathBuf::from("./1.json"),
+                dest_vc_url: SensitiveUrl::parse("http://localhost:2").unwrap(),
+                dest_vc_token_path: PathBuf::from("./2.json"),
+                validators: Validators::Count(42),
+                builder_proposals: None,
+                fee_recipient: None,
+                gas_limit: None,
+                password_source: PasswordSource::Interactive {
+                    stdin_inputs: cfg!(windows) || false,
+                },
+            };
+            assert_eq!(expected, config);
+        });
+}

validator_client/Cargo.toml

@@ -10,7 +10,6 @@ path = "src/lib.rs"
 
 [dev-dependencies]
 tokio = { version = "1.14.0", features = ["time", "rt-multi-thread", "macros"] }
-logging = { path = "../common/logging" }
 
 [dependencies]
 tree_hash = "0.5.0"
@@ -63,4 +62,3 @@ malloc_utils = { path = "../common/malloc_utils" }
 sysinfo = "0.26.5"
 system_health = { path = "../common/system_health" }
 logging = { path = "../common/logging" }
-

validator_client/src/attestation_service.rs

@@ -2,12 +2,12 @@ use crate::beacon_node_fallback::{BeaconNodeFallback, RequireSynced};
 use crate::{
     duties_service::{DutiesService, DutyAndProof},
     http_metrics::metrics,
-    validator_store::ValidatorStore,
+    validator_store::{Error as ValidatorStoreError, ValidatorStore},
     OfflineOnFailure,
 };
 use environment::RuntimeContext;
 use futures::future::join_all;
-use slog::{crit, error, info, trace};
+use slog::{crit, debug, error, info, trace, warn};
 use slot_clock::SlotClock;
 use std::collections::HashMap;
 use std::ops::Deref;
@@ -395,6 +395,20 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
                 .await
             {
                 Ok(()) => Some((attestation, duty.validator_index)),
+                Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
+                    // A pubkey can be missing when a validator was recently
+                    // removed via the API.
+                    warn!(
+                        log,
+                        "Missing pubkey for attestation";
+                        "info" => "a validator may have recently been removed from this VC",
+                        "pubkey" => ?pubkey,
+                        "validator" => ?duty.pubkey,
+                        "committee_index" => committee_index,
+                        "slot" => slot.as_u64(),
+                    );
+                    None
+                }
                 Err(e) => {
                     crit!(
                         log,
@@ -527,10 +541,20 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
                 .await
             {
                 Ok(aggregate) => Some(aggregate),
+                Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
+                    // A pubkey can be missing when a validator was recently
+                    // removed via the API.
+                    debug!(
+                        log,
+                        "Missing pubkey for aggregate";
+                        "pubkey" => ?pubkey,
+                    );
+                    None
+                }
                 Err(e) => {
                     crit!(
                         log,
-                        "Failed to sign attestation";
+                        "Failed to sign aggregate";
                         "error" => ?e,
                         "pubkey" => ?duty.pubkey,
                     );

validator_client/src/block_service.rs

@@ -5,7 +5,10 @@ use crate::{
     graffiti_file::GraffitiFile,
     OfflineOnFailure,
 };
-use crate::{http_metrics::metrics, validator_store::ValidatorStore};
+use crate::{
+    http_metrics::metrics,
+    validator_store::{Error as ValidatorStoreError, ValidatorStore},
+};
 use environment::RuntimeContext;
 use eth2::BeaconNodeHttpClient;
 use slog::{crit, debug, error, info, trace, warn};
@@ -417,17 +420,31 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
             BlockError::Recoverable("Unable to determine current slot from clock".to_string())
         })?;
 
-        let randao_reveal = self
+        let randao_reveal = match self
             .validator_store
             .randao_reveal(validator_pubkey, slot.epoch(E::slots_per_epoch()))
             .await
-            .map_err(|e| {
+        {
-                BlockError::Recoverable(format!(
+            Ok(signature) => signature.into(),
+            Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
+                // A pubkey can be missing when a validator was recently removed
+                // via the API.
+                warn!(
+                    log,
+                    "Missing pubkey for block randao";
+                    "info" => "a validator may have recently been removed from this VC",
+                    "pubkey" => ?pubkey,
+                    "slot" => ?slot
+                );
+                return Ok(());
+            }
+            Err(e) => {
+                return Err(BlockError::Recoverable(format!(
                     "Unable to produce randao reveal signature: {:?}",
                     e
-                ))
+                )))
-            })?
+            }
-            .into();
+        };
 
         let graffiti = determine_graffiti(
             &validator_pubkey,
@@ -522,11 +539,31 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
             .await?;
 
         let signing_timer = metrics::start_timer(&metrics::BLOCK_SIGNING_TIMES);
-        let signed_block = self_ref
+        let signed_block = match self_ref
             .validator_store
             .sign_block::<Payload>(*validator_pubkey_ref, block, current_slot)
             .await
-            .map_err(|e| BlockError::Recoverable(format!("Unable to sign block: {:?}", e)))?;
+        {
+            Ok(block) => block,
+            Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
+                // A pubkey can be missing when a validator was recently removed
+                // via the API.
+                warn!(
+                    log,
+                    "Missing pubkey for block";
+                    "info" => "a validator may have recently been removed from this VC",
+                    "pubkey" => ?pubkey,
+                    "slot" => ?slot
+                );
+                return Ok(());
+            }
+            Err(e) => {
+                return Err(BlockError::Recoverable(format!(
+                    "Unable to sign block: {:?}",
+                    e
+                )))
+            }
+        };
         let signing_time_ms =
             Duration::from_secs_f64(signing_timer.map_or(0.0, |t| t.stop_and_record())).as_millis();
 

validator_client/src/cli.rs

@@ -204,6 +204,26 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                     address of this server (e.g., http://localhost:5062).")
                 .takes_value(true),
         )
+        .arg(
+            Arg::with_name("http-allow-keystore-export")
+                .long("http-allow-keystore-export")
+                .help("If present, allow access to the DELETE /lighthouse/keystores HTTP \
+                    API method, which allows exporting keystores and passwords to HTTP API \
+                    consumers who have access to the API token. This method is useful for \
+                    exporting validators, however it should be used with caution since it \
+                    exposes private key data to authorized users.")
+                .required(false)
+                .takes_value(false),
+        )
+        .arg(
+            Arg::with_name("http-store-passwords-in-secrets-dir")
+                .long("http-store-passwords-in-secrets-dir")
+                .help("If present, any validators created via the HTTP will have keystore \
+                    passwords stored in the secrets-dir rather than the validator \
+                    definitions file.")
+                .required(false)
+                .takes_value(false),
+        )
         /* Prometheus metrics HTTP server related arguments */
         .arg(
             Arg::with_name("metrics")

validator_client/src/config.rs

@@ -294,6 +294,14 @@ impl Config {
             config.http_api.allow_origin = Some(allow_origin.to_string());
         }
 
+        if cli_args.is_present("http-allow-keystore-export") {
+            config.http_api.allow_keystore_export = true;
+        }
+
+        if cli_args.is_present("http-store-passwords-in-secrets-dir") {
+            config.http_api.store_passwords_in_secrets_dir = true;
+        }
+
         /*
          * Prometheus metrics HTTP server
          */

validator_client/src/duties_service.rs

@@ -932,6 +932,20 @@ async fn fill_in_selection_proofs<T: SlotClock + 'static, E: EthSpec>(
             for result in duty_and_proof_results {
                 let duty_and_proof = match result {
                     Ok(duty_and_proof) => duty_and_proof,
+                    Err(Error::FailedToProduceSelectionProof(
+                        ValidatorStoreError::UnknownPubkey(pubkey),
+                    )) => {
+                        // A pubkey can be missing when a validator was recently
+                        // removed via the API.
+                        warn!(
+                            log,
+                            "Missing pubkey for duty and proof";
+                            "info" => "a validator may have recently been removed from this VC",
+                            "pubkey" => ?pubkey,
+                        );
+                        // Do not abort the entire batch for a single failure.
+                        continue;
+                    }
                     Err(e) => {
                         error!(
                             log,

validator_client/src/duties_service/sync.rs

@@ -2,6 +2,7 @@ use crate::beacon_node_fallback::{OfflineOnFailure, RequireSynced};
 use crate::{
     doppelganger_service::DoppelgangerStatus,
     duties_service::{DutiesService, Error},
+    validator_store::Error as ValidatorStoreError,
 };
 use futures::future::join_all;
 use itertools::Itertools;
@@ -539,6 +540,18 @@ pub async fn fill_in_aggregation_proofs<T: SlotClock + 'static, E: EthSpec>(
                         .await
                     {
                         Ok(proof) => proof,
+                        Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
+                            // A pubkey can be missing when a validator was recently
+                            // removed via the API.
+                            debug!(
+                                log,
+                                "Missing pubkey for sync selection proof";
+                                "pubkey" => ?pubkey,
+                                "pubkey" => ?duty.pubkey,
+                                "slot" => slot,
+                            );
+                            return None;
+                        }
                         Err(e) => {
                             warn!(
                                 log,

validator_client/src/http_api/create_validator.rs

@@ -1,15 +1,16 @@
 use crate::ValidatorStore;
-use account_utils::validator_definitions::ValidatorDefinition;
+use account_utils::validator_definitions::{PasswordStorage, ValidatorDefinition};
 use account_utils::{
+    eth2_keystore::Keystore,
     eth2_wallet::{bip39::Mnemonic, WalletBuilder},
     random_mnemonic, random_password, ZeroizeString,
 };
 use eth2::lighthouse_vc::types::{self as api_types};
 use slot_clock::SlotClock;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use types::ChainSpec;
 use types::EthSpec;
-use validator_dir::Builder as ValidatorDirBuilder;
+use validator_dir::{keystore_password_path, Builder as ValidatorDirBuilder};
 
 /// Create some validator EIP-2335 keystores and store them on disk. Then, enroll the validators in
 /// this validator client.
@@ -27,6 +28,7 @@ pub async fn create_validators_mnemonic<P: AsRef<Path>, T: 'static + SlotClock,
     key_derivation_path_offset: Option<u32>,
     validator_requests: &[api_types::ValidatorRequest],
     validator_dir: P,
+    secrets_dir: Option<PathBuf>,
     validator_store: &ValidatorStore<T, E>,
     spec: &ChainSpec,
 ) -> Result<(Vec<api_types::CreatedValidator>, Mnemonic), warp::Rejection> {
@@ -95,7 +97,11 @@ pub async fn create_validators_mnemonic<P: AsRef<Path>, T: 'static + SlotClock,
                 ))
             })?;
 
+        let voting_password_storage =
+            get_voting_password_storage(&secrets_dir, &keystores.voting, &voting_password_string)?;
+
         let validator_dir = ValidatorDirBuilder::new(validator_dir.as_ref().into())
+            .password_dir_opt(secrets_dir.clone())
             .voting_keystore(keystores.voting, voting_password.as_bytes())
             .withdrawal_keystore(keystores.withdrawal, withdrawal_password.as_bytes())
             .create_eth1_tx_data(request.deposit_gwei, spec)
@@ -136,7 +142,7 @@ pub async fn create_validators_mnemonic<P: AsRef<Path>, T: 'static + SlotClock,
         validator_store
             .add_validator_keystore(
                 voting_keystore_path,
-                voting_password_string,
+                voting_password_storage,
                 request.enable,
                 request.graffiti.clone(),
                 request.suggested_fee_recipient,
@@ -185,3 +191,26 @@ pub async fn create_validators_web3signer<T: 'static + SlotClock, E: EthSpec>(
 
     Ok(())
 }
+
+/// Attempts to return a `PasswordStorage::File` if `secrets_dir` is defined.
+/// Otherwise, returns a `PasswordStorage::ValidatorDefinitions`.
+pub fn get_voting_password_storage(
+    secrets_dir: &Option<PathBuf>,
+    voting_keystore: &Keystore,
+    voting_password_string: &ZeroizeString,
+) -> Result<PasswordStorage, warp::Rejection> {
+    if let Some(secrets_dir) = &secrets_dir {
+        let password_path = keystore_password_path(secrets_dir, voting_keystore);
+        if password_path.exists() {
+            Err(warp_utils::reject::custom_server_error(
+                "Duplicate keystore password path".to_string(),
+            ))
+        } else {
+            Ok(PasswordStorage::File(password_path))
+        }
+    } else {
+        Ok(PasswordStorage::ValidatorDefinitions(
+            voting_password_string.clone(),
+        ))
+    }
+}

validator_client/src/http_api/keystores.rs

@@ -3,11 +3,14 @@ use crate::{
     initialized_validators::Error, signing_method::SigningMethod, InitializedValidators,
     ValidatorStore,
 };
-use account_utils::ZeroizeString;
+use account_utils::{validator_definitions::PasswordStorage, ZeroizeString};
-use eth2::lighthouse_vc::std_types::{
+use eth2::lighthouse_vc::{
-    DeleteKeystoreStatus, DeleteKeystoresRequest, DeleteKeystoresResponse, ImportKeystoreStatus,
+    std_types::{
-    ImportKeystoresRequest, ImportKeystoresResponse, InterchangeJsonStr, KeystoreJsonStr,
+        DeleteKeystoreStatus, DeleteKeystoresRequest, DeleteKeystoresResponse,
-    ListKeystoresResponse, SingleKeystoreResponse, Status,
+        ImportKeystoreStatus, ImportKeystoresRequest, ImportKeystoresResponse, InterchangeJsonStr,
+        KeystoreJsonStr, ListKeystoresResponse, SingleKeystoreResponse, Status,
+    },
+    types::{ExportKeystoresResponse, SingleExportKeystoresResponse},
 };
 use eth2_keystore::Keystore;
 use slog::{info, warn, Logger};
@@ -17,7 +20,7 @@ use std::sync::Arc;
 use task_executor::TaskExecutor;
 use tokio::runtime::Handle;
 use types::{EthSpec, PublicKeyBytes};
-use validator_dir::Builder as ValidatorDirBuilder;
+use validator_dir::{keystore_password_path, Builder as ValidatorDirBuilder};
 use warp::Rejection;
 use warp_utils::reject::{custom_bad_request, custom_server_error};
 
@@ -58,6 +61,7 @@ pub fn list<T: SlotClock + 'static, E: EthSpec>(
 pub fn import<T: SlotClock + 'static, E: EthSpec>(
     request: ImportKeystoresRequest,
     validator_dir: PathBuf,
+    secrets_dir: Option<PathBuf>,
     validator_store: Arc<ValidatorStore<T, E>>,
     task_executor: TaskExecutor,
     log: Logger,
@@ -128,6 +132,7 @@ pub fn import<T: SlotClock + 'static, E: EthSpec>(
                 keystore,
                 password,
                 validator_dir.clone(),
+                secrets_dir.clone(),
                 &validator_store,
                 handle,
             ) {
@@ -158,6 +163,7 @@ fn import_single_keystore<T: SlotClock + 'static, E: EthSpec>(
     keystore: Keystore,
     password: ZeroizeString,
     validator_dir_path: PathBuf,
+    secrets_dir: Option<PathBuf>,
     validator_store: &ValidatorStore<T, E>,
     handle: Handle,
 ) -> Result<ImportKeystoreStatus, String> {
@@ -179,6 +185,16 @@ fn import_single_keystore<T: SlotClock + 'static, E: EthSpec>(
         }
     }
 
+    let password_storage = if let Some(secrets_dir) = &secrets_dir {
+        let password_path = keystore_password_path(secrets_dir, &keystore);
+        if password_path.exists() {
+            return Ok(ImportKeystoreStatus::Duplicate);
+        }
+        PasswordStorage::File(password_path)
+    } else {
+        PasswordStorage::ValidatorDefinitions(password.clone())
+    };
+
     // Check that the password is correct.
     // In future we should re-structure to avoid the double decryption here. It's not as simple
     // as removing this check because `add_validator_keystore` will break if provided with an
@@ -189,6 +205,7 @@ fn import_single_keystore<T: SlotClock + 'static, E: EthSpec>(
         .map_err(|e| format!("incorrect password: {:?}", e))?;
 
     let validator_dir = ValidatorDirBuilder::new(validator_dir_path)
+        .password_dir_opt(secrets_dir)
         .voting_keystore(keystore, password.as_ref())
         .store_withdrawal_keystore(false)
         .build()
@@ -201,7 +218,7 @@ fn import_single_keystore<T: SlotClock + 'static, E: EthSpec>(
     handle
         .block_on(validator_store.add_validator_keystore(
             voting_keystore_path,
-            password,
+            password_storage,
             true,
             None,
             None,
@@ -219,11 +236,28 @@ pub fn delete<T: SlotClock + 'static, E: EthSpec>(
     task_executor: TaskExecutor,
     log: Logger,
 ) -> Result<DeleteKeystoresResponse, Rejection> {
+    let export_response = export(request, validator_store, task_executor, log)?;
+    Ok(DeleteKeystoresResponse {
+        data: export_response
+            .data
+            .into_iter()
+            .map(|response| response.status)
+            .collect(),
+        slashing_protection: export_response.slashing_protection,
+    })
+}
+
+pub fn export<T: SlotClock + 'static, E: EthSpec>(
+    request: DeleteKeystoresRequest,
+    validator_store: Arc<ValidatorStore<T, E>>,
+    task_executor: TaskExecutor,
+    log: Logger,
+) -> Result<ExportKeystoresResponse, Rejection> {
     // Remove from initialized validators.
     let initialized_validators_rwlock = validator_store.initialized_validators();
     let mut initialized_validators = initialized_validators_rwlock.write();
 
-    let mut statuses = request
+    let mut responses = request
         .pubkeys
         .iter()
         .map(|pubkey_bytes| {
@@ -232,7 +266,7 @@ pub fn delete<T: SlotClock + 'static, E: EthSpec>(
                 &mut initialized_validators,
                 task_executor.clone(),
             ) {
-                Ok(status) => Status::ok(status),
+                Ok(status) => status,
                 Err(error) => {
                     warn!(
                         log,
@@ -240,7 +274,11 @@ pub fn delete<T: SlotClock + 'static, E: EthSpec>(
                         "pubkey" => ?pubkey_bytes,
                         "error" => ?error,
                     );
-                    Status::error(DeleteKeystoreStatus::Error, error)
+                    SingleExportKeystoresResponse {
+                        status: Status::error(DeleteKeystoreStatus::Error, error),
+                        validating_keystore: None,
+                        validating_keystore_password: None,
+                    }
                 }
             }
         })
@@ -263,19 +301,19 @@ pub fn delete<T: SlotClock + 'static, E: EthSpec>(
         })?;
 
     // Update stasuses based on availability of slashing protection data.
-    for (pubkey, status) in request.pubkeys.iter().zip(statuses.iter_mut()) {
+    for (pubkey, response) in request.pubkeys.iter().zip(responses.iter_mut()) {
-        if status.status == DeleteKeystoreStatus::NotFound
+        if response.status.status == DeleteKeystoreStatus::NotFound
             && slashing_protection
                 .data
                 .iter()
                 .any(|interchange_data| interchange_data.pubkey == *pubkey)
         {
-            status.status = DeleteKeystoreStatus::NotActive;
+            response.status.status = DeleteKeystoreStatus::NotActive;
         }
     }
 
-    Ok(DeleteKeystoresResponse {
+    Ok(ExportKeystoresResponse {
-        data: statuses,
+        data: responses,
         slashing_protection,
     })
 }
@@ -284,7 +322,7 @@ fn delete_single_keystore(
     pubkey_bytes: &PublicKeyBytes,
     initialized_validators: &mut InitializedValidators,
     task_executor: TaskExecutor,
-) -> Result<DeleteKeystoreStatus, String> {
+) -> Result<SingleExportKeystoresResponse, String> {
     if let Some(handle) = task_executor.handle() {
         let pubkey = pubkey_bytes
             .decompress()
@@ -292,9 +330,22 @@ fn delete_single_keystore(
 
         match handle.block_on(initialized_validators.delete_definition_and_keystore(&pubkey, true))
         {
-            Ok(_) => Ok(DeleteKeystoreStatus::Deleted),
+            Ok(Some(keystore_and_password)) => Ok(SingleExportKeystoresResponse {
+                status: Status::ok(DeleteKeystoreStatus::Deleted),
+                validating_keystore: Some(KeystoreJsonStr(keystore_and_password.keystore)),
+                validating_keystore_password: keystore_and_password.password,
+            }),
+            Ok(None) => Ok(SingleExportKeystoresResponse {
+                status: Status::ok(DeleteKeystoreStatus::Deleted),
+                validating_keystore: None,
+                validating_keystore_password: None,
+            }),
             Err(e) => match e {
-                Error::ValidatorNotInitialized(_) => Ok(DeleteKeystoreStatus::NotFound),
+                Error::ValidatorNotInitialized(_) => Ok(SingleExportKeystoresResponse {
+                    status: Status::ok(DeleteKeystoreStatus::NotFound),
+                    validating_keystore: None,
+                    validating_keystore_password: None,
+                }),
                 _ => Err(format!("unable to disable and delete: {:?}", e)),
             },
         }

validator_client/src/http_api/mod.rs

@@ -5,6 +5,8 @@ mod keystores;
 mod remotekeys;
 mod tests;
 
+pub mod test_utils;
+
 use crate::http_api::create_signed_voluntary_exit::create_signed_voluntary_exit;
 use crate::{determine_graffiti, GraffitiFile, ValidatorStore};
 use account_utils::{
@@ -12,7 +14,9 @@ use account_utils::{
     validator_definitions::{SigningDefinition, ValidatorDefinition, Web3SignerDefinition},
 };
 pub use api_secret::ApiSecret;
-use create_validator::{create_validators_mnemonic, create_validators_web3signer};
+use create_validator::{
+    create_validators_mnemonic, create_validators_web3signer, get_voting_password_storage,
+};
 use eth2::lighthouse_vc::{
     std_types::{AuthResponse, GetFeeRecipientResponse, GetGasLimitResponse},
     types::{self as api_types, GenericResponse, Graffiti, PublicKey, PublicKeyBytes},
@@ -71,6 +75,7 @@ pub struct Context<T: SlotClock, E: EthSpec> {
     pub api_secret: ApiSecret,
     pub validator_store: Option<Arc<ValidatorStore<T, E>>>,
     pub validator_dir: Option<PathBuf>,
+    pub secrets_dir: Option<PathBuf>,
     pub graffiti_file: Option<GraffitiFile>,
     pub graffiti_flag: Option<Graffiti>,
     pub spec: ChainSpec,
@@ -88,6 +93,8 @@ pub struct Config {
     pub listen_addr: IpAddr,
     pub listen_port: u16,
     pub allow_origin: Option<String>,
+    pub allow_keystore_export: bool,
+    pub store_passwords_in_secrets_dir: bool,
 }
 
 impl Default for Config {
@@ -97,6 +104,8 @@ impl Default for Config {
             listen_addr: IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)),
             listen_port: 5062,
             allow_origin: None,
+            allow_keystore_export: false,
+            store_passwords_in_secrets_dir: false,
         }
     }
 }
@@ -121,6 +130,8 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
     shutdown: impl Future<Output = ()> + Send + Sync + 'static,
 ) -> Result<(SocketAddr, impl Future<Output = ()>), Error> {
     let config = &ctx.config;
+    let allow_keystore_export = config.allow_keystore_export;
+    let store_passwords_in_secrets_dir = config.store_passwords_in_secrets_dir;
     let log = ctx.log.clone();
 
     // Configure CORS.
@@ -187,6 +198,17 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
             })
         });
 
+    let inner_secrets_dir = ctx.secrets_dir.clone();
+    let secrets_dir_filter = warp::any().map(move || inner_secrets_dir.clone()).and_then(
+        |secrets_dir: Option<_>| async move {
+            secrets_dir.ok_or_else(|| {
+                warp_utils::reject::custom_not_found(
+                    "secrets_dir directory is not initialized.".to_string(),
+                )
+            })
+        },
+    );
+
     let inner_graffiti_file = ctx.graffiti_file.clone();
     let graffiti_file_filter = warp::any().map(move || inner_graffiti_file.clone());
 
@@ -394,18 +416,21 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
         .and(warp::path::end())
         .and(warp::body::json())
         .and(validator_dir_filter.clone())
+        .and(secrets_dir_filter.clone())
         .and(validator_store_filter.clone())
         .and(spec_filter.clone())
         .and(signer.clone())
         .and(task_executor_filter.clone())
         .and_then(
-            |body: Vec<api_types::ValidatorRequest>,
+            move |body: Vec<api_types::ValidatorRequest>,
-             validator_dir: PathBuf,
+                  validator_dir: PathBuf,
-             validator_store: Arc<ValidatorStore<T, E>>,
+                  secrets_dir: PathBuf,
-             spec: Arc<ChainSpec>,
+                  validator_store: Arc<ValidatorStore<T, E>>,
-             signer,
+                  spec: Arc<ChainSpec>,
-             task_executor: TaskExecutor| {
+                  signer,
+                  task_executor: TaskExecutor| {
                 blocking_signed_json_task(signer, move || {
+                    let secrets_dir = store_passwords_in_secrets_dir.then_some(secrets_dir);
                     if let Some(handle) = task_executor.handle() {
                         let (validators, mnemonic) =
                             handle.block_on(create_validators_mnemonic(
@@ -413,6 +438,7 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
                                 None,
                                 &body,
                                 &validator_dir,
+                                secrets_dir,
                                 &validator_store,
                                 &spec,
                             ))?;
@@ -437,18 +463,21 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
         .and(warp::path::end())
         .and(warp::body::json())
         .and(validator_dir_filter.clone())
+        .and(secrets_dir_filter.clone())
         .and(validator_store_filter.clone())
         .and(spec_filter)
         .and(signer.clone())
         .and(task_executor_filter.clone())
         .and_then(
-            |body: api_types::CreateValidatorsMnemonicRequest,
+            move |body: api_types::CreateValidatorsMnemonicRequest,
-             validator_dir: PathBuf,
+                  validator_dir: PathBuf,
-             validator_store: Arc<ValidatorStore<T, E>>,
+                  secrets_dir: PathBuf,
-             spec: Arc<ChainSpec>,
+                  validator_store: Arc<ValidatorStore<T, E>>,
-             signer,
+                  spec: Arc<ChainSpec>,
-             task_executor: TaskExecutor| {
+                  signer,
+                  task_executor: TaskExecutor| {
                 blocking_signed_json_task(signer, move || {
+                    let secrets_dir = store_passwords_in_secrets_dir.then_some(secrets_dir);
                     if let Some(handle) = task_executor.handle() {
                         let mnemonic =
                             mnemonic_from_phrase(body.mnemonic.as_str()).map_err(|e| {
@@ -463,6 +492,7 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
                                 Some(body.key_derivation_path_offset),
                                 &body.validators,
                                 &validator_dir,
+                                secrets_dir,
                                 &validator_store,
                                 &spec,
                             ))?;
@@ -483,15 +513,17 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
         .and(warp::path::end())
         .and(warp::body::json())
         .and(validator_dir_filter.clone())
+        .and(secrets_dir_filter.clone())
         .and(validator_store_filter.clone())
         .and(signer.clone())
         .and(task_executor_filter.clone())
         .and_then(
-            |body: api_types::KeystoreValidatorsPostRequest,
+            move |body: api_types::KeystoreValidatorsPostRequest,
-             validator_dir: PathBuf,
+                  validator_dir: PathBuf,
-             validator_store: Arc<ValidatorStore<T, E>>,
+                  secrets_dir: PathBuf,
-             signer,
+                  validator_store: Arc<ValidatorStore<T, E>>,
-             task_executor: TaskExecutor| {
+                  signer,
+                  task_executor: TaskExecutor| {
                 blocking_signed_json_task(signer, move || {
                     // Check to ensure the password is correct.
                     let keypair = body
@@ -504,7 +536,12 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
                             ))
                         })?;
 
+                    let secrets_dir = store_passwords_in_secrets_dir.then_some(secrets_dir);
+                    let password_storage =
+                        get_voting_password_storage(&secrets_dir, &body.keystore, &body.password)?;
+
                     let validator_dir = ValidatorDirBuilder::new(validator_dir.clone())
+                        .password_dir_opt(secrets_dir)
                         .voting_keystore(body.keystore.clone(), body.password.as_ref())
                         .store_withdrawal_keystore(false)
                         .build()
@@ -518,7 +555,6 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
                     // Drop validator dir so that `add_validator_keystore` can re-lock the keystore.
                     let voting_keystore_path = validator_dir.voting_keystore_path();
                     drop(validator_dir);
-                    let voting_password = body.password.clone();
                     let graffiti = body.graffiti.clone();
                     let suggested_fee_recipient = body.suggested_fee_recipient;
                     let gas_limit = body.gas_limit;
@@ -529,7 +565,7 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
                             handle
                                 .block_on(validator_store.add_validator_keystore(
                                     voting_keystore_path,
-                                    voting_password,
+                                    password_storage,
                                     body.enable,
                                     graffiti,
                                     suggested_fee_recipient,
@@ -698,6 +734,29 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
             })
         });
 
+    // DELETE /lighthouse/keystores
+    let delete_lighthouse_keystores = warp::path("lighthouse")
+        .and(warp::path("keystores"))
+        .and(warp::path::end())
+        .and(warp::body::json())
+        .and(signer.clone())
+        .and(validator_store_filter.clone())
+        .and(task_executor_filter.clone())
+        .and(log_filter.clone())
+        .and_then(
+            move |request, signer, validator_store, task_executor, log| {
+                blocking_signed_json_task(signer, move || {
+                    if allow_keystore_export {
+                        keystores::export(request, validator_store, task_executor, log)
+                    } else {
+                        Err(warp_utils::reject::custom_bad_request(
+                            "keystore export is disabled".to_string(),
+                        ))
+                    }
+                })
+            },
+        );
+
     // Standard key-manager endpoints.
     let eth_v1 = warp::path("eth").and(warp::path("v1"));
     let std_keystores = eth_v1.and(warp::path("keystores")).and(warp::path::end());
@@ -982,13 +1041,28 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
         .and(warp::body::json())
         .and(signer.clone())
         .and(validator_dir_filter)
+        .and(secrets_dir_filter)
         .and(validator_store_filter.clone())
         .and(task_executor_filter.clone())
         .and(log_filter.clone())
         .and_then(
-            |request, signer, validator_dir, validator_store, task_executor, log| {
+            move |request,
+                  signer,
+                  validator_dir,
+                  secrets_dir,
+                  validator_store,
+                  task_executor,
+                  log| {
+                let secrets_dir = store_passwords_in_secrets_dir.then_some(secrets_dir);
                 blocking_signed_json_task(signer, move || {
-                    keystores::import(request, validator_dir, validator_store, task_executor, log)
+                    keystores::import(
+                        request,
+                        validator_dir,
+                        secrets_dir,
+                        validator_store,
+                        task_executor,
+                        log,
+                    )
                 })
             },
         );
@@ -1117,7 +1191,8 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
                 ))
                 .or(warp::patch().and(patch_validators))
                 .or(warp::delete().and(
-                    delete_fee_recipient
+                    delete_lighthouse_keystores
+                        .or(delete_fee_recipient)
                         .or(delete_gas_limit)
                         .or(delete_std_keystores)
                         .or(delete_std_remotekeys),

validator_client/src/http_api/test_utils.rs

@@ -0,0 +1,631 @@
+use crate::doppelganger_service::DoppelgangerService;
+use crate::key_cache::{KeyCache, CACHE_FILENAME};
+use crate::{
+    http_api::{ApiSecret, Config as HttpConfig, Context},
+    initialized_validators::{InitializedValidators, OnDecryptFailure},
+    Config, ValidatorDefinitions, ValidatorStore,
+};
+use account_utils::{
+    eth2_wallet::WalletBuilder, mnemonic_from_phrase, random_mnemonic, random_password,
+    ZeroizeString,
+};
+use deposit_contract::decode_eth1_tx_data;
+use eth2::{
+    lighthouse_vc::{http_client::ValidatorClientHttpClient, types::*},
+    types::ErrorMessage as ApiErrorMessage,
+    Error as ApiError,
+};
+use eth2_keystore::KeystoreBuilder;
+use logging::test_logger;
+use parking_lot::RwLock;
+use sensitive_url::SensitiveUrl;
+use slashing_protection::{SlashingDatabase, SLASHING_PROTECTION_FILENAME};
+use slot_clock::{SlotClock, TestingSlotClock};
+use std::future::Future;
+use std::marker::PhantomData;
+use std::net::{IpAddr, Ipv4Addr};
+use std::sync::Arc;
+use std::time::Duration;
+use task_executor::test_utils::TestRuntime;
+use tempfile::{tempdir, TempDir};
+use tokio::sync::oneshot;
+
+pub const PASSWORD_BYTES: &[u8] = &[42, 50, 37];
+pub const TEST_DEFAULT_FEE_RECIPIENT: Address = Address::repeat_byte(42);
+
+type E = MainnetEthSpec;
+
+pub struct HdValidatorScenario {
+    pub count: usize,
+    pub specify_mnemonic: bool,
+    pub key_derivation_path_offset: u32,
+    pub disabled: Vec<usize>,
+}
+
+pub struct KeystoreValidatorScenario {
+    pub enabled: bool,
+    pub correct_password: bool,
+}
+
+pub struct Web3SignerValidatorScenario {
+    pub count: usize,
+    pub enabled: bool,
+}
+
+pub struct ApiTester {
+    pub client: ValidatorClientHttpClient,
+    pub initialized_validators: Arc<RwLock<InitializedValidators>>,
+    pub validator_store: Arc<ValidatorStore<TestingSlotClock, E>>,
+    pub url: SensitiveUrl,
+    pub api_token: String,
+    pub test_runtime: TestRuntime,
+    pub _server_shutdown: oneshot::Sender<()>,
+    pub validator_dir: TempDir,
+    pub secrets_dir: TempDir,
+}
+
+impl ApiTester {
+    pub async fn new() -> Self {
+        Self::new_with_http_config(Self::default_http_config()).await
+    }
+
+    pub async fn new_with_http_config(http_config: HttpConfig) -> Self {
+        let log = test_logger();
+
+        let validator_dir = tempdir().unwrap();
+        let secrets_dir = tempdir().unwrap();
+
+        let validator_defs = ValidatorDefinitions::open_or_create(validator_dir.path()).unwrap();
+
+        let initialized_validators = InitializedValidators::from_definitions(
+            validator_defs,
+            validator_dir.path().into(),
+            log.clone(),
+        )
+        .await
+        .unwrap();
+
+        let api_secret = ApiSecret::create_or_open(validator_dir.path()).unwrap();
+        let api_pubkey = api_secret.api_token();
+
+        let config = Config {
+            validator_dir: validator_dir.path().into(),
+            secrets_dir: secrets_dir.path().into(),
+            fee_recipient: Some(TEST_DEFAULT_FEE_RECIPIENT),
+            ..Default::default()
+        };
+
+        let spec = E::default_spec();
+
+        let slashing_db_path = config.validator_dir.join(SLASHING_PROTECTION_FILENAME);
+        let slashing_protection = SlashingDatabase::open_or_create(&slashing_db_path).unwrap();
+
+        let slot_clock =
+            TestingSlotClock::new(Slot::new(0), Duration::from_secs(0), Duration::from_secs(1));
+
+        let test_runtime = TestRuntime::default();
+
+        let validator_store = Arc::new(ValidatorStore::<_, E>::new(
+            initialized_validators,
+            slashing_protection,
+            Hash256::repeat_byte(42),
+            spec,
+            Some(Arc::new(DoppelgangerService::new(log.clone()))),
+            slot_clock.clone(),
+            &config,
+            test_runtime.task_executor.clone(),
+            log.clone(),
+        ));
+
+        validator_store
+            .register_all_in_doppelganger_protection_if_enabled()
+            .expect("Should attach doppelganger service");
+
+        let initialized_validators = validator_store.initialized_validators();
+
+        let context = Arc::new(Context {
+            task_executor: test_runtime.task_executor.clone(),
+            api_secret,
+            validator_dir: Some(validator_dir.path().into()),
+            secrets_dir: Some(secrets_dir.path().into()),
+            validator_store: Some(validator_store.clone()),
+            graffiti_file: None,
+            graffiti_flag: Some(Graffiti::default()),
+            spec: E::default_spec(),
+            config: http_config,
+            log,
+            sse_logging_components: None,
+            slot_clock,
+            _phantom: PhantomData,
+        });
+        let ctx = context;
+        let (shutdown_tx, shutdown_rx) = oneshot::channel();
+        let server_shutdown = async {
+            // It's not really interesting why this triggered, just that it happened.
+            let _ = shutdown_rx.await;
+        };
+        let (listening_socket, server) = super::serve(ctx, server_shutdown).unwrap();
+
+        tokio::spawn(server);
+
+        let url = SensitiveUrl::parse(&format!(
+            "http://{}:{}",
+            listening_socket.ip(),
+            listening_socket.port()
+        ))
+        .unwrap();
+
+        let client = ValidatorClientHttpClient::new(url.clone(), api_pubkey.clone()).unwrap();
+
+        Self {
+            client,
+            initialized_validators,
+            validator_store,
+            url,
+            api_token: api_pubkey,
+            test_runtime,
+            _server_shutdown: shutdown_tx,
+            validator_dir,
+            secrets_dir,
+        }
+    }
+
+    pub fn default_http_config() -> HttpConfig {
+        HttpConfig {
+            enabled: true,
+            listen_addr: IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)),
+            listen_port: 0,
+            allow_origin: None,
+            allow_keystore_export: true,
+            store_passwords_in_secrets_dir: false,
+        }
+    }
+
+    /// Checks that the key cache exists and can be decrypted with the current
+    /// set of known validators.
+    #[allow(clippy::await_holding_lock)] // This is a test, so it should be fine.
+    pub async fn ensure_key_cache_consistency(&self) {
+        assert!(
+            self.validator_dir.as_ref().join(CACHE_FILENAME).exists(),
+            "the key cache should exist"
+        );
+        let key_cache =
+            KeyCache::open_or_create(self.validator_dir.as_ref()).expect("should open a key cache");
+
+        self.initialized_validators
+            .read()
+            .decrypt_key_cache(key_cache, &mut <_>::default(), OnDecryptFailure::Error)
+            .await
+            .expect("key cache should decypt");
+    }
+
+    pub fn invalid_token_client(&self) -> ValidatorClientHttpClient {
+        let tmp = tempdir().unwrap();
+        let api_secret = ApiSecret::create_or_open(tmp.path()).unwrap();
+        let invalid_pubkey = api_secret.api_token();
+        ValidatorClientHttpClient::new(self.url.clone(), invalid_pubkey).unwrap()
+    }
+
+    pub async fn test_with_invalid_auth<F, A, T>(self, func: F) -> Self
+    where
+        F: Fn(ValidatorClientHttpClient) -> A,
+        A: Future<Output = Result<T, ApiError>>,
+    {
+        /*
+         * Test with an invalid Authorization header.
+         */
+        match func(self.invalid_token_client()).await {
+            Err(ApiError::ServerMessage(ApiErrorMessage { code: 403, .. })) => (),
+            Err(other) => panic!("expected authorized error, got {:?}", other),
+            Ok(_) => panic!("expected authorized error, got Ok"),
+        }
+
+        /*
+         * Test with a missing Authorization header.
+         */
+        let mut missing_token_client = self.client.clone();
+        missing_token_client.send_authorization_header(false);
+        match func(missing_token_client).await {
+            Err(ApiError::ServerMessage(ApiErrorMessage {
+                code: 401, message, ..
+            })) if message.contains("missing Authorization header") => (),
+            Err(other) => panic!("expected missing header error, got {:?}", other),
+            Ok(_) => panic!("expected missing header error, got Ok"),
+        }
+
+        self
+    }
+
+    pub fn invalidate_api_token(mut self) -> Self {
+        self.client = self.invalid_token_client();
+        self
+    }
+
+    pub async fn test_get_lighthouse_version_invalid(self) -> Self {
+        self.client.get_lighthouse_version().await.unwrap_err();
+        self
+    }
+
+    pub async fn test_get_lighthouse_spec(self) -> Self {
+        let result = self
+            .client
+            .get_lighthouse_spec::<ConfigAndPresetBellatrix>()
+            .await
+            .map(|res| ConfigAndPreset::Bellatrix(res.data))
+            .unwrap();
+        let expected = ConfigAndPreset::from_chain_spec::<E>(&E::default_spec(), None);
+
+        assert_eq!(result, expected);
+
+        self
+    }
+
+    pub async fn test_get_lighthouse_version(self) -> Self {
+        let result = self.client.get_lighthouse_version().await.unwrap().data;
+
+        let expected = VersionData {
+            version: lighthouse_version::version_with_platform(),
+        };
+
+        assert_eq!(result, expected);
+
+        self
+    }
+
+    #[cfg(target_os = "linux")]
+    pub async fn test_get_lighthouse_health(self) -> Self {
+        self.client.get_lighthouse_health().await.unwrap();
+
+        self
+    }
+
+    #[cfg(not(target_os = "linux"))]
+    pub async fn test_get_lighthouse_health(self) -> Self {
+        self.client.get_lighthouse_health().await.unwrap_err();
+
+        self
+    }
+    pub fn vals_total(&self) -> usize {
+        self.initialized_validators.read().num_total()
+    }
+
+    pub fn vals_enabled(&self) -> usize {
+        self.initialized_validators.read().num_enabled()
+    }
+
+    pub fn assert_enabled_validators_count(self, count: usize) -> Self {
+        assert_eq!(self.vals_enabled(), count);
+        self
+    }
+
+    pub fn assert_validators_count(self, count: usize) -> Self {
+        assert_eq!(self.vals_total(), count);
+        self
+    }
+
+    pub async fn create_hd_validators(self, s: HdValidatorScenario) -> Self {
+        let initial_vals = self.vals_total();
+        let initial_enabled_vals = self.vals_enabled();
+
+        let validators = (0..s.count)
+            .map(|i| ValidatorRequest {
+                enable: !s.disabled.contains(&i),
+                description: format!("boi #{}", i),
+                graffiti: None,
+                suggested_fee_recipient: None,
+                gas_limit: None,
+                builder_proposals: None,
+                deposit_gwei: E::default_spec().max_effective_balance,
+            })
+            .collect::<Vec<_>>();
+
+        let (response, mnemonic) = if s.specify_mnemonic {
+            let mnemonic = ZeroizeString::from(random_mnemonic().phrase().to_string());
+            let request = CreateValidatorsMnemonicRequest {
+                mnemonic: mnemonic.clone(),
+                key_derivation_path_offset: s.key_derivation_path_offset,
+                validators: validators.clone(),
+            };
+            let response = self
+                .client
+                .post_lighthouse_validators_mnemonic(&request)
+                .await
+                .unwrap()
+                .data;
+
+            (response, mnemonic)
+        } else {
+            assert_eq!(
+                s.key_derivation_path_offset, 0,
+                "cannot use a derivation offset without specifying a mnemonic"
+            );
+            let response = self
+                .client
+                .post_lighthouse_validators(validators.clone())
+                .await
+                .unwrap()
+                .data;
+            (response.validators.clone(), response.mnemonic)
+        };
+
+        assert_eq!(response.len(), s.count);
+        assert_eq!(self.vals_total(), initial_vals + s.count);
+        assert_eq!(
+            self.vals_enabled(),
+            initial_enabled_vals + s.count - s.disabled.len()
+        );
+
+        let server_vals = self.client.get_lighthouse_validators().await.unwrap().data;
+
+        assert_eq!(server_vals.len(), self.vals_total());
+
+        // Ensure the server lists all of these newly created validators.
+        for validator in &response {
+            assert!(server_vals
+                .iter()
+                .any(|server_val| server_val.voting_pubkey == validator.voting_pubkey));
+        }
+
+        /*
+         * Verify that we can regenerate all the keys from the mnemonic.
+         */
+
+        let mnemonic = mnemonic_from_phrase(mnemonic.as_str()).unwrap();
+        let mut wallet = WalletBuilder::from_mnemonic(&mnemonic, PASSWORD_BYTES, "".to_string())
+            .unwrap()
+            .build()
+            .unwrap();
+
+        wallet
+            .set_nextaccount(s.key_derivation_path_offset)
+            .unwrap();
+
+        for item in response.iter().take(s.count) {
+            let keypairs = wallet
+                .next_validator(PASSWORD_BYTES, PASSWORD_BYTES, PASSWORD_BYTES)
+                .unwrap();
+            let voting_keypair = keypairs.voting.decrypt_keypair(PASSWORD_BYTES).unwrap();
+
+            assert_eq!(
+                item.voting_pubkey,
+                voting_keypair.pk.clone().into(),
+                "the locally generated voting pk should match the server response"
+            );
+
+            let withdrawal_keypair = keypairs.withdrawal.decrypt_keypair(PASSWORD_BYTES).unwrap();
+
+            let deposit_bytes = serde_utils::hex::decode(&item.eth1_deposit_tx_data).unwrap();
+
+            let (deposit_data, _) =
+                decode_eth1_tx_data(&deposit_bytes, E::default_spec().max_effective_balance)
+                    .unwrap();
+
+            assert_eq!(
+                deposit_data.pubkey,
+                voting_keypair.pk.clone().into(),
+                "the locally generated voting pk should match the deposit data"
+            );
+
+            assert_eq!(
+                deposit_data.withdrawal_credentials,
+                Hash256::from_slice(&bls::get_withdrawal_credentials(
+                    &withdrawal_keypair.pk,
+                    E::default_spec().bls_withdrawal_prefix_byte
+                )),
+                "the locally generated withdrawal creds should match the deposit data"
+            );
+
+            assert_eq!(
+                deposit_data.signature,
+                deposit_data.create_signature(&voting_keypair.sk, &E::default_spec()),
+                "the locally-generated deposit sig should create the same deposit sig"
+            );
+        }
+
+        self
+    }
+
+    pub async fn create_keystore_validators(self, s: KeystoreValidatorScenario) -> Self {
+        let initial_vals = self.vals_total();
+        let initial_enabled_vals = self.vals_enabled();
+
+        let password = random_password();
+        let keypair = Keypair::random();
+        let keystore = KeystoreBuilder::new(&keypair, password.as_bytes(), String::new())
+            .unwrap()
+            .build()
+            .unwrap();
+
+        if !s.correct_password {
+            let request = KeystoreValidatorsPostRequest {
+                enable: s.enabled,
+                password: String::from_utf8(random_password().as_ref().to_vec())
+                    .unwrap()
+                    .into(),
+                keystore,
+                graffiti: None,
+                suggested_fee_recipient: None,
+                gas_limit: None,
+                builder_proposals: None,
+            };
+
+            self.client
+                .post_lighthouse_validators_keystore(&request)
+                .await
+                .unwrap_err();
+
+            return self;
+        }
+
+        let request = KeystoreValidatorsPostRequest {
+            enable: s.enabled,
+            password: String::from_utf8(password.as_ref().to_vec())
+                .unwrap()
+                .into(),
+            keystore,
+            graffiti: None,
+            suggested_fee_recipient: None,
+            gas_limit: None,
+            builder_proposals: None,
+        };
+
+        let response = self
+            .client
+            .post_lighthouse_validators_keystore(&request)
+            .await
+            .unwrap()
+            .data;
+
+        let num_enabled = s.enabled as usize;
+
+        assert_eq!(self.vals_total(), initial_vals + 1);
+        assert_eq!(self.vals_enabled(), initial_enabled_vals + num_enabled);
+
+        let server_vals = self.client.get_lighthouse_validators().await.unwrap().data;
+
+        assert_eq!(server_vals.len(), self.vals_total());
+
+        assert_eq!(response.voting_pubkey, keypair.pk.into());
+        assert_eq!(response.enabled, s.enabled);
+
+        self
+    }
+
+    pub async fn create_web3signer_validators(self, s: Web3SignerValidatorScenario) -> Self {
+        let initial_vals = self.vals_total();
+        let initial_enabled_vals = self.vals_enabled();
+
+        let request: Vec<_> = (0..s.count)
+            .map(|i| {
+                let kp = Keypair::random();
+                Web3SignerValidatorRequest {
+                    enable: s.enabled,
+                    description: format!("{}", i),
+                    graffiti: None,
+                    suggested_fee_recipient: None,
+                    gas_limit: None,
+                    builder_proposals: None,
+                    voting_public_key: kp.pk,
+                    url: format!("http://signer_{}.com/", i),
+                    root_certificate_path: None,
+                    request_timeout_ms: None,
+                    client_identity_path: None,
+                    client_identity_password: None,
+                }
+            })
+            .collect();
+
+        self.client
+            .post_lighthouse_validators_web3signer(&request)
+            .await
+            .unwrap();
+
+        assert_eq!(self.vals_total(), initial_vals + s.count);
+        if s.enabled {
+            assert_eq!(self.vals_enabled(), initial_enabled_vals + s.count);
+        } else {
+            assert_eq!(self.vals_enabled(), initial_enabled_vals);
+        };
+
+        self
+    }
+
+    pub async fn set_validator_enabled(self, index: usize, enabled: bool) -> Self {
+        let validator = &self.client.get_lighthouse_validators().await.unwrap().data[index];
+
+        self.client
+            .patch_lighthouse_validators(&validator.voting_pubkey, Some(enabled), None, None, None)
+            .await
+            .unwrap();
+
+        assert_eq!(
+            self.initialized_validators
+                .read()
+                .is_enabled(&validator.voting_pubkey.decompress().unwrap())
+                .unwrap(),
+            enabled
+        );
+
+        assert!(self
+            .client
+            .get_lighthouse_validators()
+            .await
+            .unwrap()
+            .data
+            .into_iter()
+            .find(|v| v.voting_pubkey == validator.voting_pubkey)
+            .map(|v| v.enabled == enabled)
+            .unwrap());
+
+        // Check the server via an individual request.
+        assert_eq!(
+            self.client
+                .get_lighthouse_validators_pubkey(&validator.voting_pubkey)
+                .await
+                .unwrap()
+                .unwrap()
+                .data
+                .enabled,
+            enabled
+        );
+
+        self
+    }
+
+    pub async fn set_gas_limit(self, index: usize, gas_limit: u64) -> Self {
+        let validator = &self.client.get_lighthouse_validators().await.unwrap().data[index];
+
+        self.client
+            .patch_lighthouse_validators(
+                &validator.voting_pubkey,
+                None,
+                Some(gas_limit),
+                None,
+                None,
+            )
+            .await
+            .unwrap();
+
+        self
+    }
+
+    pub async fn assert_gas_limit(self, index: usize, gas_limit: u64) -> Self {
+        let validator = &self.client.get_lighthouse_validators().await.unwrap().data[index];
+
+        assert_eq!(
+            self.validator_store.get_gas_limit(&validator.voting_pubkey),
+            gas_limit
+        );
+
+        self
+    }
+
+    pub async fn set_builder_proposals(self, index: usize, builder_proposals: bool) -> Self {
+        let validator = &self.client.get_lighthouse_validators().await.unwrap().data[index];
+
+        self.client
+            .patch_lighthouse_validators(
+                &validator.voting_pubkey,
+                None,
+                None,
+                Some(builder_proposals),
+                None,
+            )
+            .await
+            .unwrap();
+
+        self
+    }
+
+    pub async fn assert_builder_proposals(self, index: usize, builder_proposals: bool) -> Self {
+        let validator = &self.client.get_lighthouse_validators().await.unwrap().data[index];
+
+        assert_eq!(
+            self.validator_store
+                .get_builder_proposals(&validator.voting_pubkey),
+            builder_proposals
+        );
+
+        self
+    }
+}

validator_client/src/http_api/tests.rs

@@ -31,10 +31,8 @@ use std::net::{IpAddr, Ipv4Addr};
 use std::str::FromStr;
 use std::sync::Arc;
 use std::time::Duration;
-use task_executor::TaskExecutor;
+use task_executor::test_utils::TestRuntime;
 use tempfile::{tempdir, TempDir};
-use tokio::runtime::Runtime;
-use tokio::sync::oneshot;
 use types::graffiti::GraffitiString;
 
 const PASSWORD_BYTES: &[u8] = &[42, 50, 37];
@@ -48,23 +46,12 @@ struct ApiTester {
     validator_store: Arc<ValidatorStore<TestingSlotClock, E>>,
     url: SensitiveUrl,
     slot_clock: TestingSlotClock,
-    _server_shutdown: oneshot::Sender<()>,
     _validator_dir: TempDir,
-    _runtime_shutdown: exit_future::Signal,
+    _test_runtime: TestRuntime,
-}
-
-// Builds a runtime to be used in the testing configuration.
-fn build_runtime() -> Arc<Runtime> {
-    Arc::new(
-        tokio::runtime::Builder::new_multi_thread()
-            .enable_all()
-            .build()
-            .expect("Should be able to build a testing runtime"),
-    )
 }
 
 impl ApiTester {
-    pub async fn new(runtime: std::sync::Weak<Runtime>) -> Self {
+    pub async fn new() -> Self {
         let log = test_logger();
 
         let validator_dir = tempdir().unwrap();
@@ -100,9 +87,7 @@ impl ApiTester {
             Duration::from_secs(1),
         );
 
-        let (runtime_shutdown, exit) = exit_future::signal();
+        let test_runtime = TestRuntime::default();
-        let (shutdown_tx, _) = futures::channel::mpsc::channel(1);
-        let executor = TaskExecutor::new(runtime.clone(), exit, log.clone(), shutdown_tx);
 
         let validator_store = Arc::new(ValidatorStore::<_, E>::new(
             initialized_validators,
@@ -112,7 +97,7 @@ impl ApiTester {
             Some(Arc::new(DoppelgangerService::new(log.clone()))),
             slot_clock.clone(),
             &config,
-            executor.clone(),
+            test_runtime.task_executor.clone(),
             log.clone(),
         ));
 
@@ -123,9 +108,10 @@ impl ApiTester {
         let initialized_validators = validator_store.initialized_validators();
 
         let context = Arc::new(Context {
-            task_executor: executor,
+            task_executor: test_runtime.task_executor.clone(),
             api_secret,
             validator_dir: Some(validator_dir.path().into()),
+            secrets_dir: Some(secrets_dir.path().into()),
             validator_store: Some(validator_store.clone()),
             graffiti_file: None,
             graffiti_flag: Some(Graffiti::default()),
@@ -135,6 +121,8 @@ impl ApiTester {
                 listen_addr: IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)),
                 listen_port: 0,
                 allow_origin: None,
+                allow_keystore_export: true,
+                store_passwords_in_secrets_dir: false,
             },
             sse_logging_components: None,
             log,
@@ -142,12 +130,8 @@ impl ApiTester {
             _phantom: PhantomData,
         });
         let ctx = context.clone();
-        let (shutdown_tx, shutdown_rx) = oneshot::channel();
+        let (listening_socket, server) =
-        let server_shutdown = async {
+            super::serve(ctx, test_runtime.task_executor.exit()).unwrap();
-            // It's not really interesting why this triggered, just that it happened.
-            let _ = shutdown_rx.await;
-        };
-        let (listening_socket, server) = super::serve(ctx, server_shutdown).unwrap();
 
         tokio::spawn(async { server.await });
 
@@ -166,9 +150,8 @@ impl ApiTester {
             validator_store,
             url,
             slot_clock,
-            _server_shutdown: shutdown_tx,
             _validator_dir: validator_dir,
-            _runtime_shutdown: runtime_shutdown,
+            _test_runtime: test_runtime,
         }
     }
 
@@ -676,387 +659,341 @@ struct Web3SignerValidatorScenario {
     enabled: bool,
 }
 
-#[test]
+#[tokio::test]
-fn invalid_pubkey() {
+async fn invalid_pubkey() {
-    let runtime = build_runtime();
+    ApiTester::new()
-    let weak_runtime = Arc::downgrade(&runtime);
+        .await
-    runtime.block_on(async {
+        .invalidate_api_token()
-        ApiTester::new(weak_runtime)
+        .test_get_lighthouse_version_invalid()
-            .await
+        .await;
-            .invalidate_api_token()
-            .test_get_lighthouse_version_invalid()
-            .await;
-    });
 }
 
-#[test]
+#[tokio::test]
-fn routes_with_invalid_auth() {
+async fn routes_with_invalid_auth() {
-    let runtime = build_runtime();
+    ApiTester::new()
-    let weak_runtime = Arc::downgrade(&runtime);
+        .await
-    runtime.block_on(async {
+        .test_with_invalid_auth(|client| async move { client.get_lighthouse_version().await })
-        ApiTester::new(weak_runtime)
+        .await
-            .await
+        .test_with_invalid_auth(|client| async move { client.get_lighthouse_health().await })
-            .test_with_invalid_auth(|client| async move { client.get_lighthouse_version().await })
+        .await
-            .await
+        .test_with_invalid_auth(|client| async move {
-            .test_with_invalid_auth(|client| async move { client.get_lighthouse_health().await })
+            client.get_lighthouse_spec::<types::Config>().await
-            .await
+        })
-            .test_with_invalid_auth(|client| async move {
+        .await
-                client.get_lighthouse_spec::<types::Config>().await
+        .test_with_invalid_auth(|client| async move { client.get_lighthouse_validators().await })
-            })
+        .await
-            .await
+        .test_with_invalid_auth(|client| async move {
-            .test_with_invalid_auth(
+            client
-                |client| async move { client.get_lighthouse_validators().await },
+                .get_lighthouse_validators_pubkey(&PublicKeyBytes::empty())
-            )
+                .await
-            .await
+        })
-            .test_with_invalid_auth(|client| async move {
+        .await
-                client
+        .test_with_invalid_auth(|client| async move {
-                    .get_lighthouse_validators_pubkey(&PublicKeyBytes::empty())
+            client
-                    .await
+                .post_lighthouse_validators(vec![ValidatorRequest {
-            })
+                    enable: <_>::default(),
-            .await
+                    description: <_>::default(),
-            .test_with_invalid_auth(|client| async move {
+                    graffiti: <_>::default(),
-                client
+                    suggested_fee_recipient: <_>::default(),
-                    .post_lighthouse_validators(vec![ValidatorRequest {
+                    gas_limit: <_>::default(),
-                        enable: <_>::default(),
+                    builder_proposals: <_>::default(),
-                        description: <_>::default(),
+                    deposit_gwei: <_>::default(),
-                        graffiti: <_>::default(),
+                }])
-                        suggested_fee_recipient: <_>::default(),
+                .await
-                        gas_limit: <_>::default(),
+        })
-                        builder_proposals: <_>::default(),
+        .await
-                        deposit_gwei: <_>::default(),
+        .test_with_invalid_auth(|client| async move {
-                    }])
+            client
-                    .await
+                .post_lighthouse_validators_mnemonic(&CreateValidatorsMnemonicRequest {
-            })
+                    mnemonic: String::default().into(),
-            .await
+                    key_derivation_path_offset: <_>::default(),
-            .test_with_invalid_auth(|client| async move {
+                    validators: <_>::default(),
-                client
+                })
-                    .post_lighthouse_validators_mnemonic(&CreateValidatorsMnemonicRequest {
+                .await
-                        mnemonic: String::default().into(),
+        })
-                        key_derivation_path_offset: <_>::default(),
+        .await
-                        validators: <_>::default(),
+        .test_with_invalid_auth(|client| async move {
-                    })
+            let password = random_password();
-                    .await
+            let keypair = Keypair::random();
-            })
+            let keystore = KeystoreBuilder::new(&keypair, password.as_bytes(), String::new())
-            .await
+                .unwrap()
-            .test_with_invalid_auth(|client| async move {
+                .build()
-                let password = random_password();
+                .unwrap();
-                let keypair = Keypair::random();
+            client
-                let keystore = KeystoreBuilder::new(&keypair, password.as_bytes(), String::new())
+                .post_lighthouse_validators_keystore(&KeystoreValidatorsPostRequest {
-                    .unwrap()
+                    password: String::default().into(),
-                    .build()
+                    enable: <_>::default(),
-                    .unwrap();
+                    keystore,
-                client
+                    graffiti: <_>::default(),
-                    .post_lighthouse_validators_keystore(&KeystoreValidatorsPostRequest {
+                    suggested_fee_recipient: <_>::default(),
-                        password: String::default().into(),
+                    gas_limit: <_>::default(),
-                        enable: <_>::default(),
+                    builder_proposals: <_>::default(),
-                        keystore,
+                })
-                        graffiti: <_>::default(),
+                .await
-                        suggested_fee_recipient: <_>::default(),
+        })
-                        gas_limit: <_>::default(),
+        .await
-                        builder_proposals: <_>::default(),
+        .test_with_invalid_auth(|client| async move {
-                    })
+            client
-                    .await
+                .patch_lighthouse_validators(
-            })
+                    &PublicKeyBytes::empty(),
-            .await
+                    Some(false),
-            .test_with_invalid_auth(|client| async move {
+                    None,
-                client
+                    None,
-                    .patch_lighthouse_validators(
+                    None,
-                        &PublicKeyBytes::empty(),
+                )
-                        Some(false),
+                .await
-                        None,
+        })
-                        None,
+        .await
-                        None,
+        .test_with_invalid_auth(|client| async move { client.get_keystores().await })
-                    )
+        .await
-                    .await
+        .test_with_invalid_auth(|client| async move {
-            })
+            let password = random_password_string();
-            .await
+            let keypair = Keypair::random();
-            .test_with_invalid_auth(|client| async move { client.get_keystores().await })
+            let keystore = KeystoreBuilder::new(&keypair, password.as_ref(), String::new())
-            .await
+                .unwrap()
-            .test_with_invalid_auth(|client| async move {
+                .build()
-                let password = random_password_string();
+                .map(KeystoreJsonStr)
-                let keypair = Keypair::random();
+                .unwrap();
-                let keystore = KeystoreBuilder::new(&keypair, password.as_ref(), String::new())
+            client
-                    .unwrap()
+                .post_keystores(&ImportKeystoresRequest {
-                    .build()
+                    keystores: vec![keystore],
-                    .map(KeystoreJsonStr)
+                    passwords: vec![password],
-                    .unwrap();
+                    slashing_protection: None,
-                client
+                })
-                    .post_keystores(&ImportKeystoresRequest {
+                .await
-                        keystores: vec![keystore],
+        })
-                        passwords: vec![password],
+        .await
-                        slashing_protection: None,
+        .test_with_invalid_auth(|client| async move {
-                    })
+            let keypair = Keypair::random();
-                    .await
+            client
-            })
+                .delete_keystores(&DeleteKeystoresRequest {
-            .await
+                    pubkeys: vec![keypair.pk.compress()],
-            .test_with_invalid_auth(|client| async move {
+                })
-                let keypair = Keypair::random();
+                .await
-                client
+        })
-                    .delete_keystores(&DeleteKeystoresRequest {
+        .await;
-                        pubkeys: vec![keypair.pk.compress()],
-                    })
-                    .await
-            })
-            .await
-    });
 }
 
-#[test]
+#[tokio::test]
-fn simple_getters() {
+async fn simple_getters() {
-    let runtime = build_runtime();
+    ApiTester::new()
-    let weak_runtime = Arc::downgrade(&runtime);
+        .await
-    runtime.block_on(async {
+        .test_get_lighthouse_version()
-        ApiTester::new(weak_runtime)
+        .await
-            .await
+        .test_get_lighthouse_health()
-            .test_get_lighthouse_version()
+        .await
-            .await
+        .test_get_lighthouse_spec()
-            .test_get_lighthouse_health()
+        .await;
-            .await
-            .test_get_lighthouse_spec()
-            .await;
-    });
 }
 
-#[test]
+#[tokio::test]
-fn hd_validator_creation() {
+async fn hd_validator_creation() {
-    let runtime = build_runtime();
+    ApiTester::new()
-    let weak_runtime = Arc::downgrade(&runtime);
+        .await
-    runtime.block_on(async {
+        .assert_enabled_validators_count(0)
-        ApiTester::new(weak_runtime)
+        .assert_validators_count(0)
-            .await
+        .create_hd_validators(HdValidatorScenario {
-            .assert_enabled_validators_count(0)
+            count: 2,
-            .assert_validators_count(0)
+            specify_mnemonic: true,
-            .create_hd_validators(HdValidatorScenario {
+            key_derivation_path_offset: 0,
-                count: 2,
+            disabled: vec![],
-                specify_mnemonic: true,
+        })
-                key_derivation_path_offset: 0,
+        .await
-                disabled: vec![],
+        .assert_enabled_validators_count(2)
-            })
+        .assert_validators_count(2)
-            .await
+        .create_hd_validators(HdValidatorScenario {
-            .assert_enabled_validators_count(2)
+            count: 1,
-            .assert_validators_count(2)
+            specify_mnemonic: false,
-            .create_hd_validators(HdValidatorScenario {
+            key_derivation_path_offset: 0,
-                count: 1,
+            disabled: vec![0],
-                specify_mnemonic: false,
+        })
-                key_derivation_path_offset: 0,
+        .await
-                disabled: vec![0],
+        .assert_enabled_validators_count(2)
-            })
+        .assert_validators_count(3)
-            .await
+        .create_hd_validators(HdValidatorScenario {
-            .assert_enabled_validators_count(2)
+            count: 0,
-            .assert_validators_count(3)
+            specify_mnemonic: true,
-            .create_hd_validators(HdValidatorScenario {
+            key_derivation_path_offset: 4,
-                count: 0,
+            disabled: vec![],
-                specify_mnemonic: true,
+        })
-                key_derivation_path_offset: 4,
+        .await
-                disabled: vec![],
+        .assert_enabled_validators_count(2)
-            })
+        .assert_validators_count(3);
-            .await
-            .assert_enabled_validators_count(2)
-            .assert_validators_count(3);
-    });
 }
 
-#[test]
+#[tokio::test]
-fn validator_exit() {
+async fn validator_exit() {
-    let runtime = build_runtime();
+    ApiTester::new()
-    let weak_runtime = Arc::downgrade(&runtime);
+        .await
-    runtime.block_on(async {
+        .create_hd_validators(HdValidatorScenario {
-        ApiTester::new(weak_runtime)
+            count: 2,
-            .await
+            specify_mnemonic: false,
-            .create_hd_validators(HdValidatorScenario {
+            key_derivation_path_offset: 0,
-                count: 2,
+            disabled: vec![],
-                specify_mnemonic: false,
+        })
-                key_derivation_path_offset: 0,
+        .await
-                disabled: vec![],
+        .assert_enabled_validators_count(2)
-            })
+        .assert_validators_count(2)
-            .await
+        .test_sign_voluntary_exits(0, None)
-            .assert_enabled_validators_count(2)
+        .await
-            .assert_validators_count(2)
+        .test_sign_voluntary_exits(0, Some(Epoch::new(256)))
-            .test_sign_voluntary_exits(0, None)
+        .await;
-            .await
-            .test_sign_voluntary_exits(0, Some(Epoch::new(256)))
-            .await;
-    });
 }
 
-#[test]
+#[tokio::test]
-fn validator_enabling() {
+async fn validator_enabling() {
-    let runtime = build_runtime();
+    ApiTester::new()
-    let weak_runtime = Arc::downgrade(&runtime);
+        .await
-    runtime.block_on(async {
+        .create_hd_validators(HdValidatorScenario {
-        ApiTester::new(weak_runtime)
+            count: 2,
-            .await
+            specify_mnemonic: false,
-            .create_hd_validators(HdValidatorScenario {
+            key_derivation_path_offset: 0,
-                count: 2,
+            disabled: vec![],
-                specify_mnemonic: false,
+        })
-                key_derivation_path_offset: 0,
+        .await
-                disabled: vec![],
+        .assert_enabled_validators_count(2)
-            })
+        .assert_validators_count(2)
-            .await
+        .set_validator_enabled(0, false)
-            .assert_enabled_validators_count(2)
+        .await
-            .assert_validators_count(2)
+        .assert_enabled_validators_count(1)
-            .set_validator_enabled(0, false)
+        .assert_validators_count(2)
-            .await
+        .set_validator_enabled(0, true)
-            .assert_enabled_validators_count(1)
+        .await
-            .assert_validators_count(2)
+        .assert_enabled_validators_count(2)
-            .set_validator_enabled(0, true)
+        .assert_validators_count(2);
-            .await
-            .assert_enabled_validators_count(2)
-            .assert_validators_count(2);
-    });
 }
 
-#[test]
+#[tokio::test]
-fn validator_gas_limit() {
+async fn validator_gas_limit() {
-    let runtime = build_runtime();
+    ApiTester::new()
-    let weak_runtime = Arc::downgrade(&runtime);
+        .await
-    runtime.block_on(async {
+        .create_hd_validators(HdValidatorScenario {
-        ApiTester::new(weak_runtime)
+            count: 2,
-            .await
+            specify_mnemonic: false,
-            .create_hd_validators(HdValidatorScenario {
+            key_derivation_path_offset: 0,
-                count: 2,
+            disabled: vec![],
-                specify_mnemonic: false,
+        })
-                key_derivation_path_offset: 0,
+        .await
-                disabled: vec![],
+        .assert_enabled_validators_count(2)
-            })
+        .assert_validators_count(2)
-            .await
+        .set_gas_limit(0, 500)
-            .assert_enabled_validators_count(2)
+        .await
-            .assert_validators_count(2)
+        .assert_gas_limit(0, 500)
-            .set_gas_limit(0, 500)
+        .await
-            .await
+        // Update gas limit while validator is disabled.
-            .assert_gas_limit(0, 500)
+        .set_validator_enabled(0, false)
-            .await
+        .await
-            // Update gas limit while validator is disabled.
+        .assert_enabled_validators_count(1)
-            .set_validator_enabled(0, false)
+        .assert_validators_count(2)
-            .await
+        .set_gas_limit(0, 1000)
-            .assert_enabled_validators_count(1)
+        .await
-            .assert_validators_count(2)
+        .set_validator_enabled(0, true)
-            .set_gas_limit(0, 1000)
+        .await
-            .await
+        .assert_enabled_validators_count(2)
-            .set_validator_enabled(0, true)
+        .assert_gas_limit(0, 1000)
-            .await
+        .await;
-            .assert_enabled_validators_count(2)
-            .assert_gas_limit(0, 1000)
-            .await
-    });
 }
 
-#[test]
+#[tokio::test]
-fn validator_builder_proposals() {
+async fn validator_builder_proposals() {
-    let runtime = build_runtime();
+    ApiTester::new()
-    let weak_runtime = Arc::downgrade(&runtime);
+        .await
-    runtime.block_on(async {
+        .create_hd_validators(HdValidatorScenario {
-        ApiTester::new(weak_runtime)
+            count: 2,
-            .await
+            specify_mnemonic: false,
-            .create_hd_validators(HdValidatorScenario {
+            key_derivation_path_offset: 0,
-                count: 2,
+            disabled: vec![],
-                specify_mnemonic: false,
+        })
-                key_derivation_path_offset: 0,
+        .await
-                disabled: vec![],
+        .assert_enabled_validators_count(2)
-            })
+        .assert_validators_count(2)
-            .await
+        .set_builder_proposals(0, true)
-            .assert_enabled_validators_count(2)
+        .await
-            .assert_validators_count(2)
+        // Test setting builder proposals while the validator is disabled
-            .set_builder_proposals(0, true)
+        .set_validator_enabled(0, false)
-            .await
+        .await
-            // Test setting builder proposals while the validator is disabled
+        .assert_enabled_validators_count(1)
-            .set_validator_enabled(0, false)
+        .assert_validators_count(2)
-            .await
+        .set_builder_proposals(0, false)
-            .assert_enabled_validators_count(1)
+        .await
-            .assert_validators_count(2)
+        .set_validator_enabled(0, true)
-            .set_builder_proposals(0, false)
+        .await
-            .await
+        .assert_enabled_validators_count(2)
-            .set_validator_enabled(0, true)
+        .assert_builder_proposals(0, false)
-            .await
+        .await;
-            .assert_enabled_validators_count(2)
-            .assert_builder_proposals(0, false)
-            .await
-    });
 }
 
-#[test]
+#[tokio::test]
-fn validator_graffiti() {
+async fn validator_graffiti() {
-    let runtime = build_runtime();
+    ApiTester::new()
-    let weak_runtime = Arc::downgrade(&runtime);
+        .await
-    runtime.block_on(async {
+        .create_hd_validators(HdValidatorScenario {
-        ApiTester::new(weak_runtime)
+            count: 2,
-            .await
+            specify_mnemonic: false,
-            .create_hd_validators(HdValidatorScenario {
+            key_derivation_path_offset: 0,
-                count: 2,
+            disabled: vec![],
-                specify_mnemonic: false,
+        })
-                key_derivation_path_offset: 0,
+        .await
-                disabled: vec![],
+        .assert_enabled_validators_count(2)
-            })
+        .assert_validators_count(2)
-            .await
+        .set_graffiti(0, "Mr F was here")
-            .assert_enabled_validators_count(2)
+        .await
-            .assert_validators_count(2)
+        .assert_graffiti(0, "Mr F was here")
-            .set_graffiti(0, "Mr F was here")
+        .await
-            .await
+        // Test setting graffiti while the validator is disabled
-            .assert_graffiti(0, "Mr F was here")
+        .set_validator_enabled(0, false)
-            .await
+        .await
-            // Test setting graffiti while the validator is disabled
+        .assert_enabled_validators_count(1)
-            .set_validator_enabled(0, false)
+        .assert_validators_count(2)
-            .await
+        .set_graffiti(0, "Mr F was here again")
-            .assert_enabled_validators_count(1)
+        .await
-            .assert_validators_count(2)
+        .set_validator_enabled(0, true)
-            .set_graffiti(0, "Mr F was here again")
+        .await
-            .await
+        .assert_enabled_validators_count(2)
-            .set_validator_enabled(0, true)
+        .assert_graffiti(0, "Mr F was here again")
-            .await
+        .await;
-            .assert_enabled_validators_count(2)
-            .assert_graffiti(0, "Mr F was here again")
-            .await
-    });
 }
 
-#[test]
+#[tokio::test]
-fn keystore_validator_creation() {
+async fn keystore_validator_creation() {
-    let runtime = build_runtime();
+    ApiTester::new()
-    let weak_runtime = Arc::downgrade(&runtime);
+        .await
-    runtime.block_on(async {
+        .assert_enabled_validators_count(0)
-        ApiTester::new(weak_runtime)
+        .assert_validators_count(0)
-            .await
+        .create_keystore_validators(KeystoreValidatorScenario {
-            .assert_enabled_validators_count(0)
+            correct_password: true,
-            .assert_validators_count(0)
+            enabled: true,
-            .create_keystore_validators(KeystoreValidatorScenario {
+        })
-                correct_password: true,
+        .await
-                enabled: true,
+        .assert_enabled_validators_count(1)
-            })
+        .assert_validators_count(1)
-            .await
+        .create_keystore_validators(KeystoreValidatorScenario {
-            .assert_enabled_validators_count(1)
+            correct_password: false,
-            .assert_validators_count(1)
+            enabled: true,
-            .create_keystore_validators(KeystoreValidatorScenario {
+        })
-                correct_password: false,
+        .await
-                enabled: true,
+        .assert_enabled_validators_count(1)
-            })
+        .assert_validators_count(1)
-            .await
+        .create_keystore_validators(KeystoreValidatorScenario {
-            .assert_enabled_validators_count(1)
+            correct_password: true,
-            .assert_validators_count(1)
+            enabled: false,
-            .create_keystore_validators(KeystoreValidatorScenario {
+        })
-                correct_password: true,
+        .await
-                enabled: false,
+        .assert_enabled_validators_count(1)
-            })
+        .assert_validators_count(2);
-            .await
-            .assert_enabled_validators_count(1)
-            .assert_validators_count(2);
-    });
 }
 
-#[test]
+#[tokio::test]
-fn web3signer_validator_creation() {
+async fn web3signer_validator_creation() {
-    let runtime = build_runtime();
+    ApiTester::new()
-    let weak_runtime = Arc::downgrade(&runtime);
+        .await
-    runtime.block_on(async {
+        .assert_enabled_validators_count(0)
-        ApiTester::new(weak_runtime)
+        .assert_validators_count(0)
-            .await
+        .create_web3signer_validators(Web3SignerValidatorScenario {
-            .assert_enabled_validators_count(0)
+            count: 1,
-            .assert_validators_count(0)
+            enabled: true,
-            .create_web3signer_validators(Web3SignerValidatorScenario {
+        })
-                count: 1,
+        .await
-                enabled: true,
+        .assert_enabled_validators_count(1)
-            })
+        .assert_validators_count(1);
-            .await
-            .assert_enabled_validators_count(1)
-            .assert_validators_count(1);
-    });
 }

validator_client/src/http_api/tests/keystores.rs

@@ -12,6 +12,7 @@ use itertools::Itertools;
 use rand::{rngs::SmallRng, Rng, SeedableRng};
 use slashing_protection::interchange::{Interchange, InterchangeMetadata};
 use std::{collections::HashMap, path::Path};
+use tokio::runtime::Handle;
 use types::Address;
 
 fn new_keystore(password: ZeroizeString) -> Keystore {
@@ -64,31 +65,23 @@ fn remotekey_validator_with_pubkey(pubkey: PublicKey) -> SingleImportRemotekeysR
     }
 }
 
-fn run_test<F, V>(f: F)
+async fn run_test<F, V>(f: F)
 where
     F: FnOnce(ApiTester) -> V,
     V: Future<Output = ()>,
 {
-    let runtime = build_runtime();
+    let tester = ApiTester::new().await;
-    let weak_runtime = Arc::downgrade(&runtime);
+    f(tester).await
-    runtime.block_on(async {
-        let tester = ApiTester::new(weak_runtime).await;
-        f(tester).await
-    });
 }
 
-fn run_dual_vc_test<F, V>(f: F)
+async fn run_dual_vc_test<F, V>(f: F)
 where
     F: FnOnce(ApiTester, ApiTester) -> V,
     V: Future<Output = ()>,
 {
-    let runtime = build_runtime();
+    let tester1 = ApiTester::new().await;
-    let weak_runtime = Arc::downgrade(&runtime);
+    let tester2 = ApiTester::new().await;
-    runtime.block_on(async {
+    f(tester1, tester2).await
-        let tester1 = ApiTester::new(weak_runtime.clone()).await;
-        let tester2 = ApiTester::new(weak_runtime).await;
-        f(tester1, tester2).await
-    });
 }
 
 fn keystore_pubkey(keystore: &Keystore) -> PublicKeyBytes {
@@ -199,8 +192,8 @@ fn check_remotekey_delete_response(
     }
 }
 
-#[test]
+#[tokio::test]
-fn get_auth_no_token() {
+async fn get_auth_no_token() {
     run_test(|mut tester| async move {
         let _ = &tester;
         tester.client.send_authorization_header(false);
@@ -213,19 +206,21 @@ fn get_auth_no_token() {
         // The token should match the one that the client was originally initialised with.
         assert!(tester.client.api_token() == Some(&token));
     })
+    .await;
 }
 
-#[test]
+#[tokio::test]
-fn get_empty_keystores() {
+async fn get_empty_keystores() {
     run_test(|tester| async move {
         let _ = &tester;
         let res = tester.client.get_keystores().await.unwrap();
         assert_eq!(res, ListKeystoresResponse { data: vec![] });
     })
+    .await;
 }
 
-#[test]
+#[tokio::test]
-fn import_new_keystores() {
+async fn import_new_keystores() {
     run_test(|tester| async move {
         let _ = &tester;
         let password = random_password_string();
@@ -250,10 +245,11 @@ fn import_new_keystores() {
         let get_res = tester.client.get_keystores().await.unwrap();
         check_keystore_get_response(&get_res, &keystores);
     })
+    .await;
 }
 
-#[test]
+#[tokio::test]
-fn import_only_duplicate_keystores() {
+async fn import_only_duplicate_keystores() {
     run_test(|tester| async move {
         let _ = &tester;
         let password = random_password_string();
@@ -279,10 +275,11 @@ fn import_only_duplicate_keystores() {
         let get_res = tester.client.get_keystores().await.unwrap();
         check_keystore_get_response(&get_res, &keystores);
     })
+    .await;
 }
 
-#[test]
+#[tokio::test]
-fn import_some_duplicate_keystores() {
+async fn import_some_duplicate_keystores() {
     run_test(|tester| async move {
         let _ = &tester;
         let password = random_password_string();
@@ -330,10 +327,11 @@ fn import_some_duplicate_keystores() {
         let import_res = tester.client.post_keystores(&req2).await.unwrap();
         check_keystore_import_response(&import_res, expected);
     })
+    .await;
 }
 
-#[test]
+#[tokio::test]
-fn import_wrong_number_of_passwords() {
+async fn import_wrong_number_of_passwords() {
     run_test(|tester| async move {
         let _ = &tester;
         let password = random_password_string();
@@ -352,10 +350,11 @@ fn import_wrong_number_of_passwords() {
             .unwrap_err();
         assert_eq!(err.status().unwrap(), 400);
     })
+    .await;
 }
 
-#[test]
+#[tokio::test]
-fn get_web3_signer_keystores() {
+async fn get_web3_signer_keystores() {
     run_test(|tester| async move {
         let _ = &tester;
         let num_local = 3;
@@ -412,10 +411,11 @@ fn get_web3_signer_keystores() {
             assert!(get_res.data.contains(&response), "{:?}", response);
         }
     })
+    .await;
 }
 
-#[test]
+#[tokio::test]
-fn import_and_delete_conflicting_web3_signer_keystores() {
+async fn import_and_delete_conflicting_web3_signer_keystores() {
     run_test(|tester| async move {
         let _ = &tester;
         let num_keystores = 3;
@@ -477,10 +477,11 @@ fn import_and_delete_conflicting_web3_signer_keystores() {
         let delete_res = tester.client.delete_keystores(&delete_req).await.unwrap();
         check_keystore_delete_response(&delete_res, all_delete_error(keystores.len()));
     })
+    .await;
 }
 
-#[test]
+#[tokio::test]
-fn import_keystores_wrong_password() {
+async fn import_keystores_wrong_password() {
     run_test(|tester| async move {
         let _ = &tester;
         let num_keystores = 4;
@@ -551,11 +552,12 @@ fn import_keystores_wrong_password() {
             &import_res,
             (0..num_keystores).map(|_| ImportKeystoreStatus::Duplicate),
         );
-    });
+    })
+    .await;
 }
 
-#[test]
+#[tokio::test]
-fn import_invalid_slashing_protection() {
+async fn import_invalid_slashing_protection() {
     run_test(|tester| async move {
         let _ = &tester;
         let password = random_password_string();
@@ -589,10 +591,11 @@ fn import_invalid_slashing_protection() {
         let get_res = tester.client.get_keystores().await.unwrap();
         check_keystore_get_response(&get_res, &[]);
     })
+    .await;
 }
 
-#[test]
+#[tokio::test]
-fn check_get_set_fee_recipient() {
+async fn check_get_set_fee_recipient() {
     run_test(|tester: ApiTester| async move {
         let _ = &tester;
         let password = random_password_string();
@@ -768,10 +771,11 @@ fn check_get_set_fee_recipient() {
             );
         }
     })
+    .await;
 }
 
-#[test]
+#[tokio::test]
-fn check_get_set_gas_limit() {
+async fn check_get_set_gas_limit() {
     run_test(|tester: ApiTester| async move {
         let _ = &tester;
         let password = random_password_string();
@@ -943,14 +947,15 @@ fn check_get_set_gas_limit() {
             );
         }
     })
+    .await
 }
 
 fn all_indices(count: usize) -> Vec<usize> {
     (0..count).collect()
 }
 
-#[test]
+#[tokio::test]
-fn migrate_all_with_slashing_protection() {
+async fn migrate_all_with_slashing_protection() {
     let n = 3;
     generic_migration_test(
         n,
@@ -967,11 +972,12 @@ fn migrate_all_with_slashing_protection() {
             (1, make_attestation(2, 3), false),
             (2, make_attestation(1, 2), false),
         ],
-    );
+    )
+    .await;
 }
 
-#[test]
+#[tokio::test]
-fn migrate_some_with_slashing_protection() {
+async fn migrate_some_with_slashing_protection() {
     let n = 3;
     generic_migration_test(
         n,
@@ -989,11 +995,12 @@ fn migrate_some_with_slashing_protection() {
             (0, make_attestation(2, 3), true),
             (1, make_attestation(3, 4), true),
         ],
-    );
+    )
+    .await;
 }
 
-#[test]
+#[tokio::test]
-fn migrate_some_missing_slashing_protection() {
+async fn migrate_some_missing_slashing_protection() {
     let n = 3;
     generic_migration_test(
         n,
@@ -1010,11 +1017,12 @@ fn migrate_some_missing_slashing_protection() {
             (1, make_attestation(2, 3), true),
             (0, make_attestation(2, 3), true),
         ],
-    );
+    )
+    .await;
 }
 
-#[test]
+#[tokio::test]
-fn migrate_some_extra_slashing_protection() {
+async fn migrate_some_extra_slashing_protection() {
     let n = 3;
     generic_migration_test(
         n,
@@ -1033,7 +1041,8 @@ fn migrate_some_extra_slashing_protection() {
             (1, make_attestation(3, 4), true),
             (2, make_attestation(2, 3), false),
         ],
-    );
+    )
+    .await;
 }
 
 /// Run a test that creates some validators on one VC, and then migrates them to a second VC.
@@ -1051,7 +1060,7 @@ fn migrate_some_extra_slashing_protection() {
 /// - `import_indices`: validators to transfer. It needn't be a subset of `delete_indices`.
 /// - `second_vc_attestations`: attestations to sign on the second VC after the transfer. The bool
 ///   indicates whether the signing should be successful.
-fn generic_migration_test(
+async fn generic_migration_test(
     num_validators: usize,
     first_vc_attestations: Vec<(usize, Attestation<E>)>,
     delete_indices: Vec<usize>,
@@ -1169,11 +1178,12 @@ fn generic_migration_test(
                 Err(e) => assert!(!should_succeed, "{:?}", e),
             }
         }
-    });
+    })
+    .await
 }
 
-#[test]
+#[tokio::test]
-fn delete_keystores_twice() {
+async fn delete_keystores_twice() {
     run_test(|tester| async move {
         let _ = &tester;
         let password = random_password_string();
@@ -1201,10 +1211,11 @@ fn delete_keystores_twice() {
         let delete_res = tester.client.delete_keystores(&delete_req).await.unwrap();
         check_keystore_delete_response(&delete_res, all_not_active(keystores.len()));
     })
+    .await
 }
 
-#[test]
+#[tokio::test]
-fn delete_nonexistent_keystores() {
+async fn delete_nonexistent_keystores() {
     run_test(|tester| async move {
         let _ = &tester;
         let password = random_password_string();
@@ -1219,6 +1230,7 @@ fn delete_nonexistent_keystores() {
         let delete_res = tester.client.delete_keystores(&delete_req).await.unwrap();
         check_keystore_delete_response(&delete_res, all_not_found(keystores.len()));
     })
+    .await
 }
 
 fn make_attestation(source_epoch: u64, target_epoch: u64) -> Attestation<E> {
@@ -1242,9 +1254,9 @@ fn make_attestation(source_epoch: u64, target_epoch: u64) -> Attestation<E> {
     }
 }
 
-#[test]
+#[tokio::test]
-fn delete_concurrent_with_signing() {
+async fn delete_concurrent_with_signing() {
-    let runtime = build_runtime();
+    let handle = Handle::try_current().unwrap();
     let num_keys = 8;
     let num_signing_threads = 8;
     let num_attestations = 100;
@@ -1257,115 +1269,112 @@ fn delete_concurrent_with_signing() {
         "num_keys should be divisible by num threads for simplicity"
     );
 
-    let weak_runtime = Arc::downgrade(&runtime);
+    let tester = ApiTester::new().await;
-    runtime.block_on(async {
-        let tester = ApiTester::new(weak_runtime).await;
 
-        // Generate a lot of keys and import them.
+    // Generate a lot of keys and import them.
-        let password = random_password_string();
+    let password = random_password_string();
-        let keystores = (0..num_keys)
+    let keystores = (0..num_keys)
-            .map(|_| new_keystore(password.clone()))
+        .map(|_| new_keystore(password.clone()))
-            .collect::<Vec<_>>();
+        .collect::<Vec<_>>();
-        let all_pubkeys = keystores.iter().map(keystore_pubkey).collect::<Vec<_>>();
+    let all_pubkeys = keystores.iter().map(keystore_pubkey).collect::<Vec<_>>();
 
-        let import_res = tester
+    let import_res = tester
-            .client
+        .client
-            .post_keystores(&ImportKeystoresRequest {
+        .post_keystores(&ImportKeystoresRequest {
-                keystores: keystores.clone(),
+            keystores: keystores.clone(),
-                passwords: vec![password.clone(); keystores.len()],
+            passwords: vec![password.clone(); keystores.len()],
-                slashing_protection: None,
+            slashing_protection: None,
-            })
+        })
-            .await
+        .await
-            .unwrap();
+        .unwrap();
-        check_keystore_import_response(&import_res, all_imported(keystores.len()));
+    check_keystore_import_response(&import_res, all_imported(keystores.len()));
 
-        // Start several threads signing attestations at sequential epochs.
+    // Start several threads signing attestations at sequential epochs.
-        let mut join_handles = vec![];
+    let mut join_handles = vec![];
 
-        for thread_index in 0..num_signing_threads {
+    for thread_index in 0..num_signing_threads {
-            let keys_per_thread = num_keys / num_signing_threads;
+        let keys_per_thread = num_keys / num_signing_threads;
-            let validator_store = tester.validator_store.clone();
+        let validator_store = tester.validator_store.clone();
-            let thread_pubkeys = all_pubkeys
+        let thread_pubkeys = all_pubkeys
-                [thread_index * keys_per_thread..(thread_index + 1) * keys_per_thread]
+            [thread_index * keys_per_thread..(thread_index + 1) * keys_per_thread]
-                .to_vec();
+            .to_vec();
 
-            let handle = runtime.spawn(async move {
+        let handle = handle.spawn(async move {
-                for j in 0..num_attestations {
+            for j in 0..num_attestations {
-                    let mut att = make_attestation(j, j + 1);
+                let mut att = make_attestation(j, j + 1);
-                    for (_validator_id, public_key) in thread_pubkeys.iter().enumerate() {
+                for (_validator_id, public_key) in thread_pubkeys.iter().enumerate() {
-                        let _ = validator_store
+                    let _ = validator_store
-                            .sign_attestation(*public_key, 0, &mut att, Epoch::new(j + 1))
+                        .sign_attestation(*public_key, 0, &mut att, Epoch::new(j + 1))
-                            .await;
+                        .await;
-                    }
                 }
-            });
-            join_handles.push(handle);
-        }
-
-        // Concurrently, delete each validator one at a time. Store the slashing protection
-        // data so we can ensure it doesn't change after a key is exported.
-        let mut delete_handles = vec![];
-        for _ in 0..num_delete_threads {
-            let client = tester.client.clone();
-            let all_pubkeys = all_pubkeys.clone();
-
-            let handle = runtime.spawn(async move {
-                let mut rng = SmallRng::from_entropy();
-
-                let mut slashing_protection = vec![];
-                for _ in 0..num_delete_attempts {
-                    let to_delete = all_pubkeys
-                        .iter()
-                        .filter(|_| rng.gen_bool(delete_prob))
-                        .copied()
-                        .collect::<Vec<_>>();
-
-                    if !to_delete.is_empty() {
-                        let delete_res = client
-                            .delete_keystores(&DeleteKeystoresRequest { pubkeys: to_delete })
-                            .await
-                            .unwrap();
-
-                        for status in delete_res.data.iter() {
-                            assert_ne!(status.status, DeleteKeystoreStatus::Error);
-                        }
-
-                        slashing_protection.push(delete_res.slashing_protection);
-                    }
-                }
-                slashing_protection
-            });
-
-            delete_handles.push(handle);
-        }
-
-        // Collect slashing protection.
-        let mut slashing_protection_map = HashMap::new();
-        let collected_slashing_protection = futures::future::join_all(delete_handles).await;
-
-        for interchange in collected_slashing_protection
-            .into_iter()
-            .flat_map(Result::unwrap)
-        {
-            for validator_data in interchange.data {
-                slashing_protection_map
-                    .entry(validator_data.pubkey)
-                    .and_modify(|existing| {
-                        assert_eq!(
-                            *existing, validator_data,
-                            "slashing protection data changed after first export"
-                        )
-                    })
-                    .or_insert(validator_data);
             }
-        }
+        });
+        join_handles.push(handle);
+    }
 
-        futures::future::join_all(join_handles).await
+    // Concurrently, delete each validator one at a time. Store the slashing protection
-    });
+    // data so we can ensure it doesn't change after a key is exported.
+    let mut delete_handles = vec![];
+    for _ in 0..num_delete_threads {
+        let client = tester.client.clone();
+        let all_pubkeys = all_pubkeys.clone();
+
+        let handle = handle.spawn(async move {
+            let mut rng = SmallRng::from_entropy();
+
+            let mut slashing_protection = vec![];
+            for _ in 0..num_delete_attempts {
+                let to_delete = all_pubkeys
+                    .iter()
+                    .filter(|_| rng.gen_bool(delete_prob))
+                    .copied()
+                    .collect::<Vec<_>>();
+
+                if !to_delete.is_empty() {
+                    let delete_res = client
+                        .delete_keystores(&DeleteKeystoresRequest { pubkeys: to_delete })
+                        .await
+                        .unwrap();
+
+                    for status in delete_res.data.iter() {
+                        assert_ne!(status.status, DeleteKeystoreStatus::Error);
+                    }
+
+                    slashing_protection.push(delete_res.slashing_protection);
+                }
+            }
+            slashing_protection
+        });
+
+        delete_handles.push(handle);
+    }
+
+    // Collect slashing protection.
+    let mut slashing_protection_map = HashMap::new();
+    let collected_slashing_protection = futures::future::join_all(delete_handles).await;
+
+    for interchange in collected_slashing_protection
+        .into_iter()
+        .flat_map(Result::unwrap)
+    {
+        for validator_data in interchange.data {
+            slashing_protection_map
+                .entry(validator_data.pubkey)
+                .and_modify(|existing| {
+                    assert_eq!(
+                        *existing, validator_data,
+                        "slashing protection data changed after first export"
+                    )
+                })
+                .or_insert(validator_data);
+        }
+    }
+
+    futures::future::join_all(join_handles).await;
 }
 
-#[test]
+#[tokio::test]
-fn delete_then_reimport() {
+async fn delete_then_reimport() {
     run_test(|tester| async move {
         let _ = &tester;
         let password = random_password_string();
@@ -1396,19 +1405,21 @@ fn delete_then_reimport() {
         let import_res = tester.client.post_keystores(&import_req).await.unwrap();
         check_keystore_import_response(&import_res, all_imported(keystores.len()));
     })
+    .await
 }
 
-#[test]
+#[tokio::test]
-fn get_empty_remotekeys() {
+async fn get_empty_remotekeys() {
     run_test(|tester| async move {
         let _ = &tester;
         let res = tester.client.get_remotekeys().await.unwrap();
         assert_eq!(res, ListRemotekeysResponse { data: vec![] });
     })
+    .await
 }
 
-#[test]
+#[tokio::test]
-fn import_new_remotekeys() {
+async fn import_new_remotekeys() {
     run_test(|tester| async move {
         let _ = &tester;
 
@@ -1443,10 +1454,11 @@ fn import_new_remotekeys() {
         let get_res = tester.client.get_remotekeys().await.unwrap();
         check_remotekey_get_response(&get_res, expected_responses);
     })
+    .await
 }
 
-#[test]
+#[tokio::test]
-fn import_same_remotekey_different_url() {
+async fn import_same_remotekey_different_url() {
     run_test(|tester| async move {
         let _ = &tester;
 
@@ -1485,10 +1497,11 @@ fn import_same_remotekey_different_url() {
             }],
         );
     })
+    .await
 }
 
-#[test]
+#[tokio::test]
-fn delete_remotekey_then_reimport_different_url() {
+async fn delete_remotekey_then_reimport_different_url() {
     run_test(|tester| async move {
         let _ = &tester;
 
@@ -1534,10 +1547,11 @@ fn delete_remotekey_then_reimport_different_url() {
             vec![ImportRemotekeyStatus::Imported].into_iter(),
         );
     })
+    .await
 }
 
-#[test]
+#[tokio::test]
-fn import_only_duplicate_remotekeys() {
+async fn import_only_duplicate_remotekeys() {
     run_test(|tester| async move {
         let _ = &tester;
         let remotekeys = (0..3)
@@ -1582,10 +1596,11 @@ fn import_only_duplicate_remotekeys() {
         let get_res = tester.client.get_remotekeys().await.unwrap();
         check_remotekey_get_response(&get_res, expected_responses);
     })
+    .await
 }
 
-#[test]
+#[tokio::test]
-fn import_some_duplicate_remotekeys() {
+async fn import_some_duplicate_remotekeys() {
     run_test(|tester| async move {
         let _ = &tester;
         let num_remotekeys = 5;
@@ -1649,10 +1664,11 @@ fn import_some_duplicate_remotekeys() {
         let get_res = tester.client.get_remotekeys().await.unwrap();
         check_remotekey_get_response(&get_res, expected_responses);
     })
+    .await
 }
 
-#[test]
+#[tokio::test]
-fn import_remote_and_local_keys() {
+async fn import_remote_and_local_keys() {
     run_test(|tester| async move {
         let _ = &tester;
         let num_local = 3;
@@ -1714,10 +1730,11 @@ fn import_remote_and_local_keys() {
             assert!(get_res.data.contains(&response), "{:?}", response);
         }
     })
+    .await
 }
 
-#[test]
+#[tokio::test]
-fn import_same_local_and_remote_keys() {
+async fn import_same_local_and_remote_keys() {
     run_test(|tester| async move {
         let _ = &tester;
         let num_local = 3;
@@ -1782,9 +1799,10 @@ fn import_same_local_and_remote_keys() {
             assert!(get_res.data.contains(&response), "{:?}", response);
         }
     })
+    .await
 }
-#[test]
+#[tokio::test]
-fn import_same_remote_and_local_keys() {
+async fn import_same_remote_and_local_keys() {
     run_test(|tester| async move {
         let _ = &tester;
         let num_local = 3;
@@ -1847,10 +1865,11 @@ fn import_same_remote_and_local_keys() {
         let get_res = tester.client.get_remotekeys().await.unwrap();
         check_remotekey_get_response(&get_res, expected_responses);
     })
+    .await
 }
 
-#[test]
+#[tokio::test]
-fn delete_remotekeys_twice() {
+async fn delete_remotekeys_twice() {
     run_test(|tester| async move {
         let _ = &tester;
 
@@ -1893,10 +1912,11 @@ fn delete_remotekeys_twice() {
         let get_res = tester.client.get_remotekeys().await.unwrap();
         check_remotekey_get_response(&get_res, Vec::new());
     })
+    .await
 }
 
-#[test]
+#[tokio::test]
-fn delete_nonexistent_remotekey() {
+async fn delete_nonexistent_remotekey() {
     run_test(|tester| async move {
         let _ = &tester;
 
@@ -1919,10 +1939,11 @@ fn delete_nonexistent_remotekey() {
         let get_res = tester.client.get_remotekeys().await.unwrap();
         check_remotekey_get_response(&get_res, Vec::new());
     })
+    .await
 }
 
-#[test]
+#[tokio::test]
-fn delete_then_reimport_remotekeys() {
+async fn delete_then_reimport_remotekeys() {
     run_test(|tester| async move {
         let _ = &tester;
 
@@ -1984,10 +2005,11 @@ fn delete_then_reimport_remotekeys() {
         let get_res = tester.client.get_remotekeys().await.unwrap();
         check_remotekey_get_response(&get_res, expected_responses);
     })
+    .await
 }
 
-#[test]
+#[tokio::test]
-fn import_remotekey_web3signer() {
+async fn import_remotekey_web3signer() {
     run_test(|tester| async move {
         let _ = &tester;
 
@@ -2043,10 +2065,11 @@ fn import_remotekey_web3signer() {
         let get_res = tester.client.get_remotekeys().await.unwrap();
         check_remotekey_get_response(&get_res, expected_responses);
     })
+    .await
 }
 
-#[test]
+#[tokio::test]
-fn import_remotekey_web3signer_disabled() {
+async fn import_remotekey_web3signer_disabled() {
     run_test(|tester| async move {
         let _ = &tester;
 
@@ -2096,10 +2119,11 @@ fn import_remotekey_web3signer_disabled() {
         let get_res = tester.client.get_remotekeys().await.unwrap();
         check_remotekey_get_response(&get_res, expected_responses);
     })
+    .await
 }
 
-#[test]
+#[tokio::test]
-fn import_remotekey_web3signer_enabled() {
+async fn import_remotekey_web3signer_enabled() {
     run_test(|tester| async move {
         let _ = &tester;
 
@@ -2156,4 +2180,5 @@ fn import_remotekey_web3signer_enabled() {
         let get_res = tester.client.get_remotekeys().await.unwrap();
         check_remotekey_get_response(&get_res, expected_responses);
     })
+    .await
 }

validator_client/src/initialized_validators.rs

@@ -8,7 +8,7 @@
 
 use crate::signing_method::SigningMethod;
 use account_utils::{
-    read_password, read_password_from_user,
+    read_password, read_password_from_user, read_password_string,
     validator_definitions::{
         self, SigningDefinition, ValidatorDefinition, ValidatorDefinitions, Web3SignerDefinition,
         CONFIG_FILENAME,
@@ -44,6 +44,19 @@ const DEFAULT_REMOTE_SIGNER_REQUEST_TIMEOUT: Duration = Duration::from_secs(12);
 // Use TTY instead of stdin to capture passwords from users.
 const USE_STDIN: bool = false;
 
+pub enum OnDecryptFailure {
+    /// If the key cache fails to decrypt, create a new cache.
+    CreateNew,
+    /// Return an error if the key cache fails to decrypt. This should only be
+    /// used in testing.
+    Error,
+}
+
+pub struct KeystoreAndPassword {
+    pub keystore: Keystore,
+    pub password: Option<ZeroizeString>,
+}
+
 #[derive(Debug)]
 pub enum Error {
     /// Refused to open a validator with an existing lockfile since that validator may be in-use by
@@ -98,6 +111,11 @@ pub enum Error {
     UnableToBuildWeb3SignerClient(ReqwestError),
     /// Unable to apply an action to a validator.
     InvalidActionOnValidator,
+    UnableToReadValidatorPassword(String),
+    UnableToReadKeystoreFile(eth2_keystore::Error),
+    UnableToSaveKeyCache(key_cache::Error),
+    UnableToDecryptKeyCache(key_cache::Error),
+    UnableToDeletePasswordFile(PathBuf, io::Error),
 }
 
 impl From<LockfileError> for Error {
@@ -539,33 +557,78 @@ impl InitializedValidators {
         &mut self,
         pubkey: &PublicKey,
         is_local_keystore: bool,
-    ) -> Result<(), Error> {
+    ) -> Result<Option<KeystoreAndPassword>, Error> {
         // 1. Disable the validator definition.
         //
         // We disable before removing so that in case of a crash the auto-discovery mechanism
         // won't re-activate the keystore.
-        if let Some(def) = self
+        let mut uuid_opt = None;
+        let mut password_path_opt = None;
+        let keystore_and_password = if let Some(def) = self
             .definitions
             .as_mut_slice()
             .iter_mut()
             .find(|def| &def.voting_public_key == pubkey)
         {
-            // Update definition for local keystore
+            match &def.signing_definition {
-            if def.signing_definition.is_local_keystore() && is_local_keystore {
+                SigningDefinition::LocalKeystore {
-                def.enabled = false;
+                    voting_keystore_path,
-                self.definitions
+                    voting_keystore_password,
-                    .save(&self.validators_dir)
+                    voting_keystore_password_path,
-                    .map_err(Error::UnableToSaveDefinitions)?;
+                    ..
-            } else if !def.signing_definition.is_local_keystore() && !is_local_keystore {
+                } if is_local_keystore => {
-                def.enabled = false;
+                    let password = match (voting_keystore_password, voting_keystore_password_path) {
-            } else {
+                        (Some(password), _) => Some(password.clone()),
-                return Err(Error::InvalidActionOnValidator);
+                        (_, Some(path)) => {
+                            password_path_opt = Some(path.clone());
+                            read_password_string(path)
+                                .map(Option::Some)
+                                .map_err(Error::UnableToReadValidatorPassword)?
+                        }
+                        (None, None) => None,
+                    };
+                    let keystore = Keystore::from_json_file(voting_keystore_path)
+                        .map_err(Error::UnableToReadKeystoreFile)?;
+                    uuid_opt = Some(*keystore.uuid());
+
+                    def.enabled = false;
+                    self.definitions
+                        .save(&self.validators_dir)
+                        .map_err(Error::UnableToSaveDefinitions)?;
+
+                    Some(KeystoreAndPassword { keystore, password })
+                }
+                SigningDefinition::Web3Signer(_) if !is_local_keystore => {
+                    def.enabled = false;
+                    None
+                }
+                _ => return Err(Error::InvalidActionOnValidator),
             }
         } else {
             return Err(Error::ValidatorNotInitialized(pubkey.clone()));
+        };
+
+        // 2. Remove the validator from the key cache. This ensures the key
+        // cache is consistent next time the VC starts.
+        //
+        // It's not a big deal if this succeeds and something fails later in
+        // this function because the VC will self-heal from a corrupt key cache.
+        //
+        // Do this before modifying `self.validators` or deleting anything from
+        // the filesystem.
+        if let Some(uuid) = uuid_opt {
+            let key_cache = KeyCache::open_or_create(&self.validators_dir)
+                .map_err(Error::UnableToOpenKeyCache)?;
+            let mut decrypted_key_cache = self
+                .decrypt_key_cache(key_cache, &mut <_>::default(), OnDecryptFailure::CreateNew)
+                .await?;
+            decrypted_key_cache.remove(&uuid);
+            decrypted_key_cache
+                .save(&self.validators_dir)
+                .map_err(Error::UnableToSaveKeyCache)?;
         }
 
-        // 2. Delete from `self.validators`, which holds the signing method.
+        // 3. Delete from `self.validators`, which holds the signing method.
         //    Delete the keystore files.
         if let Some(initialized_validator) = self.validators.remove(&pubkey.compress()) {
             if let SigningMethod::LocalKeystore {
@@ -583,14 +646,28 @@ impl InitializedValidators {
             }
         }
 
-        // 3. Delete from validator definitions entirely.
+        // 4. Delete from validator definitions entirely.
         self.definitions
             .retain(|def| &def.voting_public_key != pubkey);
         self.definitions
             .save(&self.validators_dir)
             .map_err(Error::UnableToSaveDefinitions)?;
 
-        Ok(())
+        // 5. Delete the keystore password if it's not being used by any definition.
+        if let Some(password_path) = password_path_opt.and_then(|p| p.canonicalize().ok()) {
+            if self
+                .definitions
+                .iter_voting_keystore_password_paths()
+                // Require canonicalized paths so we can do a true equality check.
+                .filter_map(|existing| existing.canonicalize().ok())
+                .all(|existing| existing != password_path)
+            {
+                fs::remove_file(&password_path)
+                    .map_err(|e| Error::UnableToDeletePasswordFile(password_path, e))?;
+            }
+        }
+
+        Ok(keystore_and_password)
     }
 
     /// Attempt to delete the voting keystore file, or its entire validator directory.
@@ -900,10 +977,11 @@ impl InitializedValidators {
     /// filesystem accesses for keystores that are already known. In the case that a keystore
     /// from the validator definitions is not yet in this map, it will be loaded from disk and
     /// inserted into the map.
-    async fn decrypt_key_cache(
+    pub async fn decrypt_key_cache(
         &self,
         mut cache: KeyCache,
         key_stores: &mut HashMap<PathBuf, Keystore>,
+        on_failure: OnDecryptFailure,
     ) -> Result<KeyCache, Error> {
         // Read relevant key stores from the filesystem.
         let mut definitions_map = HashMap::new();
@@ -971,11 +1049,13 @@ impl InitializedValidators {
 
         //decrypt
         tokio::task::spawn_blocking(move || match cache.decrypt(passwords, public_keys) {
-            Ok(_) | Err(key_cache::Error::AlreadyDecrypted) => cache,
+            Ok(_) | Err(key_cache::Error::AlreadyDecrypted) => Ok(cache),
-            _ => KeyCache::new(),
+            _ if matches!(on_failure, OnDecryptFailure::CreateNew) => Ok(KeyCache::new()),
+            Err(e) => Err(e),
         })
         .await
-        .map_err(Error::TokioJoin)
+        .map_err(Error::TokioJoin)?
+        .map_err(Error::UnableToDecryptKeyCache)
     }
 
     /// Scans `self.definitions` and attempts to initialize and validators which are not already
@@ -1013,7 +1093,8 @@ impl InitializedValidators {
         // Only decrypt cache when there is at least one local definition.
         // Decrypting cache is a very expensive operation which is never used for web3signer.
         let mut key_cache = if has_local_definitions {
-            self.decrypt_key_cache(cache, &mut key_stores).await?
+            self.decrypt_key_cache(cache, &mut key_stores, OnDecryptFailure::CreateNew)
+                .await?
         } else {
             // Assign an empty KeyCache if all definitions are of the Web3Signer type.
             KeyCache::new()
@@ -1191,4 +1272,41 @@ impl InitializedValidators {
             val.index = Some(index);
         }
     }
+
+    /// Deletes any passwords stored in the validator definitions file and
+    /// returns a map of pubkey to deleted password.
+    ///
+    /// This should only be used for testing, it's rather destructive.
+    pub fn delete_passwords_from_validator_definitions(
+        &mut self,
+    ) -> Result<HashMap<PublicKey, ZeroizeString>, Error> {
+        let mut passwords = HashMap::default();
+
+        for def in self.definitions.as_mut_slice() {
+            match &mut def.signing_definition {
+                SigningDefinition::LocalKeystore {
+                    ref mut voting_keystore_password,
+                    ..
+                } => {
+                    if let Some(password) = voting_keystore_password.take() {
+                        passwords.insert(def.voting_public_key.clone(), password);
+                    }
+                }
+                // Remote signers don't have passwords.
+                SigningDefinition::Web3Signer { .. } => (),
+            };
+        }
+
+        self.definitions
+            .save(&self.validators_dir)
+            .map_err(Error::UnableToSaveDefinitions)?;
+
+        Ok(passwords)
+    }
+
+    /// Prefer other methods in production. Arbitrarily modifying a validator
+    /// definition manually may result in inconsistencies.
+    pub fn as_mut_slice_testing_only(&mut self) -> &mut [ValidatorDefinition] {
+        self.definitions.as_mut_slice()
+    }
 }

validator_client/src/key_cache.rs

@@ -47,6 +47,12 @@ pub struct KeyCache {
 
 type SerializedKeyMap = HashMap<Uuid, ZeroizeHash>;
 
+impl Default for KeyCache {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 impl KeyCache {
     pub fn new() -> Self {
         KeyCache {

validator_client/src/lib.rs

@@ -172,9 +172,12 @@ impl<T: EthSpec> ProductionValidatorClient<T> {
             let new_validators = validator_defs
                 .discover_local_keystores(&config.validator_dir, &config.secrets_dir, &log)
                 .map_err(|e| format!("Unable to discover local validator keystores: {:?}", e))?;
-            validator_defs
+            validator_defs.save(&config.validator_dir).map_err(|e| {
-                .save(&config.validator_dir)
+                format!(
-                .map_err(|e| format!("Unable to update validator definitions: {:?}", e))?;
+                    "Provide --suggested-fee-recipient or update validator definitions: {:?}",
+                    e
+                )
+            })?;
             info!(
                 log,
                 "Completed validator discovery";
@@ -573,6 +576,7 @@ impl<T: EthSpec> ProductionValidatorClient<T> {
                 api_secret,
                 validator_store: Some(self.validator_store.clone()),
                 validator_dir: Some(self.config.validator_dir.clone()),
+                secrets_dir: Some(self.config.secrets_dir.clone()),
                 graffiti_file: self.config.graffiti_file.clone(),
                 graffiti_flag: self.config.graffiti,
                 spec: self.context.eth2_config.spec.clone(),

validator_client/src/notifier.rs

@@ -94,8 +94,7 @@ async fn notify<T: SlotClock + 'static, E: EthSpec>(
             info!(
                 log,
                 "No validators present";
-                "msg" => "see `lighthouse account validator create --help` \
+                "msg" => "see `lighthouse vm create --help` or the HTTP API documentation"
-                or the HTTP API documentation"
             )
         } else if total_validators == attesting_validators {
             info!(

validator_client/src/preparation_service.rs

@@ -1,5 +1,5 @@
 use crate::beacon_node_fallback::{BeaconNodeFallback, RequireSynced};
-use crate::validator_store::{DoppelgangerStatus, ValidatorStore};
+use crate::validator_store::{DoppelgangerStatus, Error as ValidatorStoreError, ValidatorStore};
 use crate::OfflineOnFailure;
 use bls::PublicKeyBytes;
 use environment::RuntimeContext;
@@ -442,8 +442,23 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
                     .await
                 {
                     Ok(data) => data,
+                    Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
+                        // A pubkey can be missing when a validator was recently
+                        // removed via the API.
+                        debug!(
+                            log,
+                            "Missing pubkey for registration data";
+                            "pubkey" => ?pubkey,
+                        );
+                        continue;
+                    }
                     Err(e) => {
-                        error!(log, "Unable to sign validator registration data"; "error" => ?e, "pubkey" => ?pubkey);
+                        error!(
+                            log,
+                            "Unable to sign validator registration data";
+                            "error" => ?e,
+                            "pubkey" => ?pubkey
+                        );
                         continue;
                     }
                 };

validator_client/src/sync_committee_service.rs

@@ -1,5 +1,9 @@
 use crate::beacon_node_fallback::{BeaconNodeFallback, RequireSynced};
-use crate::{duties_service::DutiesService, validator_store::ValidatorStore, OfflineOnFailure};
+use crate::{
+    duties_service::DutiesService,
+    validator_store::{Error as ValidatorStoreError, ValidatorStore},
+    OfflineOnFailure,
+};
 use environment::RuntimeContext;
 use eth2::types::BlockId;
 use futures::future::join_all;
@@ -264,6 +268,18 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
                 .await
             {
                 Ok(signature) => Some(signature),
+                Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
+                    // A pubkey can be missing when a validator was recently
+                    // removed via the API.
+                    debug!(
+                        log,
+                        "Missing pubkey for sync committee signature";
+                        "pubkey" => ?pubkey,
+                        "validator_index" => duty.validator_index,
+                        "slot" => slot,
+                    );
+                    None
+                }
                 Err(e) => {
                     crit!(
                         log,
@@ -405,6 +421,17 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
                     .await
                 {
                     Ok(signed_contribution) => Some(signed_contribution),
+                    Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
+                        // A pubkey can be missing when a validator was recently
+                        // removed via the API.
+                        debug!(
+                            log,
+                            "Missing pubkey for sync contribution";
+                            "pubkey" => ?pubkey,
+                            "slot" => slot,
+                        );
+                        None
+                    }
                     Err(e) => {
                         crit!(
                             log,

validator_client/src/validator_store.rs

@@ -5,7 +5,7 @@ use crate::{
     signing_method::{Error as SigningError, SignableMessage, SigningContext, SigningMethod},
     Config,
 };
-use account_utils::{validator_definitions::ValidatorDefinition, ZeroizeString};
+use account_utils::validator_definitions::{PasswordStorage, ValidatorDefinition};
 use parking_lot::{Mutex, RwLock};
 use slashing_protection::{
     interchange::Interchange, InterchangeError, NotSafe, Safe, SlashingDatabase,
@@ -170,7 +170,7 @@ impl<T: SlotClock + 'static, E: EthSpec> ValidatorStore<T, E> {
     pub async fn add_validator_keystore<P: AsRef<Path>>(
         &self,
         voting_keystore_path: P,
-        password: ZeroizeString,
+        password_storage: PasswordStorage,
         enable: bool,
         graffiti: Option<GraffitiString>,
         suggested_fee_recipient: Option<Address>,
@@ -179,7 +179,7 @@ impl<T: SlotClock + 'static, E: EthSpec> ValidatorStore<T, E> {
     ) -> Result<ValidatorDefinition, String> {
         let mut validator_def = ValidatorDefinition::new_keystore_with_password(
             voting_keystore_path,
-            Some(password),
+            password_storage,
             graffiti.map(Into::into),
             suggested_fee_recipient,
             gas_limit,

validator_manager/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+name = "validator_manager"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+bls = { path = "../crypto/bls" }
+clap = "2.33.3"
+types = { path = "../consensus/types" }
+environment = { path = "../lighthouse/environment" }
+eth2_network_config = { path = "../common/eth2_network_config" }
+clap_utils = { path = "../common/clap_utils" }
+eth2_wallet = { path = "../crypto/eth2_wallet" }
+eth2_keystore = { path = "../crypto/eth2_keystore" }
+account_utils = { path = "../common/account_utils" }
+serde = { version = "1.0.116", features = ["derive"] }
+serde_json = "1.0.58"
+ethereum_serde_utils = "0.5.0"
+tree_hash = "0.5.0"
+eth2 = { path = "../common/eth2", features = ["lighthouse"]}
+hex = "0.4.2"
+tokio = { version = "1.14.0", features = ["time", "rt-multi-thread", "macros"] }
+
+[dev-dependencies]
+tempfile = "3.1.0"
+regex = "1.6.0"
+eth2_network_config = { path = "../common/eth2_network_config" }
+validator_client = { path = "../validator_client" }

validator_manager/src/common.rs

@@ -0,0 +1,361 @@
+use account_utils::{strip_off_newlines, ZeroizeString};
+use eth2::lighthouse_vc::std_types::{InterchangeJsonStr, KeystoreJsonStr};
+use eth2::{
+    lighthouse_vc::{
+        http_client::ValidatorClientHttpClient,
+        std_types::{ImportKeystoreStatus, ImportKeystoresRequest, SingleKeystoreResponse, Status},
+        types::UpdateFeeRecipientRequest,
+    },
+    SensitiveUrl,
+};
+use serde::{Deserialize, Serialize};
+use std::fs;
+use std::path::{Path, PathBuf};
+use tree_hash::TreeHash;
+use types::*;
+
+pub const IGNORE_DUPLICATES_FLAG: &str = "ignore-duplicates";
+pub const STDIN_INPUTS_FLAG: &str = "stdin-inputs";
+pub const COUNT_FLAG: &str = "count";
+
+/// When the `ethereum/staking-deposit-cli` tool generates deposit data JSON, it adds a
+/// `deposit_cli_version` to protect the web-based "Launchpad" tool against a breaking change that
+/// was introduced in `ethereum/staking-deposit-cli`. Lighthouse don't really have a version that it
+/// can use here, so we choose a static string that is:
+///
+/// 1. High enough that it's accepted by Launchpad.
+/// 2. Weird enough to identify Lighthouse.
+const LIGHTHOUSE_DEPOSIT_CLI_VERSION: &str = "20.18.20";
+
+#[derive(Debug)]
+pub enum UploadError {
+    InvalidPublicKey,
+    DuplicateValidator(PublicKeyBytes),
+    FailedToListKeys(eth2::Error),
+    KeyUploadFailed(eth2::Error),
+    IncorrectStatusCount(usize),
+    FeeRecipientUpdateFailed(eth2::Error),
+    PatchValidatorFailed(eth2::Error),
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+pub struct ValidatorSpecification {
+    pub voting_keystore: KeystoreJsonStr,
+    pub voting_keystore_password: ZeroizeString,
+    pub slashing_protection: Option<InterchangeJsonStr>,
+    pub fee_recipient: Option<Address>,
+    pub gas_limit: Option<u64>,
+    pub builder_proposals: Option<bool>,
+    pub enabled: Option<bool>,
+}
+
+impl ValidatorSpecification {
+    /// Upload the validator to a validator client via HTTP.
+    pub async fn upload(
+        self,
+        http_client: &ValidatorClientHttpClient,
+        ignore_duplicates: bool,
+    ) -> Result<Status<ImportKeystoreStatus>, UploadError> {
+        let ValidatorSpecification {
+            voting_keystore,
+            voting_keystore_password,
+            slashing_protection,
+            fee_recipient,
+            gas_limit,
+            builder_proposals,
+            enabled,
+        } = self;
+
+        let voting_public_key = voting_keystore
+            .public_key()
+            .ok_or(UploadError::InvalidPublicKey)?
+            .into();
+
+        let request = ImportKeystoresRequest {
+            keystores: vec![voting_keystore],
+            passwords: vec![voting_keystore_password],
+            slashing_protection,
+        };
+
+        // Check to see if this validator already exists on the remote validator.
+        match http_client.get_keystores().await {
+            Ok(response) => {
+                if response
+                    .data
+                    .iter()
+                    .any(|validator| validator.validating_pubkey == voting_public_key)
+                {
+                    if ignore_duplicates {
+                        eprintln!(
+                            "Duplicate validators are ignored, ignoring {:?} which exists \
+                            on the destination validator client",
+                            voting_public_key
+                        );
+                    } else {
+                        return Err(UploadError::DuplicateValidator(voting_public_key));
+                    }
+                }
+            }
+            Err(e) => {
+                return Err(UploadError::FailedToListKeys(e));
+            }
+        };
+
+        let mut statuses = http_client
+            .post_keystores(&request)
+            .await
+            .map_err(UploadError::KeyUploadFailed)?
+            .data;
+
+        let status = statuses.pop().ok_or(UploadError::IncorrectStatusCount(0))?;
+        if !statuses.is_empty() {
+            return Err(UploadError::IncorrectStatusCount(statuses.len() + 1));
+        }
+
+        // Exit early if there's an error uploading.
+        if status.status == ImportKeystoreStatus::Error {
+            return Ok(status);
+        }
+
+        if let Some(fee_recipient) = fee_recipient {
+            http_client
+                .post_fee_recipient(
+                    &voting_public_key,
+                    &UpdateFeeRecipientRequest {
+                        ethaddress: fee_recipient,
+                    },
+                )
+                .await
+                .map_err(UploadError::FeeRecipientUpdateFailed)?;
+        }
+
+        if gas_limit.is_some() || builder_proposals.is_some() || enabled.is_some() {
+            http_client
+                .patch_lighthouse_validators(
+                    &voting_public_key,
+                    enabled,
+                    gas_limit,
+                    builder_proposals,
+                    None, // Grafitti field is not maintained between validator moves.
+                )
+                .await
+                .map_err(UploadError::PatchValidatorFailed)?;
+        }
+
+        Ok(status)
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct CreateSpec {
+    pub mnemonic: String,
+    pub validator_client_url: Option<SensitiveUrl>,
+    pub validator_client_token_path: Option<PathBuf>,
+    pub json_deposit_data_path: Option<PathBuf>,
+    pub ignore_duplicates: bool,
+    pub validators: Vec<ValidatorSpecification>,
+}
+
+/// The structure generated by the `staking-deposit-cli` which has become a quasi-standard for
+/// browser-based deposit submission tools (e.g., the Ethereum Launchpad and Lido).
+///
+/// We assume this code as the canonical definition:
+///
+/// https://github.com/ethereum/staking-deposit-cli/blob/76ed78224fdfe3daca788d12442b3d1a37978296/staking_deposit/credentials.py#L131-L144
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub struct StandardDepositDataJson {
+    #[serde(with = "public_key_bytes_without_0x_prefix")]
+    pub pubkey: PublicKeyBytes,
+    #[serde(with = "hash256_without_0x_prefix")]
+    pub withdrawal_credentials: Hash256,
+    /// The `amount` field is *not* quoted (i.e., a string) like most other `u64` fields in the
+    /// consensus specs, it's a simple integer.
+    pub amount: u64,
+    #[serde(with = "signature_bytes_without_0x_prefix")]
+    pub signature: SignatureBytes,
+    #[serde(with = "bytes_4_without_0x_prefix")]
+    pub fork_version: [u8; 4],
+    pub network_name: String,
+    #[serde(with = "hash256_without_0x_prefix")]
+    pub deposit_message_root: Hash256,
+    #[serde(with = "hash256_without_0x_prefix")]
+    pub deposit_data_root: Hash256,
+    pub deposit_cli_version: String,
+}
+
+impl StandardDepositDataJson {
+    pub fn new(
+        keypair: &Keypair,
+        withdrawal_credentials: Hash256,
+        amount: u64,
+        spec: &ChainSpec,
+    ) -> Result<Self, String> {
+        let deposit_data = {
+            let mut deposit_data = DepositData {
+                pubkey: keypair.pk.clone().into(),
+                withdrawal_credentials,
+                amount,
+                signature: SignatureBytes::empty(),
+            };
+            deposit_data.signature = deposit_data.create_signature(&keypair.sk, spec);
+            deposit_data
+        };
+
+        let deposit_message_root = deposit_data.as_deposit_message().tree_hash_root();
+        let deposit_data_root = deposit_data.tree_hash_root();
+
+        let DepositData {
+            pubkey,
+            withdrawal_credentials,
+            amount,
+            signature,
+        } = deposit_data;
+
+        Ok(Self {
+            pubkey,
+            withdrawal_credentials,
+            amount,
+            signature,
+            fork_version: spec.genesis_fork_version,
+            network_name: spec
+                .config_name
+                .clone()
+                .ok_or("The network specification does not have a CONFIG_NAME set")?,
+            deposit_message_root,
+            deposit_data_root,
+            deposit_cli_version: LIGHTHOUSE_DEPOSIT_CLI_VERSION.to_string(),
+        })
+    }
+}
+
+macro_rules! without_0x_prefix {
+    ($mod_name: ident, $type: ty) => {
+        pub mod $mod_name {
+            use super::*;
+            use std::str::FromStr;
+
+            struct Visitor;
+
+            impl<'de> serde::de::Visitor<'de> for Visitor {
+                type Value = $type;
+
+                fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+                    formatter.write_str("ascii hex without a 0x prefix")
+                }
+
+                fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
+                where
+                    E: serde::de::Error,
+                {
+                    <$type>::from_str(&format!("0x{}", v)).map_err(serde::de::Error::custom)
+                }
+            }
+
+            /// Serialize with quotes.
+            pub fn serialize<S>(value: &$type, serializer: S) -> Result<S::Ok, S::Error>
+            where
+                S: serde::Serializer,
+            {
+                let with_prefix = format!("{:?}", value);
+                let without_prefix = with_prefix
+                    .strip_prefix("0x")
+                    .ok_or_else(|| serde::ser::Error::custom("serialization is missing 0x"))?;
+                serializer.serialize_str(&without_prefix)
+            }
+
+            /// Deserialize with quotes.
+            pub fn deserialize<'de, D>(deserializer: D) -> Result<$type, D::Error>
+            where
+                D: serde::Deserializer<'de>,
+            {
+                deserializer.deserialize_str(Visitor)
+            }
+        }
+    };
+}
+
+without_0x_prefix!(hash256_without_0x_prefix, Hash256);
+without_0x_prefix!(signature_bytes_without_0x_prefix, SignatureBytes);
+without_0x_prefix!(public_key_bytes_without_0x_prefix, PublicKeyBytes);
+
+mod bytes_4_without_0x_prefix {
+    use serde::de::Error;
+
+    const BYTES_LEN: usize = 4;
+
+    pub fn serialize<S>(bytes: &[u8; BYTES_LEN], serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        let hex_string = &hex::encode(bytes);
+        serializer.serialize_str(hex_string)
+    }
+
+    pub fn deserialize<'de, D>(deserializer: D) -> Result<[u8; BYTES_LEN], D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let decoded = deserializer.deserialize_str(serde_utils::hex::HexVisitor)?;
+
+        if decoded.len() != BYTES_LEN {
+            return Err(D::Error::custom(format!(
+                "expected {} bytes for array, got {}",
+                BYTES_LEN,
+                decoded.len()
+            )));
+        }
+
+        let mut array = [0; BYTES_LEN];
+        array.copy_from_slice(&decoded);
+        Ok(array)
+    }
+}
+
+pub async fn vc_http_client<P: AsRef<Path>>(
+    url: SensitiveUrl,
+    token_path: P,
+) -> Result<(ValidatorClientHttpClient, Vec<SingleKeystoreResponse>), String> {
+    let token_path = token_path.as_ref();
+    let token_bytes =
+        fs::read(token_path).map_err(|e| format!("Failed to read {:?}: {:?}", token_path, e))?;
+    let token_string = String::from_utf8(strip_off_newlines(token_bytes))
+        .map_err(|e| format!("Failed to parse {:?} as utf8: {:?}", token_path, e))?;
+    let http_client = ValidatorClientHttpClient::new(url.clone(), token_string).map_err(|e| {
+        format!(
+            "Could not instantiate HTTP client from URL and secret: {:?}",
+            e
+        )
+    })?;
+
+    // Perform a request to check that the connection works
+    let remote_keystores = http_client
+        .get_keystores()
+        .await
+        .map_err(|e| format!("Failed to list keystores on VC: {:?}", e))?
+        .data;
+
+    eprintln!(
+        "Validator client is reachable at {} and reports {} validators",
+        url,
+        remote_keystores.len()
+    );
+
+    Ok((http_client, remote_keystores))
+}
+
+/// Write some object to a file as JSON.
+///
+/// The file must be created new, it must not already exist.
+pub fn write_to_json_file<P: AsRef<Path>, S: Serialize>(
+    path: P,
+    contents: &S,
+) -> Result<(), String> {
+    eprintln!("Writing {:?}", path.as_ref());
+    let mut file = fs::OpenOptions::new()
+        .write(true)
+        .create_new(true)
+        .open(&path)
+        .map_err(|e| format!("Failed to open {:?}: {:?}", path.as_ref(), e))?;
+    serde_json::to_writer(&mut file, contents)
+        .map_err(|e| format!("Failed to write JSON to {:?}: {:?}", path.as_ref(), e))
+}

validator_manager/src/create_validators.rs

@@ -0,0 +1,934 @@
+use super::common::*;
+use crate::DumpConfig;
+use account_utils::{random_password_string, read_mnemonic_from_cli, read_password_from_user};
+use clap::{App, Arg, ArgMatches};
+use eth2::{
+    lighthouse_vc::std_types::KeystoreJsonStr,
+    types::{StateId, ValidatorId},
+    BeaconNodeHttpClient, SensitiveUrl, Timeouts,
+};
+use eth2_wallet::WalletBuilder;
+use serde::{Deserialize, Serialize};
+use std::fs;
+use std::path::PathBuf;
+use std::time::Duration;
+use types::*;
+
+pub const CMD: &str = "create";
+pub const OUTPUT_PATH_FLAG: &str = "output-path";
+pub const DEPOSIT_GWEI_FLAG: &str = "deposit-gwei";
+pub const DISABLE_DEPOSITS_FLAG: &str = "disable-deposits";
+pub const FIRST_INDEX_FLAG: &str = "first-index";
+pub const MNEMONIC_FLAG: &str = "mnemonic-path";
+pub const SPECIFY_VOTING_KEYSTORE_PASSWORD_FLAG: &str = "specify-voting-keystore-password";
+pub const ETH1_WITHDRAWAL_ADDRESS_FLAG: &str = "eth1-withdrawal-address";
+pub const GAS_LIMIT_FLAG: &str = "gas-limit";
+pub const FEE_RECIPIENT_FLAG: &str = "suggested-fee-recipient";
+pub const BUILDER_PROPOSALS_FLAG: &str = "builder-proposals";
+pub const BEACON_NODE_FLAG: &str = "beacon-node";
+pub const FORCE_BLS_WITHDRAWAL_CREDENTIALS: &str = "force-bls-withdrawal-credentials";
+
+pub const VALIDATORS_FILENAME: &str = "validators.json";
+pub const DEPOSITS_FILENAME: &str = "deposits.json";
+
+const BEACON_NODE_HTTP_TIMEOUT: Duration = Duration::from_secs(2);
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about(
+            "Creates new validators from BIP-39 mnemonic. A JSON file will be created which \
+                contains all the validator keystores and other validator data. This file can then \
+                be imported to a validator client using the \"import-validators\" command. \
+                Another, optional JSON file is created which contains a list of validator \
+                deposits in the same format as the \"ethereum/staking-deposit-cli\" tool.",
+        )
+        .arg(
+            Arg::with_name(OUTPUT_PATH_FLAG)
+                .long(OUTPUT_PATH_FLAG)
+                .value_name("DIRECTORY")
+                .help(
+                    "The path to a directory where the validator and (optionally) deposits \
+                    files will be created. The directory will be created if it does not exist.",
+                )
+                .required(true)
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(DEPOSIT_GWEI_FLAG)
+                .long(DEPOSIT_GWEI_FLAG)
+                .value_name("DEPOSIT_GWEI")
+                .help(
+                    "The GWEI value of the deposit amount. Defaults to the minimum amount \
+                    required for an active validator (MAX_EFFECTIVE_BALANCE)",
+                )
+                .conflicts_with(DISABLE_DEPOSITS_FLAG)
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(FIRST_INDEX_FLAG)
+                .long(FIRST_INDEX_FLAG)
+                .value_name("FIRST_INDEX")
+                .help("The first of consecutive key indexes you wish to create.")
+                .takes_value(true)
+                .required(false)
+                .default_value("0"),
+        )
+        .arg(
+            Arg::with_name(COUNT_FLAG)
+                .long(COUNT_FLAG)
+                .value_name("VALIDATOR_COUNT")
+                .help("The number of validators to create, regardless of how many already exist")
+                .conflicts_with("at-most")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(MNEMONIC_FLAG)
+                .long(MNEMONIC_FLAG)
+                .value_name("MNEMONIC_PATH")
+                .help("If present, the mnemonic will be read in from this file.")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(STDIN_INPUTS_FLAG)
+                .takes_value(false)
+                .hidden(cfg!(windows))
+                .long(STDIN_INPUTS_FLAG)
+                .help("If present, read all user inputs from stdin instead of tty."),
+        )
+        .arg(
+            Arg::with_name(DISABLE_DEPOSITS_FLAG)
+                .long(DISABLE_DEPOSITS_FLAG)
+                .help(
+                    "When provided don't generate the deposits JSON file that is \
+                    commonly used for submitting validator deposits via a web UI. \
+                    Using this flag will save several seconds per validator if the \
+                    user has an alternate strategy for submitting deposits.",
+                ),
+        )
+        .arg(
+            Arg::with_name(SPECIFY_VOTING_KEYSTORE_PASSWORD_FLAG)
+                .long(SPECIFY_VOTING_KEYSTORE_PASSWORD_FLAG)
+                .help(
+                    "If present, the user will be prompted to enter the voting keystore \
+                    password that will be used to encrypt the voting keystores. If this \
+                    flag is not provided, a random password will be used. It is not \
+                    necessary to keep backups of voting keystore passwords if the \
+                    mnemonic is safely backed up.",
+                ),
+        )
+        .arg(
+            Arg::with_name(ETH1_WITHDRAWAL_ADDRESS_FLAG)
+                .long(ETH1_WITHDRAWAL_ADDRESS_FLAG)
+                .value_name("ETH1_ADDRESS")
+                .help(
+                    "If this field is set, the given eth1 address will be used to create the \
+                    withdrawal credentials. Otherwise, it will generate withdrawal credentials \
+                    with the mnemonic-derived withdrawal public key in EIP-2334 format.",
+                )
+                .conflicts_with(DISABLE_DEPOSITS_FLAG)
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(GAS_LIMIT_FLAG)
+                .long(GAS_LIMIT_FLAG)
+                .value_name("UINT64")
+                .help(
+                    "All created validators will use this gas limit. It is recommended \
+                    to leave this as the default value by not specifying this flag.",
+                )
+                .required(false)
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(FEE_RECIPIENT_FLAG)
+                .long(FEE_RECIPIENT_FLAG)
+                .value_name("ETH1_ADDRESS")
+                .help(
+                    "All created validators will use this value for the suggested \
+                    fee recipient. Omit this flag to use the default value from the VC.",
+                )
+                .required(false)
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(BUILDER_PROPOSALS_FLAG)
+                .long(BUILDER_PROPOSALS_FLAG)
+                .help(
+                    "When provided, all created validators will attempt to create \
+                    blocks via builder rather than the local EL.",
+                )
+                .required(false)
+                .possible_values(&["true", "false"])
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(BEACON_NODE_FLAG)
+                .long(BEACON_NODE_FLAG)
+                .value_name("HTTP_ADDRESS")
+                .help(
+                    "A HTTP(S) address of a beacon node using the beacon-API. \
+                    If this value is provided, an error will be raised if any validator \
+                    key here is already known as a validator by that beacon node. This helps \
+                    prevent the same validator being created twice and therefore slashable \
+                    conditions.",
+                )
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(FORCE_BLS_WITHDRAWAL_CREDENTIALS)
+                .takes_value(false)
+                .long(FORCE_BLS_WITHDRAWAL_CREDENTIALS)
+                .help(
+                    "If present, allows BLS withdrawal credentials rather than an execution \
+                    address. This is not recommended.",
+                ),
+        )
+}
+
+/// The CLI arguments are parsed into this struct before running the application. This step of
+/// indirection allows for testing the underlying logic without needing to parse CLI arguments.
+#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
+pub struct CreateConfig {
+    pub output_path: PathBuf,
+    pub first_index: u32,
+    pub count: u32,
+    pub deposit_gwei: u64,
+    pub mnemonic_path: Option<PathBuf>,
+    pub stdin_inputs: bool,
+    pub disable_deposits: bool,
+    pub specify_voting_keystore_password: bool,
+    pub eth1_withdrawal_address: Option<Address>,
+    pub builder_proposals: Option<bool>,
+    pub fee_recipient: Option<Address>,
+    pub gas_limit: Option<u64>,
+    pub bn_url: Option<SensitiveUrl>,
+    pub force_bls_withdrawal_credentials: bool,
+}
+
+impl CreateConfig {
+    fn from_cli(matches: &ArgMatches, spec: &ChainSpec) -> Result<Self, String> {
+        Ok(Self {
+            output_path: clap_utils::parse_required(matches, OUTPUT_PATH_FLAG)?,
+            deposit_gwei: clap_utils::parse_optional(matches, DEPOSIT_GWEI_FLAG)?
+                .unwrap_or(spec.max_effective_balance),
+            first_index: clap_utils::parse_required(matches, FIRST_INDEX_FLAG)?,
+            count: clap_utils::parse_required(matches, COUNT_FLAG)?,
+            mnemonic_path: clap_utils::parse_optional(matches, MNEMONIC_FLAG)?,
+            stdin_inputs: cfg!(windows) || matches.is_present(STDIN_INPUTS_FLAG),
+            disable_deposits: matches.is_present(DISABLE_DEPOSITS_FLAG),
+            specify_voting_keystore_password: matches
+                .is_present(SPECIFY_VOTING_KEYSTORE_PASSWORD_FLAG),
+            eth1_withdrawal_address: clap_utils::parse_optional(
+                matches,
+                ETH1_WITHDRAWAL_ADDRESS_FLAG,
+            )?,
+            builder_proposals: clap_utils::parse_optional(matches, BUILDER_PROPOSALS_FLAG)?,
+            fee_recipient: clap_utils::parse_optional(matches, FEE_RECIPIENT_FLAG)?,
+            gas_limit: clap_utils::parse_optional(matches, GAS_LIMIT_FLAG)?,
+            bn_url: clap_utils::parse_optional(matches, BEACON_NODE_FLAG)?,
+            force_bls_withdrawal_credentials: matches.is_present(FORCE_BLS_WITHDRAWAL_CREDENTIALS),
+        })
+    }
+}
+
+struct ValidatorsAndDeposits {
+    validators: Vec<ValidatorSpecification>,
+    deposits: Option<Vec<StandardDepositDataJson>>,
+}
+
+impl ValidatorsAndDeposits {
+    async fn new<'a, T: EthSpec>(config: CreateConfig, spec: &ChainSpec) -> Result<Self, String> {
+        let CreateConfig {
+            // The output path is handled upstream.
+            output_path: _,
+            first_index,
+            count,
+            deposit_gwei,
+            mnemonic_path,
+            stdin_inputs,
+            disable_deposits,
+            specify_voting_keystore_password,
+            eth1_withdrawal_address,
+            builder_proposals,
+            fee_recipient,
+            gas_limit,
+            bn_url,
+            force_bls_withdrawal_credentials,
+        } = config;
+
+        // Since Capella, it really doesn't make much sense to use BLS
+        // withdrawal credentials. Try to guide users away from doing so.
+        if eth1_withdrawal_address.is_none() && !force_bls_withdrawal_credentials {
+            return Err(format!(
+                "--{ETH1_WITHDRAWAL_ADDRESS_FLAG} is required. See --help for more information."
+            ));
+        }
+
+        if count == 0 {
+            return Err(format!("--{} cannot be 0", COUNT_FLAG));
+        }
+
+        let bn_http_client = if let Some(bn_url) = bn_url {
+            let bn_http_client =
+                BeaconNodeHttpClient::new(bn_url, Timeouts::set_all(BEACON_NODE_HTTP_TIMEOUT));
+
+            /*
+             * Print the version of the remote beacon node.
+             */
+            let version = bn_http_client
+                .get_node_version()
+                .await
+                .map_err(|e| format!("Failed to test connection to beacon node: {:?}", e))?
+                .data
+                .version;
+            eprintln!("Connected to beacon node running version {}", version);
+
+            /*
+             * Attempt to ensure that the beacon node is on the same network.
+             */
+            let bn_config = bn_http_client
+                .get_config_spec::<types::Config>()
+                .await
+                .map_err(|e| format!("Failed to get spec from beacon node: {:?}", e))?
+                .data;
+            if let Some(config_name) = &bn_config.config_name {
+                eprintln!("Beacon node is on {} network", config_name)
+            }
+            let bn_spec = bn_config
+                .apply_to_chain_spec::<T>(&T::default_spec())
+                .ok_or("Beacon node appears to be on an incorrect network")?;
+            if bn_spec.genesis_fork_version != spec.genesis_fork_version {
+                if let Some(config_name) = bn_spec.config_name {
+                    eprintln!("Beacon node is on {} network", config_name)
+                }
+                return Err("Beacon node appears to be on the wrong network".to_string());
+            }
+
+            Some(bn_http_client)
+        } else {
+            None
+        };
+
+        let mnemonic = read_mnemonic_from_cli(mnemonic_path, stdin_inputs)?;
+        let voting_keystore_password = if specify_voting_keystore_password {
+            eprintln!("Please enter a voting keystore password when prompted.");
+            Some(read_password_from_user(stdin_inputs)?)
+        } else {
+            None
+        };
+
+        /*
+         * Generate a wallet to be used for HD key generation.
+         */
+
+        // A random password is always appropriate for the wallet since it is ephemeral.
+        let wallet_password = random_password_string();
+        // A random password is always appropriate for the withdrawal keystore since we don't ever store
+        // it anywhere.
+        let withdrawal_keystore_password = random_password_string();
+        let mut wallet =
+            WalletBuilder::from_mnemonic(&mnemonic, wallet_password.as_ref(), "".to_string())
+                .map_err(|e| format!("Unable create seed from mnemonic: {:?}", e))?
+                .build()
+                .map_err(|e| format!("Unable to create wallet: {:?}", e))?;
+
+        /*
+         * Start deriving individual validators.
+         */
+
+        eprintln!(
+            "Starting derivation of {} keystores. Each keystore may take several seconds.",
+            count
+        );
+
+        let mut validators = Vec::with_capacity(count as usize);
+        let mut deposits = (!disable_deposits).then(Vec::new);
+
+        for (i, derivation_index) in (first_index..first_index + count).enumerate() {
+            // If the voting keystore password was not provided by the user then use a unique random
+            // string for each validator.
+            let voting_keystore_password = voting_keystore_password
+                .clone()
+                .unwrap_or_else(random_password_string);
+
+            // Set the wallet to the appropriate derivation index.
+            wallet
+                .set_nextaccount(derivation_index)
+                .map_err(|e| format!("Failure to set validator derivation index: {:?}", e))?;
+
+            // Derive the keystore from the HD wallet.
+            let keystores = wallet
+                .next_validator(
+                    wallet_password.as_ref(),
+                    voting_keystore_password.as_ref(),
+                    withdrawal_keystore_password.as_ref(),
+                )
+                .map_err(|e| format!("Failed to derive keystore {}: {:?}", i, e))?;
+            let voting_keystore = keystores.voting;
+            let voting_public_key = voting_keystore
+                .public_key()
+                .ok_or_else(|| {
+                    format!("Validator keystore at index {} is missing a public key", i)
+                })?
+                .into();
+
+            // If the user has provided a beacon node URL, check that the validator doesn't already
+            // exist in the beacon chain.
+            if let Some(bn_http_client) = &bn_http_client {
+                match bn_http_client
+                    .get_beacon_states_validator_id(
+                        StateId::Head,
+                        &ValidatorId::PublicKey(voting_public_key),
+                    )
+                    .await
+                {
+                    Ok(Some(_)) => {
+                        return Err(format!(
+                            "Validator {:?} at derivation index {} already exists in the beacon chain. \
+                            This indicates a slashing risk, be sure to never run the same validator on two \
+                            different validator clients. If you understand the risks and are certain you \
+                            wish to generate this validator again, omit the --{} flag.",
+                            voting_public_key, derivation_index, BEACON_NODE_FLAG
+                        ))?
+                    }
+                    Ok(None) => eprintln!(
+                        "{:?} was not found in the beacon chain",
+                        voting_public_key
+                    ),
+                    Err(e) => {
+                        return Err(format!(
+                            "Error checking if validator exists in beacon chain: {:?}",
+                            e
+                        ))
+                    }
+                }
+            }
+
+            if let Some(deposits) = &mut deposits {
+                // Decrypt the voting keystore so a deposit message can be signed.
+                let voting_keypair = voting_keystore
+                    .decrypt_keypair(voting_keystore_password.as_ref())
+                    .map_err(|e| format!("Failed to decrypt voting keystore {}: {:?}", i, e))?;
+
+                // Sanity check to ensure the keystore is reporting the correct public key.
+                if PublicKeyBytes::from(voting_keypair.pk.clone()) != voting_public_key {
+                    return Err(format!(
+                        "Mismatch for keystore public key and derived public key \
+                        for derivation index {}",
+                        derivation_index
+                    ));
+                }
+
+                let withdrawal_credentials =
+                    if let Some(eth1_withdrawal_address) = eth1_withdrawal_address {
+                        WithdrawalCredentials::eth1(eth1_withdrawal_address, spec)
+                    } else {
+                        // Decrypt the withdrawal keystore so withdrawal credentials can be created. It's
+                        // not strictly necessary to decrypt the keystore since we can read the pubkey
+                        // directly from the keystore. However we decrypt the keystore to be more certain
+                        // that we have access to the withdrawal keys.
+                        let withdrawal_keypair = keystores
+                            .withdrawal
+                            .decrypt_keypair(withdrawal_keystore_password.as_ref())
+                            .map_err(|e| {
+                                format!("Failed to decrypt withdrawal keystore {}: {:?}", i, e)
+                            })?;
+                        WithdrawalCredentials::bls(&withdrawal_keypair.pk, spec)
+                    };
+
+                // Create a JSON structure equivalent to the one generated by
+                // `ethereum/staking-deposit-cli`.
+                let json_deposit = StandardDepositDataJson::new(
+                    &voting_keypair,
+                    withdrawal_credentials.into(),
+                    deposit_gwei,
+                    spec,
+                )?;
+
+                deposits.push(json_deposit);
+            }
+
+            let validator = ValidatorSpecification {
+                voting_keystore: KeystoreJsonStr(voting_keystore),
+                voting_keystore_password: voting_keystore_password.clone(),
+                // New validators have no slashing protection history.
+                slashing_protection: None,
+                fee_recipient,
+                gas_limit,
+                builder_proposals,
+                // Allow the VC to choose a default "enabled" state. Since "enabled" is not part of
+                // the standard API, leaving this as `None` means we are not forced to use the
+                // non-standard API.
+                enabled: None,
+            };
+
+            eprintln!(
+                "Completed {}/{}: {:?}",
+                i.saturating_add(1),
+                count,
+                voting_public_key
+            );
+
+            validators.push(validator);
+        }
+
+        Ok(Self {
+            validators,
+            deposits,
+        })
+    }
+}
+
+pub async fn cli_run<'a, T: EthSpec>(
+    matches: &'a ArgMatches<'a>,
+    spec: &ChainSpec,
+    dump_config: DumpConfig,
+) -> Result<(), String> {
+    let config = CreateConfig::from_cli(matches, spec)?;
+    if dump_config.should_exit_early(&config)? {
+        Ok(())
+    } else {
+        run::<T>(config, spec).await
+    }
+}
+
+async fn run<'a, T: EthSpec>(config: CreateConfig, spec: &ChainSpec) -> Result<(), String> {
+    let output_path = config.output_path.clone();
+
+    if !output_path.exists() {
+        fs::create_dir(&output_path)
+            .map_err(|e| format!("Failed to create {:?} directory: {:?}", output_path, e))?;
+    } else if !output_path.is_dir() {
+        return Err(format!("{:?} must be a directory", output_path));
+    }
+
+    let validators_path = output_path.join(VALIDATORS_FILENAME);
+    if validators_path.exists() {
+        return Err(format!(
+            "{:?} already exists, refusing to overwrite",
+            validators_path
+        ));
+    }
+    let deposits_path = output_path.join(DEPOSITS_FILENAME);
+    if deposits_path.exists() {
+        return Err(format!(
+            "{:?} already exists, refusing to overwrite",
+            deposits_path
+        ));
+    }
+
+    let validators_and_deposits = ValidatorsAndDeposits::new::<T>(config, spec).await?;
+
+    eprintln!("Keystore generation complete");
+
+    write_to_json_file(&validators_path, &validators_and_deposits.validators)?;
+
+    if let Some(deposits) = &validators_and_deposits.deposits {
+        write_to_json_file(&deposits_path, deposits)?;
+    }
+
+    Ok(())
+}
+
+// The tests use crypto and are too slow in debug.
+#[cfg(not(debug_assertions))]
+#[cfg(test)]
+pub mod tests {
+    use super::*;
+    use eth2_network_config::Eth2NetworkConfig;
+    use regex::Regex;
+    use std::path::Path;
+    use std::str::FromStr;
+    use tempfile::{tempdir, TempDir};
+    use tree_hash::TreeHash;
+
+    type E = MainnetEthSpec;
+
+    const TEST_VECTOR_DEPOSIT_CLI_VERSION: &str = "2.3.0";
+
+    fn junk_execution_address() -> Option<Address> {
+        Some(Address::from_str("0x0f51bb10119727a7e5ea3538074fb341f56b09ad").unwrap())
+    }
+
+    pub struct TestBuilder {
+        spec: ChainSpec,
+        output_dir: TempDir,
+        mnemonic_dir: TempDir,
+        config: CreateConfig,
+    }
+
+    impl Default for TestBuilder {
+        fn default() -> Self {
+            Self::new(E::default_spec())
+        }
+    }
+
+    impl TestBuilder {
+        pub fn new(spec: ChainSpec) -> Self {
+            let output_dir = tempdir().unwrap();
+            let mnemonic_dir = tempdir().unwrap();
+            let mnemonic_path = mnemonic_dir.path().join("mnemonic");
+            fs::write(
+                &mnemonic_path,
+                "test test test test test test test test test test test waste",
+            )
+            .unwrap();
+
+            let config = CreateConfig {
+                output_path: output_dir.path().into(),
+                first_index: 0,
+                count: 1,
+                deposit_gwei: spec.max_effective_balance,
+                mnemonic_path: Some(mnemonic_path),
+                stdin_inputs: false,
+                disable_deposits: false,
+                specify_voting_keystore_password: false,
+                eth1_withdrawal_address: junk_execution_address(),
+                builder_proposals: None,
+                fee_recipient: None,
+                gas_limit: None,
+                bn_url: None,
+                force_bls_withdrawal_credentials: false,
+            };
+
+            Self {
+                spec,
+                output_dir,
+                mnemonic_dir,
+                config,
+            }
+        }
+
+        pub fn mutate_config<F: Fn(&mut CreateConfig)>(mut self, func: F) -> Self {
+            func(&mut self.config);
+            self
+        }
+
+        pub async fn run_test(self) -> TestResult {
+            let Self {
+                spec,
+                output_dir,
+                mnemonic_dir,
+                config,
+            } = self;
+
+            let result = run::<E>(config.clone(), &spec).await;
+
+            if result.is_ok() {
+                let validators_file_contents =
+                    fs::read_to_string(output_dir.path().join(VALIDATORS_FILENAME)).unwrap();
+                let validators: Vec<ValidatorSpecification> =
+                    serde_json::from_str(&validators_file_contents).unwrap();
+
+                assert_eq!(validators.len(), config.count as usize);
+
+                for (i, validator) in validators.iter().enumerate() {
+                    let voting_keystore = &validator.voting_keystore.0;
+                    let keypair = voting_keystore
+                        .decrypt_keypair(validator.voting_keystore_password.as_ref())
+                        .unwrap();
+                    assert_eq!(keypair.pk, voting_keystore.public_key().unwrap());
+                    assert_eq!(
+                        voting_keystore.path().unwrap(),
+                        format!("m/12381/3600/{}/0/0", config.first_index as usize + i)
+                    );
+                    assert!(validator.slashing_protection.is_none());
+                    assert_eq!(validator.fee_recipient, config.fee_recipient);
+                    assert_eq!(validator.gas_limit, config.gas_limit);
+                    assert_eq!(validator.builder_proposals, config.builder_proposals);
+                    assert_eq!(validator.enabled, None);
+                }
+
+                let deposits_path = output_dir.path().join(DEPOSITS_FILENAME);
+                if config.disable_deposits {
+                    assert!(!deposits_path.exists());
+                } else {
+                    let deposits_file_contents = fs::read_to_string(&deposits_path).unwrap();
+                    let deposits: Vec<StandardDepositDataJson> =
+                        serde_json::from_str(&deposits_file_contents).unwrap();
+
+                    assert_eq!(deposits.len(), config.count as usize);
+
+                    for (validator, deposit) in validators.iter().zip(deposits.iter()) {
+                        let validator_pubkey = validator.voting_keystore.0.public_key().unwrap();
+                        assert_eq!(deposit.pubkey, validator_pubkey.clone().into());
+                        if let Some(address) = config.eth1_withdrawal_address {
+                            assert_eq!(
+                                deposit.withdrawal_credentials.as_bytes()[0],
+                                spec.eth1_address_withdrawal_prefix_byte
+                            );
+                            assert_eq!(
+                                &deposit.withdrawal_credentials.as_bytes()[12..],
+                                address.as_bytes()
+                            );
+                        } else {
+                            assert_eq!(
+                                deposit.withdrawal_credentials.as_bytes()[0],
+                                spec.bls_withdrawal_prefix_byte
+                            );
+                        }
+                        assert_eq!(deposit.amount, config.deposit_gwei);
+                        let deposit_message = DepositData {
+                            pubkey: deposit.pubkey,
+                            withdrawal_credentials: deposit.withdrawal_credentials,
+                            amount: deposit.amount,
+                            signature: SignatureBytes::empty(),
+                        }
+                        .as_deposit_message();
+                        assert!(deposit.signature.decompress().unwrap().verify(
+                            &validator_pubkey,
+                            deposit_message.signing_root(spec.get_deposit_domain())
+                        ));
+                        assert_eq!(deposit.fork_version, spec.genesis_fork_version);
+                        assert_eq!(&deposit.network_name, spec.config_name.as_ref().unwrap());
+                        assert_eq!(
+                            deposit.deposit_message_root,
+                            deposit_message.tree_hash_root()
+                        );
+                        assert_eq!(
+                            deposit.deposit_data_root,
+                            DepositData {
+                                pubkey: deposit.pubkey,
+                                withdrawal_credentials: deposit.withdrawal_credentials,
+                                amount: deposit.amount,
+                                signature: deposit.signature.clone()
+                            }
+                            .tree_hash_root()
+                        );
+                    }
+                }
+            }
+
+            // The directory containing the mnemonic can now be removed.
+            drop(mnemonic_dir);
+
+            TestResult { result, output_dir }
+        }
+    }
+
+    #[must_use] // Use the `assert_ok` or `assert_err` fns to "use" this value.
+    pub struct TestResult {
+        pub result: Result<(), String>,
+        pub output_dir: TempDir,
+    }
+
+    impl TestResult {
+        pub fn validators_file_path(&self) -> PathBuf {
+            self.output_dir.path().join(VALIDATORS_FILENAME)
+        }
+
+        pub fn validators(&self) -> Vec<ValidatorSpecification> {
+            let contents = fs::read_to_string(self.validators_file_path()).unwrap();
+            serde_json::from_str(&contents).unwrap()
+        }
+
+        fn assert_ok(self) {
+            assert_eq!(self.result, Ok(()))
+        }
+
+        fn assert_err(self) {
+            assert!(self.result.is_err())
+        }
+    }
+
+    #[tokio::test]
+    async fn default_test_values() {
+        TestBuilder::default().run_test().await.assert_ok();
+    }
+
+    #[tokio::test]
+    async fn no_eth1_address_without_force() {
+        TestBuilder::default()
+            .mutate_config(|config| {
+                config.eth1_withdrawal_address = None;
+                config.force_bls_withdrawal_credentials = false
+            })
+            .run_test()
+            .await
+            .assert_err();
+    }
+
+    #[tokio::test]
+    async fn bls_withdrawal_credentials() {
+        TestBuilder::default()
+            .mutate_config(|config| {
+                config.eth1_withdrawal_address = None;
+                config.force_bls_withdrawal_credentials = true
+            })
+            .run_test()
+            .await
+            .assert_ok();
+    }
+
+    #[tokio::test]
+    async fn default_test_values_deposits_disabled() {
+        TestBuilder::default()
+            .mutate_config(|config| config.disable_deposits = true)
+            .run_test()
+            .await
+            .assert_ok();
+    }
+
+    #[tokio::test]
+    async fn count_is_zero() {
+        TestBuilder::default()
+            .mutate_config(|config| config.count = 0)
+            .run_test()
+            .await
+            .assert_err();
+    }
+
+    #[tokio::test]
+    async fn eth1_withdrawal_addresses() {
+        TestBuilder::default()
+            .mutate_config(|config| {
+                config.count = 2;
+                config.eth1_withdrawal_address = junk_execution_address();
+            })
+            .run_test()
+            .await
+            .assert_ok();
+    }
+
+    #[tokio::test]
+    async fn non_zero_first_index() {
+        TestBuilder::default()
+            .mutate_config(|config| {
+                config.first_index = 2;
+                config.count = 2;
+            })
+            .run_test()
+            .await
+            .assert_ok();
+    }
+
+    #[tokio::test]
+    async fn misc_modifications() {
+        TestBuilder::default()
+            .mutate_config(|config| {
+                config.deposit_gwei = 42;
+                config.builder_proposals = Some(true);
+                config.gas_limit = Some(1337);
+            })
+            .run_test()
+            .await
+            .assert_ok();
+    }
+
+    #[tokio::test]
+    async fn bogus_bn_url() {
+        TestBuilder::default()
+            .mutate_config(|config| {
+                config.bn_url =
+                    Some(SensitiveUrl::from_str("http://sdjfvwfhsdhfschwkeyfwhwlga.com").unwrap());
+            })
+            .run_test()
+            .await
+            .assert_err();
+    }
+
+    #[tokio::test]
+    async fn staking_deposit_cli_vectors() {
+        let vectors_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+            .join("test_vectors")
+            .join("vectors");
+        for entry in fs::read_dir(vectors_dir).unwrap() {
+            let entry = entry.unwrap();
+            let file_name = entry.file_name();
+            let vector_name = file_name.to_str().unwrap();
+            let path = entry.path();
+            // Leave this `println!` so we can tell which test fails.
+            println!("Running test {}", vector_name);
+            run_test_vector(vector_name, &path).await;
+        }
+    }
+
+    async fn run_test_vector<P: AsRef<Path>>(name: &str, vectors_path: P) {
+        /*
+         * Parse the test vector name into a set of test parameters.
+         */
+        let re = Regex::new(r"(.*)_(.*)_(.*)_(.*)_(.*)_(.*)_(.*)").unwrap();
+        let capture = re.captures_iter(name).next().unwrap();
+        let network = capture.get(1).unwrap().as_str();
+        let first = u32::from_str(capture.get(3).unwrap().as_str()).unwrap();
+        let count = u32::from_str(capture.get(5).unwrap().as_str()).unwrap();
+        let uses_eth1 = bool::from_str(capture.get(7).unwrap().as_str()).unwrap();
+
+        /*
+         * Use the test parameters to generate equivalent files "locally" (i.e., with our code).
+         */
+
+        let spec = Eth2NetworkConfig::constant(network)
+            .unwrap()
+            .unwrap()
+            .chain_spec::<E>()
+            .unwrap();
+
+        let test_result = TestBuilder::new(spec)
+            .mutate_config(|config| {
+                config.first_index = first;
+                config.count = count;
+                if uses_eth1 {
+                    config.eth1_withdrawal_address = Some(
+                        Address::from_str("0x0f51bb10119727a7e5ea3538074fb341f56b09ad").unwrap(),
+                    );
+                } else {
+                    config.eth1_withdrawal_address = None;
+                    config.force_bls_withdrawal_credentials = true;
+                }
+            })
+            .run_test()
+            .await;
+        let TestResult { result, output_dir } = test_result;
+        result.expect("local generation should succeed");
+
+        /*
+         * Ensure the deposit data is identical when parsed as JSON.
+         */
+
+        let local_deposits = {
+            let path = output_dir.path().join(DEPOSITS_FILENAME);
+            let contents = fs::read_to_string(&path).unwrap();
+            let mut deposits: Vec<StandardDepositDataJson> =
+                serde_json::from_str(&contents).unwrap();
+            for deposit in &mut deposits {
+                // Ensures we can match test vectors.
+                deposit.deposit_cli_version = TEST_VECTOR_DEPOSIT_CLI_VERSION.to_string();
+
+                // We use "prater" and the vectors use "goerli" now. The two names refer to the same
+                // network so there should be no issue here.
+                if deposit.network_name == "prater" {
+                    deposit.network_name = "goerli".to_string();
+                }
+            }
+            deposits
+        };
+        let vector_deposits: Vec<StandardDepositDataJson> = {
+            let path = fs::read_dir(vectors_path.as_ref().join("validator_keys"))
+                .unwrap()
+                .find_map(|entry| {
+                    let entry = entry.unwrap();
+                    let file_name = entry.file_name();
+                    if file_name.to_str().unwrap().starts_with("deposit_data") {
+                        Some(entry.path())
+                    } else {
+                        None
+                    }
+                })
+                .unwrap();
+            let contents = fs::read_to_string(path).unwrap();
+            serde_json::from_str(&contents).unwrap()
+        };
+
+        assert_eq!(local_deposits, vector_deposits);
+
+        /*
+         * Note: we don't check the keystores generated by the deposit-cli since there is little
+         * value in this.
+         *
+         * If we check the deposits then we are verifying the signature across the deposit message.
+         * This implicitly verifies that the keypair generated by the deposit-cli is identical to
+         * the one created by Lighthouse.
+         */
+    }
+}

validator_manager/src/import_validators.rs

@@ -0,0 +1,436 @@
+use super::common::*;
+use crate::DumpConfig;
+use clap::{App, Arg, ArgMatches};
+use eth2::{lighthouse_vc::std_types::ImportKeystoreStatus, SensitiveUrl};
+use serde::{Deserialize, Serialize};
+use std::fs;
+use std::path::PathBuf;
+
+pub const CMD: &str = "import";
+pub const VALIDATORS_FILE_FLAG: &str = "validators-file";
+pub const VC_URL_FLAG: &str = "vc-url";
+pub const VC_TOKEN_FLAG: &str = "vc-token";
+
+pub const DETECTED_DUPLICATE_MESSAGE: &str = "Duplicate validator detected!";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about(
+            "Uploads validators to a validator client using the HTTP API. The validators \
+                are defined in a JSON file which can be generated using the \"create-validators\" \
+                command.",
+        )
+        .arg(
+            Arg::with_name(VALIDATORS_FILE_FLAG)
+                .long(VALIDATORS_FILE_FLAG)
+                .value_name("PATH_TO_JSON_FILE")
+                .help(
+                    "The path to a JSON file containing a list of validators to be \
+                    imported to the validator client. This file is usually named \
+                    \"validators.json\".",
+                )
+                .required(true)
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(VC_URL_FLAG)
+                .long(VC_URL_FLAG)
+                .value_name("HTTP_ADDRESS")
+                .help(
+                    "A HTTP(S) address of a validator client using the keymanager-API. \
+                    If this value is not supplied then a 'dry run' will be conducted where \
+                    no changes are made to the validator client.",
+                )
+                .default_value("http://localhost:5062")
+                .requires(VC_TOKEN_FLAG)
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(VC_TOKEN_FLAG)
+                .long(VC_TOKEN_FLAG)
+                .value_name("PATH")
+                .help("The file containing a token required by the validator client.")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(IGNORE_DUPLICATES_FLAG)
+                .takes_value(false)
+                .long(IGNORE_DUPLICATES_FLAG)
+                .help(
+                    "If present, ignore any validators which already exist on the VC. \
+                    Without this flag, the process will terminate without making any changes. \
+                    This flag should be used with caution, whilst it does not directly cause \
+                    slashable conditions, it might be an indicator that something is amiss. \
+                    Users should also be careful to avoid submitting duplicate deposits for \
+                    validators that already exist on the VC.",
+                ),
+        )
+}
+
+#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
+pub struct ImportConfig {
+    pub validators_file_path: PathBuf,
+    pub vc_url: SensitiveUrl,
+    pub vc_token_path: PathBuf,
+    pub ignore_duplicates: bool,
+}
+
+impl ImportConfig {
+    fn from_cli(matches: &ArgMatches) -> Result<Self, String> {
+        Ok(Self {
+            validators_file_path: clap_utils::parse_required(matches, VALIDATORS_FILE_FLAG)?,
+            vc_url: clap_utils::parse_required(matches, VC_URL_FLAG)?,
+            vc_token_path: clap_utils::parse_required(matches, VC_TOKEN_FLAG)?,
+            ignore_duplicates: matches.is_present(IGNORE_DUPLICATES_FLAG),
+        })
+    }
+}
+
+pub async fn cli_run<'a>(
+    matches: &'a ArgMatches<'a>,
+    dump_config: DumpConfig,
+) -> Result<(), String> {
+    let config = ImportConfig::from_cli(matches)?;
+    if dump_config.should_exit_early(&config)? {
+        Ok(())
+    } else {
+        run(config).await
+    }
+}
+
+async fn run<'a>(config: ImportConfig) -> Result<(), String> {
+    let ImportConfig {
+        validators_file_path,
+        vc_url,
+        vc_token_path,
+        ignore_duplicates,
+    } = config;
+
+    if !validators_file_path.exists() {
+        return Err(format!("Unable to find file at {:?}", validators_file_path));
+    }
+
+    let validators_file = fs::OpenOptions::new()
+        .read(true)
+        .create(false)
+        .open(&validators_file_path)
+        .map_err(|e| format!("Unable to open {:?}: {:?}", validators_file_path, e))?;
+    let validators: Vec<ValidatorSpecification> = serde_json::from_reader(&validators_file)
+        .map_err(|e| {
+            format!(
+                "Unable to parse JSON in {:?}: {:?}",
+                validators_file_path, e
+            )
+        })?;
+
+    let count = validators.len();
+
+    let (http_client, _keystores) = vc_http_client(vc_url.clone(), &vc_token_path).await?;
+
+    eprintln!(
+        "Starting to submit {} validators to VC, each validator may take several seconds",
+        count
+    );
+
+    for (i, validator) in validators.into_iter().enumerate() {
+        match validator.upload(&http_client, ignore_duplicates).await {
+            Ok(status) => {
+                match status.status {
+                    ImportKeystoreStatus::Imported => {
+                        eprintln!("Uploaded keystore {} of {} to the VC", i + 1, count)
+                    }
+                    ImportKeystoreStatus::Duplicate => {
+                        if ignore_duplicates {
+                            eprintln!("Re-uploaded keystore {} of {} to the VC", i + 1, count)
+                        } else {
+                            eprintln!(
+                                "Keystore {} of {} was uploaded to the VC, but it was a duplicate. \
+                                Exiting now, use --{} to allow duplicates.",
+                                i + 1, count, IGNORE_DUPLICATES_FLAG
+                            );
+                            return Err(DETECTED_DUPLICATE_MESSAGE.to_string());
+                        }
+                    }
+                    ImportKeystoreStatus::Error => {
+                        eprintln!(
+                            "Upload of keystore {} of {} failed with message: {:?}. \
+                                A potential solution is run this command again \
+                                using the --{} flag, however care should be taken to ensure \
+                                that there are no duplicate deposits submitted.",
+                            i + 1,
+                            count,
+                            status.message,
+                            IGNORE_DUPLICATES_FLAG
+                        );
+                        return Err(format!("Upload failed with {:?}", status.message));
+                    }
+                }
+            }
+            e @ Err(UploadError::InvalidPublicKey) => {
+                eprintln!("Validator {} has an invalid public key", i);
+                return Err(format!("{:?}", e));
+            }
+            ref e @ Err(UploadError::DuplicateValidator(voting_public_key)) => {
+                eprintln!(
+                    "Duplicate validator {:?} already exists on the destination validator client. \
+                    This may indicate that some validators are running in two places at once, which \
+                    can lead to slashing. If you are certain that there is no risk, add the --{} flag.",
+                    voting_public_key, IGNORE_DUPLICATES_FLAG
+                );
+                return Err(format!("{:?}", e));
+            }
+            Err(UploadError::FailedToListKeys(e)) => {
+                eprintln!(
+                    "Failed to list keystores. Some keys may have been imported whilst \
+                    others may not have been imported. A potential solution is run this command again \
+                    using the --{} flag, however care should be taken to ensure that there are no \
+                    duplicate deposits submitted.",
+                    IGNORE_DUPLICATES_FLAG
+                );
+                return Err(format!("{:?}", e));
+            }
+            Err(UploadError::KeyUploadFailed(e)) => {
+                eprintln!(
+                    "Failed to upload keystore. Some keys may have been imported whilst \
+                    others may not have been imported. A potential solution is run this command again \
+                    using the --{} flag, however care should be taken to ensure that there are no \
+                    duplicate deposits submitted.",
+                    IGNORE_DUPLICATES_FLAG
+                );
+                return Err(format!("{:?}", e));
+            }
+            Err(UploadError::IncorrectStatusCount(count)) => {
+                eprintln!(
+                    "Keystore was uploaded, however the validator client returned an invalid response. \
+                    A potential solution is run this command again using the --{} flag, however care \
+                    should be taken to ensure that there are no duplicate deposits submitted.",
+                    IGNORE_DUPLICATES_FLAG
+                );
+                return Err(format!(
+                    "Invalid status count in import response: {}",
+                    count
+                ));
+            }
+            Err(UploadError::FeeRecipientUpdateFailed(e)) => {
+                eprintln!(
+                    "Failed to set fee recipient for validator {}. This value may need \
+                    to be set manually. Continuing with other validators. Error was {:?}",
+                    i, e
+                );
+            }
+            Err(UploadError::PatchValidatorFailed(e)) => {
+                eprintln!(
+                    "Failed to set some values on validator {} (e.g., builder, enabled or gas limit. \
+                    These values value may need to be set manually. Continuing with other validators. \
+                    Error was {:?}",
+                    i, e
+                );
+            }
+        }
+    }
+
+    Ok(())
+}
+
+// The tests use crypto and are too slow in debug.
+#[cfg(not(debug_assertions))]
+#[cfg(test)]
+pub mod tests {
+    use super::*;
+    use crate::create_validators::tests::TestBuilder as CreateTestBuilder;
+    use std::fs;
+    use tempfile::{tempdir, TempDir};
+    use validator_client::http_api::{test_utils::ApiTester, Config as HttpConfig};
+
+    const VC_TOKEN_FILE_NAME: &str = "vc_token.json";
+
+    pub struct TestBuilder {
+        import_config: ImportConfig,
+        pub vc: ApiTester,
+        /// Holds the temp directory owned by the `CreateTestBuilder` so it doesn't get cleaned-up
+        /// before we can read it.
+        create_dir: Option<TempDir>,
+        _dir: TempDir,
+    }
+
+    impl TestBuilder {
+        pub async fn new() -> Self {
+            Self::new_with_http_config(ApiTester::default_http_config()).await
+        }
+
+        pub async fn new_with_http_config(http_config: HttpConfig) -> Self {
+            let dir = tempdir().unwrap();
+            let vc = ApiTester::new_with_http_config(http_config).await;
+            let vc_token_path = dir.path().join(VC_TOKEN_FILE_NAME);
+            fs::write(&vc_token_path, &vc.api_token).unwrap();
+
+            Self {
+                import_config: ImportConfig {
+                    // This field will be overwritten later on.
+                    validators_file_path: dir.path().into(),
+                    vc_url: vc.url.clone(),
+                    vc_token_path,
+                    ignore_duplicates: false,
+                },
+                vc,
+                create_dir: None,
+                _dir: dir,
+            }
+        }
+
+        pub fn mutate_import_config<F: Fn(&mut ImportConfig)>(mut self, func: F) -> Self {
+            func(&mut self.import_config);
+            self
+        }
+
+        pub async fn create_validators(mut self, count: u32, first_index: u32) -> Self {
+            let create_result = CreateTestBuilder::default()
+                .mutate_config(|config| {
+                    config.count = count;
+                    config.first_index = first_index;
+                })
+                .run_test()
+                .await;
+            assert!(
+                create_result.result.is_ok(),
+                "precondition: validators are created"
+            );
+            self.import_config.validators_file_path = create_result.validators_file_path();
+            self.create_dir = Some(create_result.output_dir);
+            self
+        }
+
+        /// Imports validators without running the entire test suite in `Self::run_test`. This is
+        /// useful for simulating duplicate imports.
+        pub async fn import_validators_without_checks(self) -> Self {
+            run(self.import_config.clone()).await.unwrap();
+            self
+        }
+
+        pub async fn run_test(self) -> TestResult {
+            let result = run(self.import_config.clone()).await;
+
+            if result.is_ok() {
+                self.vc.ensure_key_cache_consistency().await;
+
+                let local_validators: Vec<ValidatorSpecification> = {
+                    let contents =
+                        fs::read_to_string(&self.import_config.validators_file_path).unwrap();
+                    serde_json::from_str(&contents).unwrap()
+                };
+                let list_keystores_response = self.vc.client.get_keystores().await.unwrap().data;
+
+                assert_eq!(
+                    local_validators.len(),
+                    list_keystores_response.len(),
+                    "vc should have exactly the number of validators imported"
+                );
+
+                for local_validator in &local_validators {
+                    let local_keystore = &local_validator.voting_keystore.0;
+                    let local_pubkey = local_keystore.public_key().unwrap().into();
+                    let remote_validator = list_keystores_response
+                        .iter()
+                        .find(|validator| validator.validating_pubkey == local_pubkey)
+                        .expect("validator must exist on VC");
+                    assert_eq!(&remote_validator.derivation_path, &local_keystore.path());
+                    assert_eq!(remote_validator.readonly, Some(false));
+                }
+            }
+
+            TestResult {
+                result,
+                vc: self.vc,
+            }
+        }
+    }
+
+    #[must_use] // Use the `assert_ok` or `assert_err` fns to "use" this value.
+    pub struct TestResult {
+        pub result: Result<(), String>,
+        pub vc: ApiTester,
+    }
+
+    impl TestResult {
+        fn assert_ok(self) {
+            assert_eq!(self.result, Ok(()))
+        }
+
+        fn assert_err_contains(self, msg: &str) {
+            assert!(self.result.unwrap_err().contains(msg))
+        }
+    }
+
+    #[tokio::test]
+    async fn create_one_validator() {
+        TestBuilder::new()
+            .await
+            .create_validators(1, 0)
+            .await
+            .run_test()
+            .await
+            .assert_ok();
+    }
+
+    #[tokio::test]
+    async fn create_three_validators() {
+        TestBuilder::new()
+            .await
+            .create_validators(3, 0)
+            .await
+            .run_test()
+            .await
+            .assert_ok();
+    }
+
+    #[tokio::test]
+    async fn create_one_validator_with_offset() {
+        TestBuilder::new()
+            .await
+            .create_validators(1, 42)
+            .await
+            .run_test()
+            .await
+            .assert_ok();
+    }
+
+    #[tokio::test]
+    async fn create_three_validators_with_offset() {
+        TestBuilder::new()
+            .await
+            .create_validators(3, 1337)
+            .await
+            .run_test()
+            .await
+            .assert_ok();
+    }
+
+    #[tokio::test]
+    async fn import_duplicates_when_disallowed() {
+        TestBuilder::new()
+            .await
+            .create_validators(1, 0)
+            .await
+            .import_validators_without_checks()
+            .await
+            .run_test()
+            .await
+            .assert_err_contains("DuplicateValidator");
+    }
+
+    #[tokio::test]
+    async fn import_duplicates_when_allowed() {
+        TestBuilder::new()
+            .await
+            .mutate_import_config(|config| {
+                config.ignore_duplicates = true;
+            })
+            .create_validators(1, 0)
+            .await
+            .import_validators_without_checks()
+            .await
+            .run_test()
+            .await
+            .assert_ok();
+    }
+}

validator_manager/src/lib.rs

@@ -0,0 +1,85 @@
+use clap::App;
+use clap::ArgMatches;
+use common::write_to_json_file;
+use environment::Environment;
+use serde::Serialize;
+use std::path::PathBuf;
+use types::EthSpec;
+
+pub mod common;
+pub mod create_validators;
+pub mod import_validators;
+pub mod move_validators;
+
+pub const CMD: &str = "validator_manager";
+
+/// This flag is on the top-level `lighthouse` binary.
+const DUMP_CONFIGS_FLAG: &str = "dump-config";
+
+/// Used only in testing, this allows a command to dump its configuration to a file and then exit
+/// successfully. This allows for testing how the CLI arguments translate to some configuration.
+pub enum DumpConfig {
+    Disabled,
+    Enabled(PathBuf),
+}
+
+impl DumpConfig {
+    /// Returns `Ok(true)` if the configuration was successfully written to a file and the
+    /// application should exit successfully without doing anything else.
+    pub fn should_exit_early<T: Serialize>(&self, config: &T) -> Result<bool, String> {
+        match self {
+            DumpConfig::Disabled => Ok(false),
+            DumpConfig::Enabled(dump_path) => {
+                dbg!(dump_path);
+                write_to_json_file(dump_path, config)?;
+                Ok(true)
+            }
+        }
+    }
+}
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .visible_aliases(&["vm", "validator-manager", CMD])
+        .about("Utilities for managing a Lighthouse validator client via the HTTP API.")
+        .subcommand(create_validators::cli_app())
+        .subcommand(import_validators::cli_app())
+        .subcommand(move_validators::cli_app())
+}
+
+/// Run the account manager, returning an error if the operation did not succeed.
+pub fn run<'a, T: EthSpec>(matches: &'a ArgMatches<'a>, env: Environment<T>) -> Result<(), String> {
+    let context = env.core_context();
+    let spec = context.eth2_config.spec;
+    let dump_config = clap_utils::parse_optional(matches, DUMP_CONFIGS_FLAG)?
+        .map(DumpConfig::Enabled)
+        .unwrap_or_else(|| DumpConfig::Disabled);
+
+    context
+        .executor
+        // This `block_on_dangerous` call reasonable since it is at the very highest level of the
+        // application, the rest of which is all async. All other functions below this should be
+        // async and should never call `block_on_dangerous` themselves.
+        .block_on_dangerous(
+            async {
+                match matches.subcommand() {
+                    (create_validators::CMD, Some(matches)) => {
+                        create_validators::cli_run::<T>(matches, &spec, dump_config).await
+                    }
+                    (import_validators::CMD, Some(matches)) => {
+                        import_validators::cli_run(matches, dump_config).await
+                    }
+                    (move_validators::CMD, Some(matches)) => {
+                        move_validators::cli_run(matches, dump_config).await
+                    }
+                    ("", _) => Err("No command supplied. See --help.".to_string()),
+                    (unknown, _) => Err(format!(
+                        "{} is not a valid {} command. See --help.",
+                        unknown, CMD
+                    )),
+                }
+            },
+            "validator_manager",
+        )
+        .ok_or("Shutting down")?
+}

validator_manager/test_vectors/.gitignore

@@ -0,0 +1 @@
+tmp/

validator_manager/test_vectors/generate.py

@@ -0,0 +1,123 @@
+# This script uses the `ethereum/staking-deposit-cli` tool to generate
+# deposit data files which are then used for testing by Lighthouse.
+#
+# To generate vectors, simply run this Python script:
+#
+# `python generate.py`
+#
+import os
+import sys
+import shutil
+import subprocess
+from subprocess import Popen, PIPE, STDOUT
+
+
+NUM_VALIDATORS=3
+TEST_MNEMONIC = "test test test test test test test test test test test waste"
+WALLET_NAME="test_wallet"
+
+
+tmp_dir = os.path.join(".", "tmp")
+mnemonic_path = os.path.join(tmp_dir, "mnemonic.txt")
+sdc_dir = os.path.join(tmp_dir, "sdc")
+sdc_git_dir = os.path.join(sdc_dir, "staking-deposit-cli")
+vectors_dir = os.path.join(".", "vectors")
+
+
+def setup():
+    cleanup()
+
+    if os.path.exists(vectors_dir):
+        shutil.rmtree(vectors_dir)
+
+    os.mkdir(tmp_dir)
+    os.mkdir(sdc_dir)
+    os.mkdir(vectors_dir)
+
+    setup_sdc()
+    with open(mnemonic_path, "x") as file:
+        file.write(TEST_MNEMONIC)
+
+
+def cleanup():
+    if os.path.exists(tmp_dir):
+        shutil.rmtree(tmp_dir)
+
+    # Remove all the keystores since we don't use them in testing.
+    if os.path.exists(vectors_dir):
+        for root, dirs, files in os.walk(vectors_dir):
+            for file in files:
+                if file.startswith("keystore"):
+                    os.remove(os.path.join(root, file))
+
+
+def setup_sdc():
+    result = subprocess.run([
+        "git",
+        "clone",
+        "--single-branch",
+        "https://github.com/ethereum/staking-deposit-cli.git",
+        str(sdc_git_dir)
+    ])
+    assert(result.returncode == 0)
+    result = subprocess.run([
+        "pip",
+        "install",
+        "-r",
+        "requirements.txt",
+    ], cwd=sdc_git_dir)
+    assert(result.returncode == 0)
+    result = subprocess.run([
+        "python",
+        "setup.py",
+        "install",
+    ], cwd=sdc_git_dir)
+    assert(result.returncode == 0)
+
+
+def sdc_generate(network, first_index, count, eth1_withdrawal_address=None):
+    if eth1_withdrawal_address is not None:
+        eth1_flags = ['--eth1_withdrawal_address', eth1_withdrawal_address]
+        uses_eth1 = True
+    else:
+        eth1_flags = []
+        uses_eth1 = False
+
+    test_name = "{}_first_{}_count_{}_eth1_{}".format(network, first_index, count,
+                                                      str(uses_eth1).lower())
+    output_dir = os.path.join(vectors_dir, test_name)
+    os.mkdir(output_dir)
+
+    command = [
+        '/bin/sh',
+        'deposit.sh',
+        '--language', 'english',
+        '--non_interactive',
+        'existing-mnemonic',
+        '--validator_start_index', str(first_index),
+        '--num_validators', str(count),
+        '--mnemonic', TEST_MNEMONIC,
+        '--chain', network,
+        '--keystore_password', 'MyPassword',
+        '--folder', os.path.abspath(output_dir),
+    ] + eth1_flags
+
+    print("Running " + test_name)
+    process = Popen(command, cwd=sdc_git_dir, text=True, stdin = PIPE)
+    process.wait()
+
+
+def test_network(network):
+    sdc_generate(network, first_index=0, count=1)
+    sdc_generate(network, first_index=0, count=2)
+    sdc_generate(network, first_index=12, count=1)
+    sdc_generate(network, first_index=99, count=2)
+    sdc_generate(network, first_index=1024, count=3)
+    sdc_generate(network, first_index=0, count=2,
+                 eth1_withdrawal_address="0x0f51bb10119727a7e5ea3538074fb341f56b09ad")
+
+
+setup()
+test_network("mainnet")
+test_network("prater")
+cleanup()

validator_manager/test_vectors/vectors/mainnet_first_0_count_1_eth1_false/validator_keys/deposit_data-1660803666.json

@@ -0,0 +1 @@
+[{"pubkey": "88b6b3a9b391fa5593e8bce8d06102df1a56248368086929709fbb4a8570dc6a560febeef8159b19789e9c1fd13572f0", "withdrawal_credentials": "0049b6188ed20314309f617dd4030b8ddfac3c6e65759a03c226a13b2fe4cc72", "amount": 32000000000, "signature": "8ac88247c1b431a2d1eb2c5f00e7b8467bc21d6dc267f1af9ef727a12e32b4299e3b289ae5734a328b3202478dd746a80bf9e15a2217240dca1fc1b91a6b7ff7a0f5830d9a2610c1c30f19912346271357c21bd9af35a74097ebbdda2ddaf491", "deposit_message_root": "a9bc1d21cc009d9b10782a07213e37592c0d235463ed0117dec755758da90d51", "deposit_data_root": "807a20b2801eabfd9065c1b74ed6ae3e991a1ab770e4eaf268f30b37cfd2cbd7", "fork_version": "00000000", "network_name": "mainnet", "deposit_cli_version": "2.3.0"}]

validator_manager/test_vectors/vectors/mainnet_first_0_count_2_eth1_false/validator_keys/deposit_data-1660803669.json

@@ -0,0 +1 @@
+[{"pubkey": "88b6b3a9b391fa5593e8bce8d06102df1a56248368086929709fbb4a8570dc6a560febeef8159b19789e9c1fd13572f0", "withdrawal_credentials": "0049b6188ed20314309f617dd4030b8ddfac3c6e65759a03c226a13b2fe4cc72", "amount": 32000000000, "signature": "8ac88247c1b431a2d1eb2c5f00e7b8467bc21d6dc267f1af9ef727a12e32b4299e3b289ae5734a328b3202478dd746a80bf9e15a2217240dca1fc1b91a6b7ff7a0f5830d9a2610c1c30f19912346271357c21bd9af35a74097ebbdda2ddaf491", "deposit_message_root": "a9bc1d21cc009d9b10782a07213e37592c0d235463ed0117dec755758da90d51", "deposit_data_root": "807a20b2801eabfd9065c1b74ed6ae3e991a1ab770e4eaf268f30b37cfd2cbd7", "fork_version": "00000000", "network_name": "mainnet", "deposit_cli_version": "2.3.0"}, {"pubkey": "a33ab9d93fb53c4f027944aaa11a13be0c150b7cc2e379d85d1ed4db38d178b4e4ebeae05832158b8c746c1961da00ce", "withdrawal_credentials": "00ad3748cbd1adc855c2bdab431f7e755a21663f4f6447ac888e5855c588af5a", "amount": 32000000000, "signature": "84b9fc8f260a1488c4c9a438f875edfa2bac964d651b2bc886d8442829b13f89752e807c8ca9bae9d50b1b506d3a64730015dd7f91e271ff9c1757d1996dcf6082fe5205cf6329fa2b6be303c21b66d75be608757a123da6ee4a4f14c01716d7", "deposit_message_root": "c5271aba974c802ff5b02b11fa33b545d7f430ff3b85c0f9eeef4cd59d83abf3", "deposit_data_root": "cd991ea8ff32e6b3940aed43b476c720fc1abd3040893b77a8a3efb306320d4c", "fork_version": "00000000", "network_name": "mainnet", "deposit_cli_version": "2.3.0"}]

validator_manager/test_vectors/vectors/mainnet_first_0_count_2_eth1_true/validator_keys/deposit_data-1660803684.json

@@ -0,0 +1 @@
+[{"pubkey": "88b6b3a9b391fa5593e8bce8d06102df1a56248368086929709fbb4a8570dc6a560febeef8159b19789e9c1fd13572f0", "withdrawal_credentials": "0100000000000000000000000f51bb10119727a7e5ea3538074fb341f56b09ad", "amount": 32000000000, "signature": "a8461b58a5a5a0573c4af37da6ee4ba63e35894cffad6797d4a2c80f8f2c79d2c30c0de0299d8edde76e0c3f3e6d4f1e03cc377969f56d8760717d6e86f9316da9375573ce7bb87a8520daedb13c49284377f7a4f64a70aa2ca44b1581d47e20", "deposit_message_root": "62967565d11471da4af7769911926cd1826124048036b25616216f99bc320f13", "deposit_data_root": "d26d642a880ff8a109260fe69681840f6e1868c8c1cd2163a1db5a094e8db03a", "fork_version": "00000000", "network_name": "mainnet", "deposit_cli_version": "2.3.0"}, {"pubkey": "a33ab9d93fb53c4f027944aaa11a13be0c150b7cc2e379d85d1ed4db38d178b4e4ebeae05832158b8c746c1961da00ce", "withdrawal_credentials": "0100000000000000000000000f51bb10119727a7e5ea3538074fb341f56b09ad", "amount": 32000000000, "signature": "93a398c09143203beb94c9223c7e18f36e5ea36090875284b222c2fcb16982e6f2e26f27ca9d30e3c6f6b5ad44857fc50f531925f4736810712f68a9d7a9c0eb664a851180f3b7d2e44a35717d43b3d3e4fd555354fa1dfa92f451870f36084d", "deposit_message_root": "ce110433298ffb78d827d67dcc13655344a139cb7e3ce10b341937c0a76b25b7", "deposit_data_root": "7c7617a2c11870ec49e975b3691b9f822d63938df38555161e23aa245b150c66", "fork_version": "00000000", "network_name": "mainnet", "deposit_cli_version": "2.3.0"}]

validator_manager/test_vectors/vectors/mainnet_first_1024_count_3_eth1_false/validator_keys/deposit_data-1660803679.json

@@ -0,0 +1 @@
+[{"pubkey": "92ca8dddba4ae7ada6584c377fc53fb978ad9d5ee8db585b18e226c27682b326b3c68e10f5d99a453e233268c144e0ef", "withdrawal_credentials": "00dd4f8bfd1a48be288c2af8bb7315f6198900b5b3f56df010420d5328e682cb", "amount": 32000000000, "signature": "a0a96851892b257c032284928641021e58e0bcd277c3da5a2c41bcce6633d144781e4761261138277b5a8cf0ead59cce073e5a3bbc4704a37abf8cd1e290dc52e56cb0c334303945ebbb79be453c8177937e44e08f980679f1a2997fe58d2d86", "deposit_message_root": "5421d9177b4d035e6525506509ab702c5f458c53458dad437097b37cb8209b43", "deposit_data_root": "2bedaf48f8315d8631defc97c1c4c05a8152e2dc3fe779fc8e800dd67bd839a2", "fork_version": "00000000", "network_name": "mainnet", "deposit_cli_version": "2.3.0"}, {"pubkey": "86474cd2874663445ef0ee02aca81b2b942a383fd4c7085fa675388e26c67afc0fef44a8666d46f571723e349ae4a0cb", "withdrawal_credentials": "001c31aa161ed1d3c481c1ee8f3ad1853217296a15877917fe3c2f680580ac01", "amount": 32000000000, "signature": "b469179ad8ba9d6ad71b99a3c7ae662d9b77cca3ee53b20ab2eb20beee31874ad47224e94e75578fa6ecd30c1d40a0b300053817f934169d84425691edf13216445fbc6dd9b0953ad3af20c834fba63c1f50c0b0f92dd8bf383cd2cc8e0431f1", "deposit_message_root": "279271f7065c83868c37021c32c014516b21e6188fb2cee4e8543c5d38427698", "deposit_data_root": "69862477671957ab0b3f1167c5cd550c107132a0079eb70eaa4bc5c5fe06b5a0", "fork_version": "00000000", "network_name": "mainnet", "deposit_cli_version": "2.3.0"}, {"pubkey": "997e27aa262238beb01464434694a466321b5270297bdfdb944b65a3b6617b6ce2613628ac35a8f4cf2e9b4b55c46ef8", "withdrawal_credentials": "0097fffee9cf9fd91a6fa89af90e73f1cb8b8a043e742afaeb2e57b83b0845fe", "amount": 32000000000, "signature": "a8b05626657ce5b1801e0824aaeb21de2e1a11bc16cad6100ac911bcb873aaf7e7282f1f8465df4aaea998a1a4e1645f075e7e65f8c6b8688b0162f86be2128541f91fc9feb628bcab3b4afec1f7aeccaba04aaa54dc17c738233d360f94b97e", "deposit_message_root": "187e177721bfdd8ea13cb52c8de2dead29164a0e093efb640457a0e6ac918191", "deposit_data_root": "34ef32901d793cd9a0a3d93e7ee40e7be9abe6fb26f0b49a86b8ff29dc649930", "fork_version": "00000000", "network_name": "mainnet", "deposit_cli_version": "2.3.0"}]

validator_manager/test_vectors/vectors/mainnet_first_12_count_1_eth1_false/validator_keys/deposit_data-1660803672.json

@@ -0,0 +1 @@
+[{"pubkey": "8b181759a027c09a409ef24f6b35db213982c2474e2017f3851d76b1c4e560a4238072f67a0c22cb667f940da4ea9ec9", "withdrawal_credentials": "00cbec90e8570679f565bd4645f73a078981067a705564283e61c93c81707842", "amount": 32000000000, "signature": "a57299cde3c2ea8dc17ad3ce5a38a5f6de69d198599150dc4df02624ba1d8672440d02c0d27c3dc3b8c9f86c679571ab14c798426acd9b059895f1f5887bdee805fb4e31bd8f93ec9e78403c23d7924f23eae6af056154f35fee03bf9ffe0e98", "deposit_message_root": "fcdf3d94740766299a95b3e477e64abadff6ab8978400578f241c93eb367b938", "deposit_data_root": "246619823b45d80f53a30404542ec4be447d4e268cc0afcdf480e6a846d58411", "fork_version": "00000000", "network_name": "mainnet", "deposit_cli_version": "2.3.0"}]

validator_manager/test_vectors/vectors/mainnet_first_99_count_2_eth1_false/validator_keys/deposit_data-1660803675.json

@@ -0,0 +1 @@
+[{"pubkey": "a57a4ed429e415b862cc758e75c93936e3f6339640d0763b969ba133a82c03717827fbdd8ec42fc862ed50e3b5b528dc", "withdrawal_credentials": "00864081ef2f5aec1aa667872615e25027f1fdc256a4948b6318cf75a8d635a3", "amount": 32000000000, "signature": "8ca8a6f30b4346d7b9912e3dcd820652bc472511f89d91fd102acfb0c8df1cfc7a2629f44170727e126e88f2847fe5c9081b13fb0838a2b2343a95cabf16f57708fc0cf846bc5307209ae976c34500cc826ff48ab64169d8bebec99dded5dd1d", "deposit_message_root": "c08d0ecd085bc0f50c35f1b34d8b8937b2b9c8a172a9808de70f8d448c526f07", "deposit_data_root": "c0c6cd40b43ea0fe7fcc284de9acd9c1bd001bb88c059c155393af22a6c85d46", "fork_version": "00000000", "network_name": "mainnet", "deposit_cli_version": "2.3.0"}, {"pubkey": "a2801622bc391724989004b5de78cb85746f85a303572691ecc945d9f5c61ec512127e58482e0dfcb4de77be3294ab01", "withdrawal_credentials": "00edff674c66a7f58285554e700183aeee5e740691de8087f7ce4d81f3597108", "amount": 32000000000, "signature": "8c0784645c611b4f514a6519b737f2d02df3eba0e04cd30efebffcca769af8cc599ce28e4421cefe665ec31d3c34e44c174e0cca4891d8196796085e712459b45e411efecd07cf3258f1d6309a07a6dd52a0ae186e6184d37bf11cee36ec84e8", "deposit_message_root": "f5a530bee9698c2447961ecd210184fbb130bbb8e8916988d802d47e3b147842", "deposit_data_root": "c57790b77ef97318d4ec7b97ea07ea458d08209ba372bfe76171e2ece22d6130", "fork_version": "00000000", "network_name": "mainnet", "deposit_cli_version": "2.3.0"}]

validator_manager/test_vectors/vectors/prater_first_0_count_1_eth1_false/validator_keys/deposit_data-1660803687.json

@@ -0,0 +1 @@
+[{"pubkey": "88b6b3a9b391fa5593e8bce8d06102df1a56248368086929709fbb4a8570dc6a560febeef8159b19789e9c1fd13572f0", "withdrawal_credentials": "0049b6188ed20314309f617dd4030b8ddfac3c6e65759a03c226a13b2fe4cc72", "amount": 32000000000, "signature": "a940e0142ad9b56a1310326137347d1ada275b31b3748af4accc63bd189573376615be8e8ae047766c6d10864e54b2e7098177598edf3a043eb560bbdf1a1c12588375a054d1323a0900e2286d0993cde9675e5b74523e6e8e03715cc96b3ce5", "deposit_message_root": "a9bc1d21cc009d9b10782a07213e37592c0d235463ed0117dec755758da90d51", "deposit_data_root": "28484efb20c961a1354689a556d4c352fe9deb24684efdb32d22e1af17e2a45d", "fork_version": "00001020", "network_name": "goerli", "deposit_cli_version": "2.3.0"}]

validator_manager/test_vectors/vectors/prater_first_0_count_2_eth1_false/validator_keys/deposit_data-1660803690.json

@@ -0,0 +1 @@
+[{"pubkey": "88b6b3a9b391fa5593e8bce8d06102df1a56248368086929709fbb4a8570dc6a560febeef8159b19789e9c1fd13572f0", "withdrawal_credentials": "0049b6188ed20314309f617dd4030b8ddfac3c6e65759a03c226a13b2fe4cc72", "amount": 32000000000, "signature": "a940e0142ad9b56a1310326137347d1ada275b31b3748af4accc63bd189573376615be8e8ae047766c6d10864e54b2e7098177598edf3a043eb560bbdf1a1c12588375a054d1323a0900e2286d0993cde9675e5b74523e6e8e03715cc96b3ce5", "deposit_message_root": "a9bc1d21cc009d9b10782a07213e37592c0d235463ed0117dec755758da90d51", "deposit_data_root": "28484efb20c961a1354689a556d4c352fe9deb24684efdb32d22e1af17e2a45d", "fork_version": "00001020", "network_name": "goerli", "deposit_cli_version": "2.3.0"}, {"pubkey": "a33ab9d93fb53c4f027944aaa11a13be0c150b7cc2e379d85d1ed4db38d178b4e4ebeae05832158b8c746c1961da00ce", "withdrawal_credentials": "00ad3748cbd1adc855c2bdab431f7e755a21663f4f6447ac888e5855c588af5a", "amount": 32000000000, "signature": "87b4b4e9c923aa9e1687219e9df0e838956ee6e15b7ab18142467430d00940dc7aa243c9996e85125dfe72d9dbdb00a30a36e16a2003ee0c86f29c9f5d74f12bfe5b7f62693dbf5187a093555ae8d6b48acd075788549c4b6a249b397af24cd0", "deposit_message_root": "c5271aba974c802ff5b02b11fa33b545d7f430ff3b85c0f9eeef4cd59d83abf3", "deposit_data_root": "ea80b639356a03f6f58e4acbe881fabefc9d8b93375a6aa7e530c77d7e45d3e4", "fork_version": "00001020", "network_name": "goerli", "deposit_cli_version": "2.3.0"}]

validator_manager/test_vectors/vectors/prater_first_0_count_2_eth1_true/validator_keys/deposit_data-1660803705.json

@@ -0,0 +1 @@
+[{"pubkey": "88b6b3a9b391fa5593e8bce8d06102df1a56248368086929709fbb4a8570dc6a560febeef8159b19789e9c1fd13572f0", "withdrawal_credentials": "0100000000000000000000000f51bb10119727a7e5ea3538074fb341f56b09ad", "amount": 32000000000, "signature": "ab32595d8201c2b4e8173aece9151fdc15f4d2ad36008462d0416598ddbf0f37ed0877f06d284a9669e73dbc0885bd2207fe64385e95a4488dc2bcb2c324d5c20da3248a6244463583dfbba8db20805765421e59cb56b0bc3ee6d24a9218216d", "deposit_message_root": "62967565d11471da4af7769911926cd1826124048036b25616216f99bc320f13", "deposit_data_root": "b4df3a3a26dd5f6eb32999d8a7051a7d1a8573a16553d4b45ee706a0d59c1066", "fork_version": "00001020", "network_name": "goerli", "deposit_cli_version": "2.3.0"}, {"pubkey": "a33ab9d93fb53c4f027944aaa11a13be0c150b7cc2e379d85d1ed4db38d178b4e4ebeae05832158b8c746c1961da00ce", "withdrawal_credentials": "0100000000000000000000000f51bb10119727a7e5ea3538074fb341f56b09ad", "amount": 32000000000, "signature": "9655e195eda5517efe6f36bcebd45250c889a4177d7bf5fcd59598d2d03f37f038b5ee2ec079a30a8382ea42f351943f08a6f006bab9c2130db2742bd7315c8ad5aa1f03a0801c26d4c9efdef71c4c59c449c7f9b21fa62600ab8f5f1e2b938a", "deposit_message_root": "ce110433298ffb78d827d67dcc13655344a139cb7e3ce10b341937c0a76b25b7", "deposit_data_root": "7661474fba11bfb453274f62df022cab3c0b6f4a58af4400f6bce83c9cb5fcb8", "fork_version": "00001020", "network_name": "goerli", "deposit_cli_version": "2.3.0"}]

validator_manager/test_vectors/vectors/prater_first_1024_count_3_eth1_false/validator_keys/deposit_data-1660803701.json

@@ -0,0 +1 @@
+[{"pubkey": "92ca8dddba4ae7ada6584c377fc53fb978ad9d5ee8db585b18e226c27682b326b3c68e10f5d99a453e233268c144e0ef", "withdrawal_credentials": "00dd4f8bfd1a48be288c2af8bb7315f6198900b5b3f56df010420d5328e682cb", "amount": 32000000000, "signature": "b5dae79ce8f3d7326b46f93182981c5f3d64257a457f038caa78ec8e5cc25a9fdac52c7beb221ab2a3205404131366ad18e1e13801393b3d486819e8cca96128bf1244884a91d05dced092c74bc1e7259788f30dd3432df15f3d2f629645f345", "deposit_message_root": "5421d9177b4d035e6525506509ab702c5f458c53458dad437097b37cb8209b43", "deposit_data_root": "94213d76aba9e6a434589d4939dd3764e0832df78f66d30db22a760c14ba1b89", "fork_version": "00001020", "network_name": "goerli", "deposit_cli_version": "2.3.0"}, {"pubkey": "86474cd2874663445ef0ee02aca81b2b942a383fd4c7085fa675388e26c67afc0fef44a8666d46f571723e349ae4a0cb", "withdrawal_credentials": "001c31aa161ed1d3c481c1ee8f3ad1853217296a15877917fe3c2f680580ac01", "amount": 32000000000, "signature": "816f38a321c4f84ad5187eda58f6d9c1fd1e81c860ed1722bdb76b920fdd430a1e814b9bb893837ae3b38ad738684fbf1795fa687f617c52121472b1ac8d2e34e5c1127186233a8833ffb54c509d9e52cb7242c6c6a65b5e496296b3caa90d89", "deposit_message_root": "279271f7065c83868c37021c32c014516b21e6188fb2cee4e8543c5d38427698", "deposit_data_root": "7ad1d059d69794680a1deef5e72c33827f0c449a5f0917095821c0343572789d", "fork_version": "00001020", "network_name": "goerli", "deposit_cli_version": "2.3.0"}, {"pubkey": "997e27aa262238beb01464434694a466321b5270297bdfdb944b65a3b6617b6ce2613628ac35a8f4cf2e9b4b55c46ef8", "withdrawal_credentials": "0097fffee9cf9fd91a6fa89af90e73f1cb8b8a043e742afaeb2e57b83b0845fe", "amount": 32000000000, "signature": "95d20c35484dea6b2a0bd7c2da2d2e810d7829e14c03657b2524adfc2111aa5ed95908ecb975ff75ff742c68ce8df417016c048959b0f807675430f6d981478e26d48e594e0830a0406da9817f8a1ecb94bd8be1f9281eeb5e952a82173c72bb", "deposit_message_root": "187e177721bfdd8ea13cb52c8de2dead29164a0e093efb640457a0e6ac918191", "deposit_data_root": "83abfb2a166f7af708526a9bdd2767c4be3cd231c9bc4e2f047a80df88a2860c", "fork_version": "00001020", "network_name": "goerli", "deposit_cli_version": "2.3.0"}]

validator_manager/test_vectors/vectors/prater_first_12_count_1_eth1_false/validator_keys/deposit_data-1660803693.json

@@ -0,0 +1 @@
+[{"pubkey": "8b181759a027c09a409ef24f6b35db213982c2474e2017f3851d76b1c4e560a4238072f67a0c22cb667f940da4ea9ec9", "withdrawal_credentials": "00cbec90e8570679f565bd4645f73a078981067a705564283e61c93c81707842", "amount": 32000000000, "signature": "8f75836ceb390dd4fc8c16bc4be52ca09b9c5aa0ab5bc16dcfdb344787b29ddfd76d877b0a2330bc8e904b233397c6bd124845d1b868e4951cb6daacea023c986bdf0c6ac28d73f65681d941ea96623bc23acc7c84dcfc1304686240d9171cfc", "deposit_message_root": "fcdf3d94740766299a95b3e477e64abadff6ab8978400578f241c93eb367b938", "deposit_data_root": "3011f5cac32f13e86ecc061e89ed6675c27a46ab6ecb1ec6f6e5f133ae1d0287", "fork_version": "00001020", "network_name": "goerli", "deposit_cli_version": "2.3.0"}]

validator_manager/test_vectors/vectors/prater_first_99_count_2_eth1_false/validator_keys/deposit_data-1660803696.json

@@ -0,0 +1 @@
+[{"pubkey": "a57a4ed429e415b862cc758e75c93936e3f6339640d0763b969ba133a82c03717827fbdd8ec42fc862ed50e3b5b528dc", "withdrawal_credentials": "00864081ef2f5aec1aa667872615e25027f1fdc256a4948b6318cf75a8d635a3", "amount": 32000000000, "signature": "a7706e102bfb0b986a5c8050044f7e221919463149771a92c3ca46ff7d4564867db48eaf89b5237fed8db2cdb9c9c057099d0982bbdb3fbfcbe0ab7259ad3f31f7713692b78ee25e6251982e7081d049804632b70b8a24d8c3e59b624a0bd221", "deposit_message_root": "c08d0ecd085bc0f50c35f1b34d8b8937b2b9c8a172a9808de70f8d448c526f07", "deposit_data_root": "8a26fbee0c3a99fe090af1fce68afc525b4e7efa70df72abaa91f29148b2f672", "fork_version": "00001020", "network_name": "goerli", "deposit_cli_version": "2.3.0"}, {"pubkey": "a2801622bc391724989004b5de78cb85746f85a303572691ecc945d9f5c61ec512127e58482e0dfcb4de77be3294ab01", "withdrawal_credentials": "00edff674c66a7f58285554e700183aeee5e740691de8087f7ce4d81f3597108", "amount": 32000000000, "signature": "8b7aa5b0e97d15ec8c2281b919fde9e064f6ac064b163445ea99441ab063f9d10534bfde861b5606021ae46614ff075e0c2305ce5a6cbcc9f0bc8e7df1a177c4d969a5ed4ac062b0ea959bdac963fe206b73565a1a3937adcca736c6117c15f0", "deposit_message_root": "f5a530bee9698c2447961ecd210184fbb130bbb8e8916988d802d47e3b147842", "deposit_data_root": "d38575167a94b516455c5b7e36d24310a612fa0f4580446c5f9d45e4e94f0642", "fork_version": "00001020", "network_name": "goerli", "deposit_cli_version": "2.3.0"}]