cerc-io/lighthouse
10
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity
c973bfc90c
lighthouse/common/account_utils/src/validator_definitions.rs
realbigsean 6c2d8b2262 Builder Specs v0.2.0 (#3134) 
## Issue Addressed

https://github.com/sigp/lighthouse/issues/3091

Extends https://github.com/sigp/lighthouse/pull/3062, adding pre-bellatrix block support on blinded endpoints and allowing the normal proposal flow (local payload construction) on blinded endpoints. This resulted in better fallback logic because the VC will not have to switch endpoints on failure in the BN <> Builder API, the BN can just fallback immediately and without repeating block processing that it shouldn't need to. We can also keep VC fallback from the VC<>BN API's blinded endpoint to full endpoint.

## Proposed Changes

- Pre-bellatrix blocks on blinded endpoints
- Add a new `PayloadCache` to the execution layer
- Better fallback-from-builder logic

## Todos

- [x] Remove VC transition logic
- [x] Add logic to only enable builder flow after Merge transition finalization
- [x] Tests
- [x] Fix metrics
- [x] Rustdocs


Co-authored-by: Mac L <mjladson@pm.me>
Co-authored-by: realbigsean <sean@sigmaprime.io>
2022-07-30 00:22:37 +00:00

622 lines
23 KiB
Rust
Raw Blame History

//! Provides a file format for defining validators that should be initialized by this validator.
//!
//! Serves as the source-of-truth of which validators this validator client should attempt (or not
//! attempt) to load into the `crate::intialized_validators::InitializedValidators` struct.
use crate::{default_keystore_password_path, write_file_via_temporary, ZeroizeString};
use directory::ensure_dir_exists;
use eth2_keystore::Keystore;
use regex::Regex;
use serde_derive::{Deserialize, Serialize};
use slog::{error, Logger};
use std::collections::HashSet;
use std::fs::{self, File};
use std::io;
use std::path::{Path, PathBuf};
use types::{graffiti::GraffitiString, Address, PublicKey};
use validator_dir::VOTING_KEYSTORE_FILE;
/// The file name for the serialized `ValidatorDefinitions` struct.
pub const CONFIG_FILENAME: &str = "validator_definitions.yml";
/// The temporary file name for the serialized `ValidatorDefinitions` struct.
///
/// This is used to achieve an atomic update of the contents on disk, without truncation.
/// See: https://github.com/sigp/lighthouse/issues/2159
pub const CONFIG_TEMP_FILENAME: &str = ".validator_definitions.yml.tmp";
#[derive(Debug)]
pub enum Error {
/// The config file could not be opened.
UnableToOpenFile(io::Error),
/// The config file could not be parsed as YAML.
UnableToParseFile(serde_yaml::Error),
/// There was an error whilst performing the recursive keystore search function.
UnableToSearchForKeystores(io::Error),
/// The config file could not be serialized as YAML.
UnableToEncodeFile(serde_yaml::Error),
/// The config file or temp file could not be written to the filesystem.
UnableToWriteFile(filesystem::Error),
/// The public key from the keystore is invalid.
InvalidKeystorePubkey,
/// The keystore was unable to be opened.
UnableToOpenKeystore(eth2_keystore::Error),
/// The validator directory could not be created.
UnableToCreateValidatorDir(PathBuf),
}
#[derive(Clone, PartialEq, Serialize, Deserialize, Hash, Eq)]
pub struct Web3SignerDefinition {
pub url: String,
/// Path to a .pem file.
#[serde(skip_serializing_if = "Option::is_none")]
pub root_certificate_path: Option<PathBuf>,
/// Specifies a request timeout.
///
/// The timeout is applied from when the request starts connecting until the response body has finished.
#[serde(skip_serializing_if = "Option::is_none")]
pub request_timeout_ms: Option<u64>,
/// Path to a PKCS12 file.
#[serde(skip_serializing_if = "Option::is_none")]
pub client_identity_path: Option<PathBuf>,
/// Password for the PKCS12 file.
///
/// An empty password will be used if this is omitted.
#[serde(skip_serializing_if = "Option::is_none")]
pub client_identity_password: Option<String>,
}
/// Defines how the validator client should attempt to sign messages for this validator.
#[derive(Clone, PartialEq, Serialize, Deserialize)]
#[serde(tag = "type")]
pub enum SigningDefinition {
/// A validator that is defined by an EIP-2335 keystore on the local filesystem.
#[serde(rename = "local_keystore")]
LocalKeystore {
voting_keystore_path: PathBuf,
#[serde(skip_serializing_if = "Option::is_none")]
voting_keystore_password_path: Option<PathBuf>,
#[serde(skip_serializing_if = "Option::is_none")]
voting_keystore_password: Option<ZeroizeString>,
},
/// A validator that defers to a Web3Signer HTTP server for signing.
///
/// https://github.com/ConsenSys/web3signer
#[serde(rename = "web3signer")]
Web3Signer(Web3SignerDefinition),
}
impl SigningDefinition {
pub fn is_local_keystore(&self) -> bool {
matches!(self, SigningDefinition::LocalKeystore { .. })
}
}
/// A validator that may be initialized by this validator client.
///
/// Presently there is only a single variant, however we expect more variants to arise (e.g.,
/// remote signing).
#[derive(Clone, PartialEq, Serialize, Deserialize)]
pub struct ValidatorDefinition {
pub enabled: bool,
pub voting_public_key: PublicKey,
#[serde(default)]
#[serde(skip_serializing_if = "Option::is_none")]
pub graffiti: Option<GraffitiString>,
#[serde(default)]
#[serde(skip_serializing_if = "Option::is_none")]
pub suggested_fee_recipient: Option<Address>,
#[serde(default)]
#[serde(skip_serializing_if = "Option::is_none")]
pub gas_limit: Option<u64>,
#[serde(default)]
#[serde(skip_serializing_if = "Option::is_none")]
pub builder_proposals: Option<bool>,
#[serde(default)]
pub description: String,
#[serde(flatten)]
pub signing_definition: SigningDefinition,
}
impl ValidatorDefinition {
/// Create a new definition for a voting keystore at the given `voting_keystore_path` that can
/// be unlocked with `voting_keystore_password`.
///
/// ## Notes
///
/// This function does not check the password against the keystore.
pub fn new_keystore_with_password<P: AsRef<Path>>(
voting_keystore_path: P,
voting_keystore_password: Option<ZeroizeString>,
graffiti: Option<GraffitiString>,
suggested_fee_recipient: Option<Address>,
gas_limit: Option<u64>,
builder_proposals: Option<bool>,
) -> Result<Self, Error> {
let voting_keystore_path = voting_keystore_path.as_ref().into();
let keystore =
Keystore::from_json_file(&voting_keystore_path).map_err(Error::UnableToOpenKeystore)?;
let voting_public_key = keystore.public_key().ok_or(Error::InvalidKeystorePubkey)?;
Ok(ValidatorDefinition {
enabled: true,
voting_public_key,
description: keystore.description().unwrap_or("").to_string(),
graffiti,
suggested_fee_recipient,
gas_limit,
builder_proposals,
signing_definition: SigningDefinition::LocalKeystore {
voting_keystore_path,
voting_keystore_password_path: None,
voting_keystore_password,
},
})
}
}
/// A list of `ValidatorDefinition` that serves as a serde-able configuration file which defines a
/// list of validators to be initialized by this validator client.
#[derive(Default, Serialize, Deserialize)]
pub struct ValidatorDefinitions(Vec<ValidatorDefinition>);
impl From<Vec<ValidatorDefinition>> for ValidatorDefinitions {
fn from(vec: Vec<ValidatorDefinition>) -> Self {
Self(vec)
}
}
impl ValidatorDefinitions {
/// Open an existing file or create a new, empty one if it does not exist.
pub fn open_or_create<P: AsRef<Path>>(validators_dir: P) -> Result<Self, Error> {
ensure_dir_exists(validators_dir.as_ref()).map_err(|_| {
Error::UnableToCreateValidatorDir(PathBuf::from(validators_dir.as_ref()))
})?;
let config_path = validators_dir.as_ref().join(CONFIG_FILENAME);
if !config_path.exists() {
let this = Self::default();
this.save(&validators_dir)?;
}
Self::open(validators_dir)
}
/// Open an existing file, returning an error if the file does not exist.
pub fn open<P: AsRef<Path>>(validators_dir: P) -> Result<Self, Error> {
let config_path = validators_dir.as_ref().join(CONFIG_FILENAME);
let file = File::options()
.write(true)
.read(true)
.create_new(false)
.open(&config_path)
.map_err(Error::UnableToOpenFile)?;
serde_yaml::from_reader(file).map_err(Error::UnableToParseFile)
}
/// Perform a recursive, exhaustive search through `validators_dir` and add any keystores
/// matching the `validator_dir::VOTING_KEYSTORE_FILE` file name.
///
/// Returns the count of *new* keystores that were added to `self` during this search.
///
/// ## Notes
///
/// Determines the path for the password file based upon the scheme defined by
/// `account_utils::default_keystore_password_path`.
///
/// If a keystore cannot be parsed the function does not exit early. Instead it logs an `error`
/// and continues searching.
pub fn discover_local_keystores<P: AsRef<Path>>(
&mut self,
validators_dir: P,
secrets_dir: P,
log: &Logger,
) -> Result<usize, Error> {
let mut keystore_paths = vec![];
recursively_find_voting_keystores(validators_dir, &mut keystore_paths)
.map_err(Error::UnableToSearchForKeystores)?;
let known_paths: HashSet<&PathBuf> = self
.0
.iter()
.filter_map(|def| match &def.signing_definition {
SigningDefinition::LocalKeystore {
voting_keystore_path,
..
} => Some(voting_keystore_path),
// A Web3Signer validator does not use a local keystore file.
SigningDefinition::Web3Signer { .. } => None,
})
.collect();
let known_pubkeys: HashSet<PublicKey> = self
.0
.iter()
.map(|def| def.voting_public_key.clone())
.collect();
let mut new_defs = keystore_paths
.into_iter()
.filter_map(|voting_keystore_path| {
if known_paths.contains(&voting_keystore_path) {
return None;
}
let keystore_result = File::options()
.read(true)
.create(false)
.open(&voting_keystore_path)
.map_err(|e| format!("{:?}", e))
.and_then(|file| {
Keystore::from_json_reader(file).map_err(|e| format!("{:?}", e))
});
let keystore = match keystore_result {
Ok(keystore) => keystore,
Err(e) => {
error!(
log,
"Unable to read validator keystore";
"error" => e,
"keystore" => format!("{:?}", voting_keystore_path)
);
return None;
}
};
let voting_keystore_password_path = Some(default_keystore_password_path(
&keystore,
secrets_dir.as_ref(),
))
.filter(|path| path.exists());
let voting_public_key = match keystore.public_key() {
Some(pubkey) => {
if known_pubkeys.contains(&pubkey) {
return None;
} else {
pubkey
}
}
None => {
error!(
log,
"Invalid keystore public key";
"keystore" => format!("{:?}", voting_keystore_path)
);
return None;
}
};
Some(ValidatorDefinition {
enabled: true,
voting_public_key,
description: keystore.description().unwrap_or("").to_string(),
graffiti: None,
suggested_fee_recipient: None,
gas_limit: None,
builder_proposals: None,
signing_definition: SigningDefinition::LocalKeystore {
voting_keystore_path,
voting_keystore_password_path,
voting_keystore_password: None,
},
})
})
.collect::<Vec<_>>();
let new_defs_count = new_defs.len();
self.0.append(&mut new_defs);
Ok(new_defs_count)
}
/// Encodes `self` as a YAML string and atomically writes it to the `CONFIG_FILENAME` file in
/// the `validators_dir` directory.
///
/// Will create a new file if it does not exist or overwrite any existing file.
pub fn save<P: AsRef<Path>>(&self, validators_dir: P) -> Result<(), Error> {
let config_path = validators_dir.as_ref().join(CONFIG_FILENAME);
let temp_path = validators_dir.as_ref().join(CONFIG_TEMP_FILENAME);
let bytes = serde_yaml::to_vec(self).map_err(Error::UnableToEncodeFile)?;
write_file_via_temporary(&config_path, &temp_path, &bytes)
.map_err(Error::UnableToWriteFile)?;
Ok(())
}
/// Retain only the definitions matching the given predicate.
pub fn retain(&mut self, f: impl FnMut(&ValidatorDefinition) -> bool) {
self.0.retain(f);
}
/// Adds a new `ValidatorDefinition` to `self`.
pub fn push(&mut self, def: ValidatorDefinition) {
self.0.push(def)
}
/// Returns a slice of all `ValidatorDefinition` in `self`.
pub fn as_slice(&self) -> &[ValidatorDefinition] {
self.0.as_slice()
}
/// Returns a mutable slice of all `ValidatorDefinition` in `self`.
pub fn as_mut_slice(&mut self) -> &mut [ValidatorDefinition] {
self.0.as_mut_slice()
}
}
/// Perform an exhaustive tree search of `dir`, adding any discovered voting keystore paths to
/// `matches`.
///
/// ## Errors
///
/// Returns with an error immediately if any filesystem error is raised.
pub fn recursively_find_voting_keystores<P: AsRef<Path>>(
dir: P,
matches: &mut Vec<PathBuf>,
) -> Result<(), io::Error> {
fs::read_dir(dir)?.try_for_each(|dir_entry| {
let dir_entry = dir_entry?;
let file_type = dir_entry.file_type()?;
if file_type.is_dir() {
recursively_find_voting_keystores(dir_entry.path(), matches)?
} else if file_type.is_file()
&& dir_entry
.file_name()
.to_str()
.map_or(false, is_voting_keystore)
{
matches.push(dir_entry.path())
}
Ok(())
})
}
/// Returns `true` if we should consider the `file_name` to represent a voting keystore.
pub fn is_voting_keystore(file_name: &str) -> bool {
// All formats end with `.json`.
if !file_name.ends_with(".json") {
return false;
}
// The format used by Lighthouse.
if file_name == VOTING_KEYSTORE_FILE {
return true;
}
// The format exported by the `eth2.0-deposit-cli` library.
//
// Reference to function that generates keystores:
//
// https://github.com/ethereum/eth2.0-deposit-cli/blob/7cebff15eac299b3b1b090c896dd3410c8463450/eth2deposit/credentials.py#L58-L62
//
// Since we include the key derivation path of `m/12381/3600/x/0/0` this should only ever match
// with a voting keystore and never a withdrawal keystore.
//
// Key derivation path reference:
//
// https://eips.ethereum.org/EIPS/eip-2334
if Regex::new("keystore-m_12381_3600_[0-9]+_0_0-[0-9]+.json")
.expect("regex is valid")
.is_match(file_name)
{
return true;
}
// The format exported by Prysm. I don't have a reference for this, but it was shared via
// Discord to Paul H.
if Regex::new("keystore-[0-9]+.json")
.expect("regex is valid")
.is_match(file_name)
{
return true;
}
false
}
#[cfg(test)]
mod tests {
use super::*;
use std::str::FromStr;
#[test]
fn voting_keystore_filename_lighthouse() {
assert!(is_voting_keystore(VOTING_KEYSTORE_FILE));
}
#[test]
fn voting_keystore_filename_launchpad() {
assert!(!is_voting_keystore("cats"));
assert!(!is_voting_keystore(&format!("a{}", VOTING_KEYSTORE_FILE)));
assert!(!is_voting_keystore(&format!("{}b", VOTING_KEYSTORE_FILE)));
assert!(is_voting_keystore(
"keystore-m_12381_3600_0_0_0-1593476250.json"
));
assert!(is_voting_keystore(
"keystore-m_12381_3600_1_0_0-1593476250.json"
));
assert!(is_voting_keystore("keystore-m_12381_3600_1_0_0-1593.json"));
assert!(!is_voting_keystore(
"keystore-m_12381_3600_0_0-1593476250.json"
));
assert!(!is_voting_keystore(
"keystore-m_12381_3600_1_0-1593476250.json"
));
}
#[test]
fn voting_keystore_filename_prysm() {
assert!(is_voting_keystore("keystore-0.json"));
assert!(is_voting_keystore("keystore-1.json"));
assert!(is_voting_keystore("keystore-101238259.json"));
assert!(!is_voting_keystore("keystore-.json"));
assert!(!is_voting_keystore("keystore-0a.json"));
assert!(!is_voting_keystore("keystore-cats.json"));
}
#[test]
fn graffiti_checks() {
let no_graffiti = r#"---
description: ""
enabled: true
type: local_keystore
voting_keystore_path: ""
voting_public_key: "0xaf3c7ddab7e293834710fca2d39d068f884455ede270e0d0293dc818e4f2f0f975355067e8437955cb29aec674e5c9e7"
"#;
let def: ValidatorDefinition = serde_yaml::from_str(no_graffiti).unwrap();
assert!(def.graffiti.is_none());
let invalid_graffiti = r#"---
description: ""
enabled: true
type: local_keystore
graffiti: "mrfwasheremrfwasheremrfwasheremrf"
voting_keystore_path: ""
voting_public_key: "0xaf3c7ddab7e293834710fca2d39d068f884455ede270e0d0293dc818e4f2f0f975355067e8437955cb29aec674e5c9e7"
"#;
let def: Result<ValidatorDefinition, _> = serde_yaml::from_str(invalid_graffiti);
assert!(def.is_err());
let valid_graffiti = r#"---
description: ""
enabled: true
type: local_keystore
graffiti: "mrfwashere"
voting_keystore_path: ""
voting_public_key: "0xaf3c7ddab7e293834710fca2d39d068f884455ede270e0d0293dc818e4f2f0f975355067e8437955cb29aec674e5c9e7"
"#;
let def: ValidatorDefinition = serde_yaml::from_str(valid_graffiti).unwrap();
assert_eq!(
def.graffiti,
Some(GraffitiString::from_str("mrfwashere").unwrap())
);
}
#[test]
fn suggested_fee_recipient_checks() {
let no_suggested_fee_recipient = r#"---
description: ""
enabled: true
type: local_keystore
voting_keystore_path: ""
voting_public_key: "0xaf3c7ddab7e293834710fca2d39d068f884455ede270e0d0293dc818e4f2f0f975355067e8437955cb29aec674e5c9e7"
"#;
let def: ValidatorDefinition = serde_yaml::from_str(no_suggested_fee_recipient).unwrap();
assert!(def.suggested_fee_recipient.is_none());
let invalid_suggested_fee_recipient = r#"---
description: ""
enabled: true
type: local_keystore
suggested_fee_recipient: "foopy"
voting_keystore_path: ""
voting_public_key: "0xaf3c7ddab7e293834710fca2d39d068f884455ede270e0d0293dc818e4f2f0f975355067e8437955cb29aec674e5c9e7"
"#;
let def: Result<ValidatorDefinition, _> =
serde_yaml::from_str(invalid_suggested_fee_recipient);
assert!(def.is_err());
let valid_suggested_fee_recipient = r#"---
description: ""
enabled: true
type: local_keystore
suggested_fee_recipient: "0xa2e334e71511686bcfe38bb3ee1ad8f6babcc03d"
voting_keystore_path: ""
voting_public_key: "0xaf3c7ddab7e293834710fca2d39d068f884455ede270e0d0293dc818e4f2f0f975355067e8437955cb29aec674e5c9e7"
"#;
let def: ValidatorDefinition = serde_yaml::from_str(valid_suggested_fee_recipient).unwrap();
assert_eq!(
def.suggested_fee_recipient,
Some(Address::from_str("0xa2e334e71511686bcfe38bb3ee1ad8f6babcc03d").unwrap())
);
}
#[test]
fn gas_limit_checks() {
let no_gas_limit = r#"---
description: ""
enabled: true
type: local_keystore
suggested_fee_recipient: "0xa2e334e71511686bcfe38bb3ee1ad8f6babcc03d"
voting_keystore_path: ""
voting_public_key: "0xaf3c7ddab7e293834710fca2d39d068f884455ede270e0d0293dc818e4f2f0f975355067e8437955cb29aec674e5c9e7"
"#;
let def: ValidatorDefinition = serde_yaml::from_str(no_gas_limit).unwrap();
assert!(def.gas_limit.is_none());
let invalid_gas_limit = r#"---
description: ""
enabled: true
type: local_keystore
suggested_fee_recipient: "0xa2e334e71511686bcfe38bb3ee1ad8f6babcc03d"
gas_limit: "banana"
voting_keystore_path: ""
voting_public_key: "0xaf3c7ddab7e293834710fca2d39d068f884455ede270e0d0293dc818e4f2f0f975355067e8437955cb29aec674e5c9e7"
"#;
let def: Result<ValidatorDefinition, _> = serde_yaml::from_str(invalid_gas_limit);
assert!(def.is_err());
let valid_gas_limit = r#"---
description: ""
enabled: true
type: local_keystore
suggested_fee_recipient: "0xa2e334e71511686bcfe38bb3ee1ad8f6babcc03d"
gas_limit: 35000000
voting_keystore_path: ""
voting_public_key: "0xaf3c7ddab7e293834710fca2d39d068f884455ede270e0d0293dc818e4f2f0f975355067e8437955cb29aec674e5c9e7"
"#;
let def: ValidatorDefinition = serde_yaml::from_str(valid_gas_limit).unwrap();
assert_eq!(def.gas_limit, Some(35000000));
}
#[test]
fn builder_proposals_checks() {
let no_builder_proposals = r#"---
description: ""
enabled: true
type: local_keystore
suggested_fee_recipient: "0xa2e334e71511686bcfe38bb3ee1ad8f6babcc03d"
voting_keystore_path: ""
voting_public_key: "0xaf3c7ddab7e293834710fca2d39d068f884455ede270e0d0293dc818e4f2f0f975355067e8437955cb29aec674e5c9e7"
"#;
let def: ValidatorDefinition = serde_yaml::from_str(no_builder_proposals).unwrap();
assert!(def.builder_proposals.is_none());
let invalid_builder_proposals = r#"---
description: ""
enabled: true
type: local_keystore
suggested_fee_recipient: "0xa2e334e71511686bcfe38bb3ee1ad8f6babcc03d"
builder_proposals: "banana"
voting_keystore_path: ""
voting_public_key: "0xaf3c7ddab7e293834710fca2d39d068f884455ede270e0d0293dc818e4f2f0f975355067e8437955cb29aec674e5c9e7"
"#;
let def: Result<ValidatorDefinition, _> = serde_yaml::from_str(invalid_builder_proposals);
assert!(def.is_err());
let valid_builder_proposals = r#"---
description: ""
enabled: true
type: local_keystore
suggested_fee_recipient: "0xa2e334e71511686bcfe38bb3ee1ad8f6babcc03d"
builder_proposals: true
voting_keystore_path: ""
voting_public_key: "0xaf3c7ddab7e293834710fca2d39d068f884455ede270e0d0293dc818e4f2f0f975355067e8437955cb29aec674e5c9e7"
"#;
let def: ValidatorDefinition = serde_yaml::from_str(valid_builder_proposals).unwrap();
assert_eq!(def.builder_proposals, Some(true));
}
}
Reference in New Issue View Git Blame Copy Permalink