VC: accept unknown fields in chain spec (#2277)

## Issue Addressed

Closes #2274

## Proposed Changes

* Modify the `YamlConfig` to collect unknown fields into an `extra_fields` map, instead of failing hard.
* Log a debug message if there are extra fields returned to the VC from one of its BNs.

This restores Lighthouse's compatibility with Teku beacon nodes (and therefore Infura)
This commit is contained in:
Michael Sproul 2021-03-26 04:53:57 +00:00
parent 9a71a7e486
commit f9d60f5436
23 changed files with 102 additions and 54 deletions

View File

@ -121,7 +121,11 @@ test-full: cargo-fmt test-release test-debug test-ef
# Lints the code for bad style and potentially unsafe arithmetic using Clippy.
# Clippy lints are opt-in per-crate for now. By default, everything is allowed except for performance and correctness lints.
lint:
cargo clippy --all --tests -- -D warnings
cargo clippy --all --tests -- \
-D warnings \
-A clippy::from-over-into \
-A clippy::upper-case-acronyms \
-A clippy::vec-init-then-push
# Runs the makefile in the `ef_tests` repo.
#

View File

@ -10,7 +10,7 @@ use eth2_keystore::Keystore;
use eth2_network_config::Eth2NetworkConfig;
use safe_arith::SafeArith;
use slot_clock::{SlotClock, SystemTimeSlotClock};
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use std::time::Duration;
use tokio::time::sleep;
use types::{ChainSpec, Epoch, EthSpec, Fork, VoluntaryExit};
@ -91,7 +91,7 @@ pub fn cli_run<E: EthSpec>(matches: &ArgMatches, env: Environment<E>) -> Result<
/// Gets the keypair and validator_index for every validator and calls `publish_voluntary_exit` on it.
async fn publish_voluntary_exit<E: EthSpec>(
keystore_path: &PathBuf,
keystore_path: &Path,
password_file_path: Option<&PathBuf>,
client: &BeaconNodeHttpClient,
spec: &ChainSpec,
@ -310,7 +310,7 @@ fn get_current_epoch<E: EthSpec>(genesis_time: u64, spec: &ChainSpec) -> Option<
/// If the `password_file_path` is Some, unlock keystore using password in given file
/// otherwise, prompts user for a password to unlock the keystore.
fn load_voting_keypair(
voting_keystore_path: &PathBuf,
voting_keystore_path: &Path,
password_file_path: Option<&PathBuf>,
stdin_inputs: bool,
) -> Result<Keypair, String> {

View File

@ -53,7 +53,7 @@ impl Inner {
pub fn from_bytes(bytes: &[u8], config: Config, spec: ChainSpec) -> Result<Self, String> {
let ssz_cache = SszEth1Cache::from_ssz_bytes(bytes)
.map_err(|e| format!("Ssz decoding error: {:?}", e))?;
Ok(ssz_cache.to_inner(config, spec)?)
ssz_cache.to_inner(config, spec)
}
/// Returns a reference to the specification.

View File

@ -36,7 +36,7 @@ use ssz::Encode;
use std::collections::HashSet;
use std::fs::File;
use std::io::Write;
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use std::{
collections::VecDeque,
marker::PhantomData,
@ -1336,7 +1336,7 @@ impl<TSpec: EthSpec> std::convert::From<Response<TSpec>> for RPCCodedResponse<TS
}
/// Persist metadata to disk
pub fn save_metadata_to_disk<E: EthSpec>(dir: &PathBuf, metadata: MetaData<E>, log: &slog::Logger) {
pub fn save_metadata_to_disk<E: EthSpec>(dir: &Path, metadata: MetaData<E>, log: &slog::Logger) {
let _ = std::fs::create_dir_all(&dir);
match File::create(dir.join(METADATA_FILENAME))
.and_then(|mut f| f.write_all(&metadata.as_ssz_bytes()))

View File

@ -474,7 +474,7 @@ fn strip_peer_id(addr: &mut Multiaddr) {
/// Load metadata from persisted file. Return default metadata if loading fails.
fn load_or_build_metadata<E: EthSpec>(
network_dir: &std::path::PathBuf,
network_dir: &std::path::Path,
log: &slog::Logger,
) -> MetaData<E> {
// Default metadata

View File

@ -10,7 +10,7 @@ use std::cmp::max;
use std::fs;
use std::net::{IpAddr, Ipv4Addr, ToSocketAddrs};
use std::net::{TcpListener, UdpSocket};
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use std::str::FromStr;
use types::{ChainSpec, Checkpoint, Epoch, EthSpec, Hash256, PublicKeyBytes, GRAFFITI_BYTES_LEN};
@ -422,7 +422,7 @@ pub fn get_config<E: EthSpec>(
pub fn set_network_config(
config: &mut NetworkConfig,
cli_args: &ArgMatches,
data_dir: &PathBuf,
data_dir: &Path,
log: &Logger,
use_listening_port_as_enr_port_by_default: bool,
) -> Result<(), String> {

View File

@ -660,7 +660,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
partial_state.load_historical_roots(&self.cold_db, &self.spec)?;
partial_state.load_randao_mixes(&self.cold_db, &self.spec)?;
Ok(partial_state.try_into()?)
partial_state.try_into()
}
/// Load a restore point state by its `restore_point_index`.

View File

@ -35,7 +35,7 @@ fn testnet_url() -> String {
fn main() {
match get_all_contracts() {
Ok(()) => (),
Err(e) => panic!(e),
Err(e) => panic!("{}", e),
}
}

View File

@ -68,7 +68,7 @@ pub fn decode_eth1_tx_data(
)
.map_err(DecodeError::SszDecodeError)?
};
};
}
let root = decode_token!(Hash256, to_fixed_bytes);

View File

@ -38,6 +38,10 @@ mod test {
fn version_formatting() {
let re = Regex::new(r"^Lighthouse/v[0-9]+\.[0-9]+\.[0-9]+(-rc.[0-9])?-[[:xdigit:]]{7}\+?$")
.unwrap();
assert!(re.is_match(VERSION), VERSION);
assert!(
re.is_match(VERSION),
"version doesn't match regex: {}",
VERSION
);
}
}

View File

@ -149,7 +149,7 @@ mod post {
let r = run_testcase(u).unwrap_err();
for msg in msgs.iter() {
assert!(r.contains(msg), format!("{:?} should contain {:?}", r, msg));
assert!(r.contains(msg), "{:?} should contain {:?}", r, msg);
}
};

View File

@ -89,7 +89,7 @@ impl ValidatorDir {
}
/// Returns the `dir` provided to `Self::open`.
pub fn dir(&self) -> &PathBuf {
pub fn dir(&self) -> &Path {
&self.dir
}
@ -204,7 +204,7 @@ impl ValidatorDir {
/// Attempts to load and decrypt a Keypair given path to the keystore.
pub fn unlock_keypair<P: AsRef<Path>>(
keystore_path: &PathBuf,
keystore_path: &Path,
password_dir: P,
) -> Result<Keypair, Error> {
let keystore = Keystore::from_json_reader(
@ -229,8 +229,8 @@ pub fn unlock_keypair<P: AsRef<Path>>(
/// Attempts to load and decrypt a Keypair given path to the keystore and the password file.
pub fn unlock_keypair_from_password_path(
keystore_path: &PathBuf,
password_path: &PathBuf,
keystore_path: &Path,
password_path: &Path,
) -> Result<Keypair, Error> {
let keystore = Keystore::from_json_reader(
&mut OpenOptions::new()
@ -242,7 +242,7 @@ pub fn unlock_keypair_from_password_path(
.map_err(Error::UnableToReadKeystore)?;
let password: PlainText = read(password_path)
.map_err(|_| Error::UnableToReadPassword(password_path.clone()))?
.map_err(|_| Error::UnableToReadPassword(password_path.into()))?
.into();
keystore
.decrypt_keypair(password.as_bytes())

View File

@ -182,7 +182,7 @@ fn concurrency() {
let harness = Harness::new();
let val_dir = harness.create_and_test(&BuildConfig::default());
let path = val_dir.dir().clone();
let path = val_dir.dir().to_owned();
// Should not re-open whilst opened after build.
ValidatorDir::open(&path).unwrap_err();

View File

@ -55,8 +55,8 @@ impl<'de> Visitor<'de> for HexVisitor {
where
E: de::Error,
{
Ok(hex::decode(value.trim_start_matches("0x"))
.map_err(|e| de::Error::custom(format!("invalid hex ({:?})", e)))?)
hex::decode(value.trim_start_matches("0x"))
.map_err(|e| de::Error::custom(format!("invalid hex ({:?})", e)))
}
}

View File

@ -353,7 +353,7 @@ macro_rules! impl_decodable_for_u8_array {
Err(DecodeError::InvalidByteLength { len, expected })
} else {
let mut array: [u8; $len] = [0; $len];
array.copy_from_slice(&bytes[..]);
array.copy_from_slice(bytes);
Ok(array)
}

View File

@ -108,9 +108,9 @@ impl<T: Default, N: Unsigned> From<Vec<T>> for FixedVector<T, N> {
}
}
impl<T, N: Unsigned> Into<Vec<T>> for FixedVector<T, N> {
fn into(self) -> Vec<T> {
self.vec
impl<T, N: Unsigned> From<FixedVector<T, N>> for Vec<T> {
fn from(vector: FixedVector<T, N>) -> Vec<T> {
vector.vec
}
}

View File

@ -120,9 +120,9 @@ impl<T, N: Unsigned> From<Vec<T>> for VariableList<T, N> {
}
}
impl<T, N: Unsigned> Into<Vec<T>> for VariableList<T, N> {
fn into(self) -> Vec<T> {
self.vec
impl<T, N: Unsigned> From<VariableList<T, N>> for Vec<T> {
fn from(list: VariableList<T, N>) -> Vec<T> {
list.vec
}
}

View File

@ -1,6 +1,7 @@
use crate::*;
use int_to_bytes::int_to_bytes4;
use serde_derive::{Deserialize, Serialize};
use std::collections::HashMap;
use std::fs::File;
use std::path::Path;
use tree_hash::TreeHash;
@ -449,10 +450,8 @@ mod tests {
}
/// YAML config file as defined by the spec.
///
/// Spec v0.12.3
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
#[serde(rename_all = "UPPERCASE", deny_unknown_fields)]
#[serde(rename_all = "UPPERCASE")]
pub struct YamlConfig {
pub config_name: String,
// ChainSpec
@ -576,6 +575,10 @@ pub struct YamlConfig {
#[serde(with = "serde_utils::quoted_u64")]
deposit_network_id: u64,
deposit_contract_address: Address,
// Extra fields (could be from a future hard-fork that we don't yet know).
#[serde(flatten)]
pub extra_fields: HashMap<String, String>,
}
impl Default for YamlConfig {
@ -671,6 +674,8 @@ impl YamlConfig {
deposit_chain_id: spec.deposit_chain_id,
deposit_network_id: spec.deposit_network_id,
deposit_contract_address: spec.deposit_contract_address,
extra_fields: HashMap::new(),
}
}
@ -849,6 +854,31 @@ mod yaml_tests {
assert_eq!(from, yamlconfig);
}
#[test]
fn extra_fields_round_trip() {
let tmp_file = NamedTempFile::new().expect("failed to create temp file");
let writer = OpenOptions::new()
.read(false)
.write(true)
.open(tmp_file.as_ref())
.expect("error opening file");
let mainnet_spec = ChainSpec::mainnet();
let mut yamlconfig = YamlConfig::from_spec::<MainnetEthSpec>(&mainnet_spec);
let (k1, v1) = ("SAMPLE_HARDFORK_KEY1", "123456789");
let (k2, v2) = ("SAMPLE_HARDFORK_KEY2", "987654321");
yamlconfig.extra_fields.insert(k1.into(), v1.into());
yamlconfig.extra_fields.insert(k2.into(), v2.into());
serde_yaml::to_writer(writer, &yamlconfig).expect("failed to write or serialize");
let reader = OpenOptions::new()
.read(true)
.write(false)
.open(tmp_file.as_ref())
.expect("error while opening the file");
let from: YamlConfig = serde_yaml::from_reader(reader).expect("error while deserializing");
assert_eq!(from, yamlconfig);
}
#[test]
fn apply_to_spec() {
let mut spec = ChainSpec::minimal();

View File

@ -19,35 +19,35 @@ where
aggregate: Cow<'a, GenericAggregateSignature<Pub, AggPub, Sig, AggSig>>,
}
impl<'a, Pub, AggPub, Sig, AggSig> Into<WrappedSignature<'a, Pub, AggPub, Sig, AggSig>>
for &'a GenericSignature<Pub, Sig>
impl<'a, Pub, AggPub, Sig, AggSig> From<&'a GenericSignature<Pub, Sig>>
for WrappedSignature<'a, Pub, AggPub, Sig, AggSig>
where
Pub: TPublicKey + Clone,
AggPub: Clone,
Sig: TSignature<Pub> + Clone,
AggSig: TAggregateSignature<Pub, AggPub, Sig> + Clone,
{
fn into(self) -> WrappedSignature<'a, Pub, AggPub, Sig, AggSig> {
fn from(sig: &'a GenericSignature<Pub, Sig>) -> Self {
let mut aggregate: GenericAggregateSignature<Pub, AggPub, Sig, AggSig> =
GenericAggregateSignature::infinity();
aggregate.add_assign(self);
aggregate.add_assign(sig);
WrappedSignature {
aggregate: Cow::Owned(aggregate),
}
}
}
impl<'a, Pub, AggPub, Sig, AggSig> Into<WrappedSignature<'a, Pub, AggPub, Sig, AggSig>>
for &'a GenericAggregateSignature<Pub, AggPub, Sig, AggSig>
impl<'a, Pub, AggPub, Sig, AggSig> From<&'a GenericAggregateSignature<Pub, AggPub, Sig, AggSig>>
for WrappedSignature<'a, Pub, AggPub, Sig, AggSig>
where
Pub: TPublicKey + Clone,
AggPub: Clone,
Sig: Clone,
AggSig: Clone,
{
fn into(self) -> WrappedSignature<'a, Pub, AggPub, Sig, AggSig> {
fn from(aggregate: &'a GenericAggregateSignature<Pub, AggPub, Sig, AggSig>) -> Self {
WrappedSignature {
aggregate: Cow::Borrowed(self),
aggregate: Cow::Borrowed(aggregate),
}
}
}

View File

@ -1,6 +1,7 @@
#![cfg(feature = "ef_tests")]
use ef_tests::*;
use std::collections::HashMap;
use std::path::PathBuf;
use types::*;
@ -17,6 +18,11 @@ fn config_test<E: EthSpec + TypeName>() {
let yaml_from_spec = YamlConfig::from_spec::<E>(&spec);
assert_eq!(yaml_config.apply_to_chain_spec::<E>(&spec), Some(spec));
assert_eq!(yaml_from_spec, yaml_config);
assert_eq!(
yaml_config.extra_fields,
HashMap::new(),
"not all config fields read"
);
}
#[test]

View File

@ -7,7 +7,7 @@ use state_processing::test_utils::BlockProcessingBuilder;
use std::env;
use std::fs::{self, File};
use std::io::Write;
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use std::process::exit;
use types::MainnetEthSpec;
use types::{BeaconState, ChainSpec, EthSpec, SignedBeaconBlock};
@ -91,12 +91,12 @@ fn write_vectors_to_file(title: &str, vectors: &[TestVector]) -> Result<(), Stri
}
/// Write some SSZ object to file.
fn write_to_ssz_file<T: Encode>(path: &PathBuf, item: &T) -> Result<(), String> {
fn write_to_ssz_file<T: Encode>(path: &Path, item: &T) -> Result<(), String> {
write_to_file(path, &item.as_ssz_bytes())
}
/// Write some bytes to file.
fn write_to_file(path: &PathBuf, item: &[u8]) -> Result<(), String> {
fn write_to_file(path: &Path, item: &[u8]) -> Result<(), String> {
File::create(path)
.map_err(|e| format!("Unable to create {:?}: {:?}", path, e))
.and_then(|mut file| {

View File

@ -7,7 +7,7 @@ use crate::http_metrics::metrics::{inc_counter_vec, ENDPOINT_ERRORS, ENDPOINT_RE
use environment::RuntimeContext;
use eth2::BeaconNodeHttpClient;
use futures::future;
use slog::{error, info, warn, Logger};
use slog::{debug, error, info, warn, Logger};
use slot_clock::SlotClock;
use std::fmt;
use std::fmt::Debug;
@ -236,6 +236,14 @@ impl<E: EthSpec> CandidateBeaconNode<E> {
CandidateError::Incompatible
})?;
if !yaml_config.extra_fields.is_empty() {
debug!(
log,
"Beacon spec includes unknown fields";
"fields" => ?yaml_config.extra_fields
);
}
if *spec == beacon_node_spec {
Ok(())
} else {

View File

@ -19,7 +19,7 @@ use slog::{debug, error, info, warn, Logger};
use std::collections::{HashMap, HashSet};
use std::fs::File;
use std::io;
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use types::{Graffiti, Keypair, PublicKey, PublicKeyBytes};
use crate::key_cache;
@ -101,20 +101,16 @@ impl InitializedValidator {
}
}
fn open_keystore(path: &PathBuf) -> Result<Keystore, Error> {
fn open_keystore(path: &Path) -> Result<Keystore, Error> {
let keystore_file = File::open(path).map_err(Error::UnableToOpenVotingKeystore)?;
Keystore::from_json_reader(keystore_file).map_err(Error::UnableToParseVotingKeystore)
}
fn get_lockfile_path(file_path: &PathBuf) -> Option<PathBuf> {
fn get_lockfile_path(file_path: &Path) -> Option<PathBuf> {
file_path
.file_name()
.and_then(|os_str| os_str.to_str())
.map(|filename| {
file_path
.clone()
.with_file_name(format!("{}.lock", filename))
})
.map(|filename| file_path.with_file_name(format!("{}.lock", filename)))
}
impl InitializedValidator {
@ -238,7 +234,7 @@ impl InitializedValidator {
/// Try to unlock `keystore` at `keystore_path` by prompting the user via `stdin`.
fn unlock_keystore_via_stdin_password(
keystore: &Keystore,
keystore_path: &PathBuf,
keystore_path: &Path,
) -> Result<(ZeroizeString, Keypair), Error> {
eprintln!();
eprintln!(