From b3c7e59a5b28775861e50c0695c907acb8ebbf92 Mon Sep 17 00:00:00 2001 From: Mac L Date: Mon, 12 Jul 2021 01:47:48 +0000 Subject: [PATCH] Adjust beacon node timeouts for validator client HTTP requests (#2352) ## Issue Addressed Resolves #2313 ## Proposed Changes Provide `BeaconNodeHttpClient` with a dedicated `Timeouts` struct. This will allow granular adjustment of the timeout duration for different calls made from the VC to the BN. These can either be a constant value, or as a ratio of the slot duration. Improve timeout performance by using these adjusted timeout duration's only whenever a fallback endpoint is available. Add a CLI flag called `use-long-timeouts` to revert to the old behavior. ## Additional Info Additionally set the default `BeaconNodeHttpClient` timeouts to the be the slot duration of the network, rather than a constant 12 seconds. This will allow it to adjust to different network specifications. Co-authored-by: Paul Hauner --- account_manager/src/validator/exit.rs | 3 +- beacon_node/http_api/tests/tests.rs | 5 +- common/eth2/src/lib.rs | 126 +++++++++++++++++-- lighthouse/tests/validator_client.rs | 8 ++ testing/node_test_rig/src/lib.rs | 3 +- validator_client/src/attestation_service.rs | 16 +++ validator_client/src/block_service.rs | 9 ++ validator_client/src/cli.rs | 7 ++ validator_client/src/config.rs | 4 + validator_client/src/duties_service.rs | 16 +++ validator_client/src/fork_service.rs | 1 + validator_client/src/http_metrics/metrics.rs | 10 ++ validator_client/src/lib.rs | 49 ++++++-- 13 files changed, 234 insertions(+), 23 deletions(-) diff --git a/account_manager/src/validator/exit.rs b/account_manager/src/validator/exit.rs index 986ae8dc2..c8bac168e 100644 --- a/account_manager/src/validator/exit.rs +++ b/account_manager/src/validator/exit.rs @@ -4,7 +4,7 @@ use clap::{App, Arg, ArgMatches}; use environment::Environment; use eth2::{ types::{GenesisData, StateId, ValidatorData, ValidatorId, ValidatorStatus}, - BeaconNodeHttpClient, + BeaconNodeHttpClient, Timeouts, }; use eth2_keystore::Keystore; use eth2_network_config::Eth2NetworkConfig; @@ -81,6 +81,7 @@ pub fn cli_run(matches: &ArgMatches, env: Environment) -> Result< let client = BeaconNodeHttpClient::new( SensitiveUrl::parse(&server_url) .map_err(|e| format!("Failed to parse beacon http server: {:?}", e))?, + Timeouts::set_all(Duration::from_secs(env.eth2_config.spec.seconds_per_slot)), ); let testnet_config = env diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 6cb60832f..b82dd25fa 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -9,7 +9,7 @@ use discv5::enr::{CombinedKey, EnrBuilder}; use environment::null_logger; use eth2::Error; use eth2::StatusCode; -use eth2::{types::*, BeaconNodeHttpClient}; +use eth2::{types::*, BeaconNodeHttpClient, Timeouts}; use eth2_libp2p::{ rpc::methods::MetaData, types::{EnrBitfield, SyncState}, @@ -35,6 +35,7 @@ use types::{ type E = MainnetEthSpec; +const SECONDS_PER_SLOT: u64 = 12; const SLOTS_PER_EPOCH: u64 = 32; const VALIDATOR_COUNT: usize = SLOTS_PER_EPOCH as usize; const CHAIN_LENGTH: u64 = SLOTS_PER_EPOCH * 5 - 1; // Make `next_block` an epoch transition @@ -213,6 +214,7 @@ impl ApiTester { listening_socket.port() )) .unwrap(), + Timeouts::set_all(Duration::from_secs(SECONDS_PER_SLOT)), ); Self { @@ -327,6 +329,7 @@ impl ApiTester { listening_socket.port() )) .unwrap(), + Timeouts::set_all(Duration::from_secs(SECONDS_PER_SLOT)), ); Self { diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 192693376..ea4f013a1 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -24,6 +24,7 @@ use serde::{de::DeserializeOwned, Serialize}; use std::convert::TryFrom; use std::fmt; use std::iter::Iterator; +use std::time::Duration; #[derive(Debug)] pub enum Error { @@ -76,12 +77,34 @@ impl fmt::Display for Error { } } +/// A struct to define a variety of different timeouts for different validator tasks to ensure +/// proper fallback behaviour. +#[derive(Clone)] +pub struct Timeouts { + pub attestation: Duration, + pub attester_duties: Duration, + pub proposal: Duration, + pub proposer_duties: Duration, +} + +impl Timeouts { + pub fn set_all(timeout: Duration) -> Self { + Timeouts { + attestation: timeout, + attester_duties: timeout, + proposal: timeout, + proposer_duties: timeout, + } + } +} + /// A wrapper around `reqwest::Client` which provides convenience methods for interfacing with a /// Lighthouse Beacon Node HTTP server (`http_api`). #[derive(Clone)] pub struct BeaconNodeHttpClient { client: reqwest::Client, server: SensitiveUrl, + timeouts: Timeouts, } impl fmt::Display for BeaconNodeHttpClient { @@ -97,15 +120,24 @@ impl AsRef for BeaconNodeHttpClient { } impl BeaconNodeHttpClient { - pub fn new(server: SensitiveUrl) -> Self { + pub fn new(server: SensitiveUrl, timeouts: Timeouts) -> Self { Self { client: reqwest::Client::new(), server, + timeouts, } } - pub fn from_components(server: SensitiveUrl, client: reqwest::Client) -> Self { - Self { client, server } + pub fn from_components( + server: SensitiveUrl, + client: reqwest::Client, + timeouts: Timeouts, + ) -> Self { + Self { + client, + server, + timeouts, + } } /// Return the path with the standard `/eth1/v1` prefix applied. @@ -130,6 +162,26 @@ impl BeaconNodeHttpClient { .map_err(Error::Reqwest) } + /// Perform a HTTP GET request with a custom timeout. + async fn get_with_timeout( + &self, + url: U, + timeout: Duration, + ) -> Result { + let response = self + .client + .get(url) + .timeout(timeout) + .send() + .await + .map_err(Error::Reqwest)?; + ok_or_error(response) + .await? + .json() + .await + .map_err(Error::Reqwest) + } + /// Perform a HTTP GET request, returning `None` on a 404 error. async fn get_opt(&self, url: U) -> Result, Error> { let response = self.client.get(url).send().await.map_err(Error::Reqwest)?; @@ -145,6 +197,31 @@ impl BeaconNodeHttpClient { } } + /// Perform a HTTP GET request with a custom timeout, returning `None` on a 404 error. + async fn get_opt_with_timeout( + &self, + url: U, + timeout: Duration, + ) -> Result, Error> { + let response = self + .client + .get(url) + .timeout(timeout) + .send() + .await + .map_err(Error::Reqwest)?; + match ok_or_error(response).await { + Ok(resp) => resp.json().await.map(Option::Some).map_err(Error::Reqwest), + Err(err) => { + if err.status() == Some(StatusCode::NOT_FOUND) { + Ok(None) + } else { + Err(err) + } + } + } + } + /// Perform a HTTP GET request using an 'accept' header, returning `None` on a 404 error. pub async fn get_bytes_opt_accept_header( &self, @@ -189,15 +266,36 @@ impl BeaconNodeHttpClient { Ok(()) } - /// Perform a HTTP POST request, returning a JSON response. - async fn post_with_response( + /// Perform a HTTP POST request with a custom timeout. + async fn post_with_timeout( + &self, + url: U, + body: &T, + timeout: Duration, + ) -> Result<(), Error> { + let response = self + .client + .post(url) + .timeout(timeout) + .json(body) + .send() + .await + .map_err(Error::Reqwest)?; + ok_or_error(response).await?; + Ok(()) + } + + /// Perform a HTTP POST request with a custom timeout, returning a JSON response. + async fn post_with_timeout_and_response( &self, url: U, body: &V, + timeout: Duration, ) -> Result { let response = self .client .post(url) + .timeout(timeout) .json(body) .send() .await @@ -468,7 +566,8 @@ impl BeaconNodeHttpClient { .push("beacon") .push("blocks"); - self.post(path, block).await?; + self.post_with_timeout(path, block, self.timeouts.proposal) + .await?; Ok(()) } @@ -567,6 +666,7 @@ impl BeaconNodeHttpClient { let response = self .client .post(path) + .timeout(self.timeouts.attestation) .json(attestations) .send() .await @@ -929,7 +1029,8 @@ impl BeaconNodeHttpClient { .push("proposer") .push(&epoch.to_string()); - self.get(path).await + self.get_with_timeout(path, self.timeouts.proposer_duties) + .await } /// `GET validator/blocks/{slot}` @@ -975,10 +1076,10 @@ impl BeaconNodeHttpClient { .append_pair("slot", &slot.to_string()) .append_pair("committee_index", &committee_index.to_string()); - self.get(path).await + self.get_with_timeout(path, self.timeouts.attestation).await } - /// `GET validator/attestation_attestation?slot,attestation_data_root` + /// `GET validator/aggregate_attestation?slot,attestation_data_root` pub async fn get_validator_aggregate_attestation( &self, slot: Slot, @@ -998,7 +1099,8 @@ impl BeaconNodeHttpClient { &format!("{:?}", attestation_data_root), ); - self.get_opt(path).await + self.get_opt_with_timeout(path, self.timeouts.attestation) + .await } /// `POST validator/duties/attester/{epoch}` @@ -1016,7 +1118,8 @@ impl BeaconNodeHttpClient { .push("attester") .push(&epoch.to_string()); - self.post_with_response(path, &indices).await + self.post_with_timeout_and_response(path, &indices, self.timeouts.attester_duties) + .await } /// `POST validator/aggregate_and_proofs` @@ -1034,6 +1137,7 @@ impl BeaconNodeHttpClient { let response = self .client .post(path) + .timeout(self.timeouts.attestation) .json(aggregates) .send() .await diff --git a/lighthouse/tests/validator_client.rs b/lighthouse/tests/validator_client.rs index a0982064b..a6a644fd4 100644 --- a/lighthouse/tests/validator_client.rs +++ b/lighthouse/tests/validator_client.rs @@ -194,6 +194,14 @@ fn init_slashing_protections_flag() { .with_config(|config| assert!(config.init_slashing_protection)); } +#[test] +fn use_long_timeouts_flag() { + CommandLineTest::new() + .flag("use-long-timeouts", None) + .run() + .with_config(|config| assert!(config.use_long_timeouts)); +} + // Tests for Graffiti flags. #[test] fn graffiti_flag() { diff --git a/testing/node_test_rig/src/lib.rs b/testing/node_test_rig/src/lib.rs index 436b852c2..acf9bb9e6 100644 --- a/testing/node_test_rig/src/lib.rs +++ b/testing/node_test_rig/src/lib.rs @@ -4,7 +4,7 @@ use beacon_node::ProductionBeaconNode; use environment::RuntimeContext; -use eth2::{reqwest::ClientBuilder, BeaconNodeHttpClient}; +use eth2::{reqwest::ClientBuilder, BeaconNodeHttpClient, Timeouts}; use sensitive_url::SensitiveUrl; use std::path::PathBuf; use std::time::Duration; @@ -77,6 +77,7 @@ impl LocalBeaconNode { Ok(BeaconNodeHttpClient::from_components( beacon_node_url, beacon_node_http_client, + Timeouts::set_all(HTTP_TIMEOUT), )) } } diff --git a/validator_client/src/attestation_service.rs b/validator_client/src/attestation_service.rs index 905c11117..af016f7cf 100644 --- a/validator_client/src/attestation_service.rs +++ b/validator_client/src/attestation_service.rs @@ -337,6 +337,10 @@ impl AttestationService { let attestation_data = self .beacon_nodes .first_success(RequireSynced::No, |beacon_node| async move { + let _timer = metrics::start_timer_vec( + &metrics::ATTESTATION_SERVICE_TIMES, + &[metrics::ATTESTATIONS_HTTP_GET], + ); beacon_node .get_validator_attestation_data(slot, committee_index) .await @@ -399,6 +403,10 @@ impl AttestationService { match self .beacon_nodes .first_success(RequireSynced::No, |beacon_node| async move { + let _timer = metrics::start_timer_vec( + &metrics::ATTESTATION_SERVICE_TIMES, + &[metrics::ATTESTATIONS_HTTP_POST], + ); beacon_node .post_beacon_pool_attestations(attestations_slice) .await @@ -451,6 +459,10 @@ impl AttestationService { let aggregated_attestation = self .beacon_nodes .first_success(RequireSynced::No, |beacon_node| async move { + let _timer = metrics::start_timer_vec( + &metrics::ATTESTATION_SERVICE_TIMES, + &[metrics::AGGREGATES_HTTP_GET], + ); beacon_node .get_validator_aggregate_attestation( attestation_data_ref.slot, @@ -503,6 +515,10 @@ impl AttestationService { match self .beacon_nodes .first_success(RequireSynced::No, |beacon_node| async move { + let _timer = metrics::start_timer_vec( + &metrics::ATTESTATION_SERVICE_TIMES, + &[metrics::AGGREGATES_HTTP_POST], + ); beacon_node .post_validator_aggregate_and_proof(signed_aggregate_and_proofs_slice) .await diff --git a/validator_client/src/block_service.rs b/validator_client/src/block_service.rs index 36ca283f2..091a89c63 100644 --- a/validator_client/src/block_service.rs +++ b/validator_client/src/block_service.rs @@ -263,17 +263,26 @@ impl BlockService { let signed_block = self .beacon_nodes .first_success(RequireSynced::No, |beacon_node| async move { + let get_timer = metrics::start_timer_vec( + &metrics::BLOCK_SERVICE_TIMES, + &[metrics::BEACON_BLOCK_HTTP_GET], + ); let block = beacon_node .get_validator_blocks(slot, randao_reveal_ref, graffiti.as_ref()) .await .map_err(|e| format!("Error from beacon node when producing block: {:?}", e))? .data; + drop(get_timer); let signed_block = self_ref .validator_store .sign_block(validator_pubkey_ref, block, current_slot) .ok_or("Unable to sign block")?; + let _post_timer = metrics::start_timer_vec( + &metrics::BLOCK_SERVICE_TIMES, + &[metrics::BEACON_BLOCK_HTTP_POST], + ); beacon_node .post_beacon_blocks(&signed_block) .await diff --git a/validator_client/src/cli.rs b/validator_client/src/cli.rs index 13e4f4e02..08d3d9ae5 100644 --- a/validator_client/src/cli.rs +++ b/validator_client/src/cli.rs @@ -94,6 +94,13 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { node is not synced.", ), ) + .arg( + Arg::with_name("use-long-timeouts") + .long("use-long-timeouts") + .help("If present, the validator client will use longer timeouts for requests \ + made to the beacon node. This flag is generally not recommended, \ + longer timeouts can cause missed duties when fallbacks are used.") + ) // This overwrites the graffiti configured in the beacon node. .arg( Arg::with_name("graffiti") diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index cb5c862da..b299247de 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -35,6 +35,8 @@ pub struct Config { pub disable_auto_discover: bool, /// If true, re-register existing validators in definitions.yml for slashing protection. pub init_slashing_protection: bool, + /// If true, use longer timeouts for requests made to the beacon node. + pub use_long_timeouts: bool, /// Graffiti to be inserted everytime we create a block. pub graffiti: Option, /// Graffiti file to load per validator graffitis. @@ -68,6 +70,7 @@ impl Default for Config { allow_unsynced_beacon_node: false, disable_auto_discover: false, init_slashing_protection: false, + use_long_timeouts: false, graffiti: None, graffiti_file: None, http_api: <_>::default(), @@ -156,6 +159,7 @@ impl Config { config.allow_unsynced_beacon_node = cli_args.is_present("allow-unsynced"); config.disable_auto_discover = cli_args.is_present("disable-auto-discover"); config.init_slashing_protection = cli_args.is_present("init-slashing-protection"); + config.use_long_timeouts = cli_args.is_present("use-long-timeouts"); if let Some(graffiti_file_path) = cli_args.value_of("graffiti-file") { let mut graffiti_file = GraffitiFile::new(graffiti_file_path.into()); diff --git a/validator_client/src/duties_service.rs b/validator_client/src/duties_service.rs index 0ad85a0dd..aa7edd29e 100644 --- a/validator_client/src/duties_service.rs +++ b/validator_client/src/duties_service.rs @@ -285,6 +285,10 @@ async fn poll_validator_indices( let download_result = duties_service .beacon_nodes .first_success(duties_service.require_synced, |beacon_node| async move { + let _timer = metrics::start_timer_vec( + &metrics::DUTIES_SERVICE_TIMES, + &[metrics::VALIDATOR_ID_HTTP_GET], + ); beacon_node .get_beacon_states_validator_id( StateId::Head, @@ -453,6 +457,10 @@ async fn poll_beacon_attesters( if let Err(e) = duties_service .beacon_nodes .first_success(duties_service.require_synced, |beacon_node| async move { + let _timer = metrics::start_timer_vec( + &metrics::DUTIES_SERVICE_TIMES, + &[metrics::SUBSCRIPTIONS_HTTP_POST], + ); beacon_node .post_validator_beacon_committee_subscriptions(subscriptions_ref) .await @@ -509,6 +517,10 @@ async fn poll_beacon_attesters_for_epoch( let response = duties_service .beacon_nodes .first_success(duties_service.require_synced, |beacon_node| async move { + let _timer = metrics::start_timer_vec( + &metrics::DUTIES_SERVICE_TIMES, + &[metrics::ATTESTER_DUTIES_HTTP_POST], + ); beacon_node .post_validator_duties_attester(epoch, local_indices) .await @@ -640,6 +652,10 @@ async fn poll_beacon_proposers( let download_result = duties_service .beacon_nodes .first_success(duties_service.require_synced, |beacon_node| async move { + let _timer = metrics::start_timer_vec( + &metrics::DUTIES_SERVICE_TIMES, + &[metrics::PROPOSER_DUTIES_HTTP_GET], + ); beacon_node .get_validator_duties_proposer(current_epoch) .await diff --git a/validator_client/src/fork_service.rs b/validator_client/src/fork_service.rs index 979a61d10..2c2df187b 100644 --- a/validator_client/src/fork_service.rs +++ b/validator_client/src/fork_service.rs @@ -84,6 +84,7 @@ impl ForkServiceBuilder { ); let candidates = vec![CandidateBeaconNode::new(eth2::BeaconNodeHttpClient::new( sensitive_url::SensitiveUrl::parse("http://127.0.0.1").unwrap(), + eth2::Timeouts::set_all(Duration::from_secs(12)), ))]; let mut beacon_nodes = BeaconNodeFallback::new(candidates, spec, log.clone()); beacon_nodes.set_slot_clock(slot_clock); diff --git a/validator_client/src/http_metrics/metrics.rs b/validator_client/src/http_metrics/metrics.rs index d5e5bf481..120e6c2c7 100644 --- a/validator_client/src/http_metrics/metrics.rs +++ b/validator_client/src/http_metrics/metrics.rs @@ -9,8 +9,14 @@ pub const SAME_DATA: &str = "same_data"; pub const UNREGISTERED: &str = "unregistered"; pub const FULL_UPDATE: &str = "full_update"; pub const BEACON_BLOCK: &str = "beacon_block"; +pub const BEACON_BLOCK_HTTP_GET: &str = "beacon_block_http_get"; +pub const BEACON_BLOCK_HTTP_POST: &str = "beacon_block_http_post"; pub const ATTESTATIONS: &str = "attestations"; +pub const ATTESTATIONS_HTTP_GET: &str = "attestations_http_get"; +pub const ATTESTATIONS_HTTP_POST: &str = "attestations_http_post"; pub const AGGREGATES: &str = "aggregates"; +pub const AGGREGATES_HTTP_GET: &str = "aggregates_http_get"; +pub const AGGREGATES_HTTP_POST: &str = "aggregates_http_post"; pub const CURRENT_EPOCH: &str = "current_epoch"; pub const NEXT_EPOCH: &str = "next_epoch"; pub const UPDATE_INDICES: &str = "update_indices"; @@ -18,6 +24,10 @@ pub const UPDATE_ATTESTERS_CURRENT_EPOCH: &str = "update_attesters_current_epoch pub const UPDATE_ATTESTERS_NEXT_EPOCH: &str = "update_attesters_next_epoch"; pub const UPDATE_ATTESTERS_FETCH: &str = "update_attesters_fetch"; pub const UPDATE_ATTESTERS_STORE: &str = "update_attesters_store"; +pub const ATTESTER_DUTIES_HTTP_POST: &str = "attester_duties_http_post"; +pub const PROPOSER_DUTIES_HTTP_GET: &str = "proposer_duties_http_get"; +pub const VALIDATOR_ID_HTTP_GET: &str = "validator_id_http_get"; +pub const SUBSCRIPTIONS_HTTP_POST: &str = "subscriptions_http_post"; pub const UPDATE_PROPOSERS: &str = "update_proposers"; pub const SUBSCRIPTIONS: &str = "subscriptions"; diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 0f462aca7..d9fe21111 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -30,7 +30,7 @@ use clap::ArgMatches; use duties_service::DutiesService; use environment::RuntimeContext; use eth2::types::StateId; -use eth2::{reqwest::ClientBuilder, BeaconNodeHttpClient, StatusCode}; +use eth2::{reqwest::ClientBuilder, BeaconNodeHttpClient, StatusCode, Timeouts}; use fork_service::{ForkService, ForkServiceBuilder}; use http_api::ApiSecret; use initialized_validators::InitializedValidators; @@ -57,8 +57,12 @@ const RETRY_DELAY: Duration = Duration::from_secs(2); /// The time between polls when waiting for genesis. const WAITING_FOR_GENESIS_POLL_TIME: Duration = Duration::from_secs(12); -/// The global timeout for HTTP requests to the beacon node. -const HTTP_TIMEOUT: Duration = Duration::from_secs(12); +/// Specific timeout constants for HTTP requests involved in different validator duties. +/// This can help ensure that proper endpoint fallback occurs. +const HTTP_ATTESTATION_TIMEOUT_QUOTIENT: u32 = 4; +const HTTP_ATTESTER_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; +const HTTP_PROPOSAL_TIMEOUT_QUOTIENT: u32 = 2; +const HTTP_PROPOSER_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; #[derive(Clone)] pub struct ProductionValidatorClient { @@ -222,18 +226,45 @@ impl ProductionValidatorClient { })?; } + let last_beacon_node_index = config + .beacon_nodes + .len() + .checked_sub(1) + .ok_or_else(|| "No beacon nodes defined.".to_string())?; + let beacon_nodes: Vec = config .beacon_nodes - .clone() - .into_iter() - .map(|url| { + .iter() + .enumerate() + .map(|(i, url)| { + let slot_duration = Duration::from_secs(context.eth2_config.spec.seconds_per_slot); + let beacon_node_http_client = ClientBuilder::new() - .timeout(HTTP_TIMEOUT) + // Set default timeout to be the full slot duration. + .timeout(slot_duration) .build() .map_err(|e| format!("Unable to build HTTP client: {:?}", e))?; + + // Use quicker timeouts if a fallback beacon node exists. + let timeouts = if i < last_beacon_node_index && !config.use_long_timeouts { + info!( + log, + "Fallback endpoints are available, using optimized timeouts."; + ); + Timeouts { + attestation: slot_duration / HTTP_ATTESTATION_TIMEOUT_QUOTIENT, + attester_duties: slot_duration / HTTP_ATTESTER_DUTIES_TIMEOUT_QUOTIENT, + proposal: slot_duration / HTTP_PROPOSAL_TIMEOUT_QUOTIENT, + proposer_duties: slot_duration / HTTP_PROPOSER_DUTIES_TIMEOUT_QUOTIENT, + } + } else { + Timeouts::set_all(slot_duration) + }; + Ok(BeaconNodeHttpClient::from_components( - url, + url.clone(), beacon_node_http_client, + timeouts, )) }) .collect::, String>>()?; @@ -244,7 +275,7 @@ impl ProductionValidatorClient { .map(CandidateBeaconNode::new) .collect(); - // Set the count for beacon node fallbacks excluding the primary beacon node + // Set the count for beacon node fallbacks excluding the primary beacon node. set_gauge( &http_metrics::metrics::ETH2_FALLBACK_CONFIGURED, num_nodes.saturating_sub(1) as i64,