diff --git a/Cargo.lock b/Cargo.lock index 2768bcd4a..c3290ebc8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,6 +25,7 @@ dependencies = [ "rand 0.7.3", "rayon", "safe_arith", + "sensitive_url", "slashing_protection", "slog", "slog-async", @@ -650,6 +651,7 @@ dependencies = [ "logging", "node_test_rig", "rand 0.7.3", + "sensitive_url", "serde", "slasher", "slog", @@ -1896,6 +1898,7 @@ dependencies = [ "merkle_proof", "parking_lot", "reqwest", + "sensitive_url", "serde", "serde_json", "slog", @@ -1942,6 +1945,7 @@ dependencies = [ "psutil", "reqwest", "ring", + "sensitive_url", "serde", "serde_json", "serde_utils", @@ -2514,6 +2518,7 @@ dependencies = [ "merkle_proof", "parking_lot", "rayon", + "sensitive_url", "serde", "serde_derive", "slog", @@ -2844,6 +2849,7 @@ dependencies = [ "lighthouse_version", "network", "parking_lot", + "sensitive_url", "serde", "slog", "slot_clock", @@ -3347,6 +3353,7 @@ dependencies = [ "log", "rand 0.7.3", "regex", + "sensitive_url", "serde", "serde_yaml", "simple_logger", @@ -4189,6 +4196,7 @@ dependencies = [ "futures 0.3.14", "genesis", "reqwest", + "sensitive_url", "serde", "tempfile", "types", @@ -5155,6 +5163,7 @@ dependencies = [ "rand 0.7.3", "remote_signer_test", "reqwest", + "sensitive_url", "serde", "tokio 1.5.0", "types", @@ -5171,6 +5180,7 @@ dependencies = [ "remote_signer_client", "remote_signer_consumer", "reqwest", + "sensitive_url", "serde", "serde_json", "tempfile", @@ -5499,6 +5509,14 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +[[package]] +name = "sensitive_url" +version = "0.1.0" +dependencies = [ + "serde", + "url", +] + [[package]] name = "serde" version = "1.0.125" @@ -5714,6 +5732,7 @@ dependencies = [ "node_test_rig", "parking_lot", "rayon", + "sensitive_url", "tokio 1.5.0", "types", "validator_client", @@ -7052,6 +7071,7 @@ dependencies = [ "ring", "safe_arith", "scrypt", + "sensitive_url", "serde", "serde_derive", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 8fd60b5ab..c1c4cfd49 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ members = [ "common/logging", "common/lru_cache", "common/remote_signer_consumer", + "common/sensitive_url", "common/slot_clock", "common/task_executor", "common/test_random_derive", diff --git a/account_manager/Cargo.toml b/account_manager/Cargo.toml index 93b06c18a..5cd5d0398 100644 --- a/account_manager/Cargo.toml +++ b/account_manager/Cargo.toml @@ -34,6 +34,7 @@ slashing_protection = { path = "../validator_client/slashing_protection" } eth2 = {path = "../common/eth2"} safe_arith = {path = "../consensus/safe_arith"} slot_clock = { path = "../common/slot_clock" } +sensitive_url = { path = "../common/sensitive_url" } [dev-dependencies] tempfile = "3.1.0" diff --git a/account_manager/src/validator/exit.rs b/account_manager/src/validator/exit.rs index 7d57ebcf6..0cf066610 100644 --- a/account_manager/src/validator/exit.rs +++ b/account_manager/src/validator/exit.rs @@ -4,11 +4,12 @@ use clap::{App, Arg, ArgMatches}; use environment::Environment; use eth2::{ types::{GenesisData, StateId, ValidatorData, ValidatorId, ValidatorStatus}, - BeaconNodeHttpClient, Url, + BeaconNodeHttpClient, }; use eth2_keystore::Keystore; use eth2_network_config::Eth2NetworkConfig; use safe_arith::SafeArith; +use sensitive_url::SensitiveUrl; use slot_clock::{SlotClock, SystemTimeSlotClock}; use std::path::{Path, PathBuf}; use std::time::Duration; @@ -75,7 +76,7 @@ pub fn cli_run(matches: &ArgMatches, env: Environment) -> Result< let spec = env.eth2_config().spec.clone(); let server_url: String = clap_utils::parse_required(matches, BEACON_SERVER_FLAG)?; let client = BeaconNodeHttpClient::new( - Url::parse(&server_url) + SensitiveUrl::parse(&server_url) .map_err(|e| format!("Failed to parse beacon http server: {:?}", e))?, ); diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 1661a255e..f4a901dc2 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -44,3 +44,4 @@ hyper = "0.14.4" lighthouse_version = { path = "../common/lighthouse_version" } hex = "0.4.2" slasher = { path = "../slasher" } +sensitive_url = { path = "../common/sensitive_url" } diff --git a/beacon_node/eth1/Cargo.toml b/beacon_node/eth1/Cargo.toml index bf4835d71..618b59676 100644 --- a/beacon_node/eth1/Cargo.toml +++ b/beacon_node/eth1/Cargo.toml @@ -34,3 +34,4 @@ lazy_static = "1.4.0" task_executor = { path = "../../common/task_executor" } eth2 = { path = "../../common/eth2" } fallback = { path = "../../common/fallback" } +sensitive_url = { path = "../../common/sensitive_url" } diff --git a/beacon_node/eth1/src/http.rs b/beacon_node/eth1/src/http.rs index 88e322244..547715a06 100644 --- a/beacon_node/eth1/src/http.rs +++ b/beacon_node/eth1/src/http.rs @@ -12,6 +12,7 @@ use futures::future::TryFutureExt; use reqwest::{header::CONTENT_TYPE, ClientBuilder, StatusCode}; +use sensitive_url::SensitiveUrl; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use std::ops::Range; @@ -79,7 +80,7 @@ impl FromStr for Eth1Id { } /// Get the eth1 network id of the given endpoint. -pub async fn get_network_id(endpoint: &str, timeout: Duration) -> Result { +pub async fn get_network_id(endpoint: &SensitiveUrl, timeout: Duration) -> Result { let response_body = send_rpc_request(endpoint, "net_version", json!([]), timeout).await?; Eth1Id::from_str( response_result(&response_body)? @@ -90,7 +91,7 @@ pub async fn get_network_id(endpoint: &str, timeout: Duration) -> Result Result { +pub async fn get_chain_id(endpoint: &SensitiveUrl, timeout: Duration) -> Result { let response_body = send_rpc_request(endpoint, "eth_chainId", json!([]), timeout).await?; hex_to_u64_be( response_result(&response_body)? @@ -111,7 +112,7 @@ pub struct Block { /// Returns the current block number. /// /// Uses HTTP JSON RPC at `endpoint`. E.g., `http://localhost:8545`. -pub async fn get_block_number(endpoint: &str, timeout: Duration) -> Result { +pub async fn get_block_number(endpoint: &SensitiveUrl, timeout: Duration) -> Result { let response_body = send_rpc_request(endpoint, "eth_blockNumber", json!([]), timeout).await?; hex_to_u64_be( response_result(&response_body)? @@ -126,7 +127,7 @@ pub async fn get_block_number(endpoint: &str, timeout: Duration) -> Result Result { @@ -191,7 +192,7 @@ pub async fn get_block( /// /// Uses HTTP JSON RPC at `endpoint`. E.g., `http://localhost:8545`. pub async fn get_deposit_count( - endpoint: &str, + endpoint: &SensitiveUrl, address: &str, block_number: u64, timeout: Duration, @@ -229,7 +230,7 @@ pub async fn get_deposit_count( /// /// Uses HTTP JSON RPC at `endpoint`. E.g., `http://localhost:8545`. pub async fn get_deposit_root( - endpoint: &str, + endpoint: &SensitiveUrl, address: &str, block_number: u64, timeout: Duration, @@ -266,7 +267,7 @@ pub async fn get_deposit_root( /// /// Uses HTTP JSON RPC at `endpoint`. E.g., `http://localhost:8545`. async fn call( - endpoint: &str, + endpoint: &SensitiveUrl, address: &str, hex_data: &str, block_number: u64, @@ -308,7 +309,7 @@ pub struct Log { /// /// Uses HTTP JSON RPC at `endpoint`. E.g., `http://localhost:8545`. pub async fn get_deposit_logs_in_range( - endpoint: &str, + endpoint: &SensitiveUrl, address: &str, block_height_range: Range, timeout: Duration, @@ -353,7 +354,7 @@ pub async fn get_deposit_logs_in_range( /// /// Tries to receive the response and parse the body as a `String`. pub async fn send_rpc_request( - endpoint: &str, + endpoint: &SensitiveUrl, method: &str, params: Value, timeout: Duration, @@ -374,7 +375,7 @@ pub async fn send_rpc_request( .timeout(timeout) .build() .expect("The builder should always build a client") - .post(endpoint) + .post(endpoint.full.clone()) .header(CONTENT_TYPE, "application/json") .body(body) .send() diff --git a/beacon_node/eth1/src/service.rs b/beacon_node/eth1/src/service.rs index 55aae3b6c..0584a4b71 100644 --- a/beacon_node/eth1/src/service.rs +++ b/beacon_node/eth1/src/service.rs @@ -11,6 +11,7 @@ use crate::{ use fallback::{Fallback, FallbackError}; use futures::future::TryFutureExt; use parking_lot::{RwLock, RwLockReadGuard}; +use sensitive_url::SensitiveUrl; use serde::{Deserialize, Serialize}; use slog::{crit, debug, error, info, trace, warn, Logger}; use std::fmt::Debug; @@ -26,6 +27,8 @@ use types::{ChainSpec, EthSpec, Unsigned}; pub const DEFAULT_NETWORK_ID: Eth1Id = Eth1Id::Goerli; /// Indicates the default eth1 chain id we use for the deposit contract. pub const DEFAULT_CHAIN_ID: Eth1Id = Eth1Id::Goerli; +/// Indicates the default eth1 endpoint. +pub const DEFAULT_ETH1_ENDPOINT: &str = "http://localhost:8545"; const STANDARD_TIMEOUT_MILLIS: u64 = 15_000; @@ -51,7 +54,7 @@ pub enum EndpointError { type EndpointState = Result<(), EndpointError>; -type EndpointWithState = (String, TRwLock>); +type EndpointWithState = (SensitiveUrl, TRwLock>); /// A cache structure to lazily check usability of endpoints. An endpoint is usable if it is /// reachable and has the correct network id and chain id. Emits a `WARN` log if a checked endpoint @@ -74,7 +77,10 @@ impl EndpointsCache { if let Some(result) = *value { return result; } - crate::metrics::inc_counter_vec(&crate::metrics::ENDPOINT_REQUESTS, &[&endpoint.0]); + crate::metrics::inc_counter_vec( + &crate::metrics::ENDPOINT_REQUESTS, + &[&endpoint.0.to_string()], + ); let state = endpoint_state( &endpoint.0, &self.config_network_id, @@ -84,7 +90,10 @@ impl EndpointsCache { .await; *value = Some(state); if state.is_err() { - crate::metrics::inc_counter_vec(&crate::metrics::ENDPOINT_ERRORS, &[&endpoint.0]); + crate::metrics::inc_counter_vec( + &crate::metrics::ENDPOINT_ERRORS, + &[&endpoint.0.to_string()], + ); } state } @@ -94,7 +103,7 @@ impl EndpointsCache { func: F, ) -> Result> where - F: Fn(&'a str) -> R, + F: Fn(&'a SensitiveUrl) -> R, R: Future>, { let func = &func; @@ -102,7 +111,7 @@ impl EndpointsCache { .first_success(|endpoint| async move { match self.state(endpoint).await { Ok(()) => { - let endpoint_str = &endpoint.0; + let endpoint_str = &endpoint.0.to_string(); crate::metrics::inc_counter_vec( &crate::metrics::ENDPOINT_REQUESTS, &[endpoint_str], @@ -131,7 +140,7 @@ impl EndpointsCache { /// Returns `Ok` if the endpoint is usable, i.e. is reachable and has a correct network id and /// chain id. Otherwise it returns `Err`. async fn endpoint_state( - endpoint: &str, + endpoint: &SensitiveUrl, config_network_id: &Eth1Id, config_chain_id: &Eth1Id, log: &Logger, @@ -140,7 +149,7 @@ async fn endpoint_state( warn!( log, "Error connecting to eth1 node endpoint"; - "endpoint" => endpoint, + "endpoint" => %endpoint, "action" => "trying fallbacks" ); EndpointError::NotReachable @@ -152,7 +161,7 @@ async fn endpoint_state( warn!( log, "Invalid eth1 network id on endpoint. Please switch to correct network id"; - "endpoint" => endpoint, + "endpoint" => %endpoint, "action" => "trying fallbacks", "expected" => format!("{:?}",config_network_id), "received" => format!("{:?}",network_id), @@ -168,7 +177,7 @@ async fn endpoint_state( warn!( log, "Remote eth1 node is not synced"; - "endpoint" => endpoint, + "endpoint" => %endpoint, "action" => "trying fallbacks" ); return Err(EndpointError::FarBehind); @@ -177,7 +186,7 @@ async fn endpoint_state( warn!( log, "Invalid eth1 chain id. Please switch to correct chain id on endpoint"; - "endpoint" => endpoint, + "endpoint" => %endpoint, "action" => "trying fallbacks", "expected" => format!("{:?}",config_chain_id), "received" => format!("{:?}", chain_id), @@ -198,7 +207,7 @@ pub enum HeadType { /// Returns the head block and the new block ranges relevant for deposits and the block cache /// from the given endpoint. async fn get_remote_head_and_new_block_ranges( - endpoint: &str, + endpoint: &SensitiveUrl, service: &Service, node_far_behind_seconds: u64, ) -> Result< @@ -218,7 +227,7 @@ async fn get_remote_head_and_new_block_ranges( warn!( service.log, "Eth1 endpoint is not synced"; - "endpoint" => endpoint, + "endpoint" => %endpoint, "last_seen_block_unix_timestamp" => remote_head_block.timestamp, "action" => "trying fallback" ); @@ -230,7 +239,7 @@ async fn get_remote_head_and_new_block_ranges( warn!( service.log, "Eth1 endpoint is not synced"; - "endpoint" => endpoint, + "endpoint" => %endpoint, "action" => "trying fallbacks" ); } @@ -252,7 +261,7 @@ async fn get_remote_head_and_new_block_ranges( /// Returns the range of new block numbers to be considered for the given head type from the given /// endpoint. async fn relevant_new_block_numbers_from_endpoint( - endpoint: &str, + endpoint: &SensitiveUrl, service: &Service, head_type: HeadType, ) -> Result>, SingleEndpointError> { @@ -319,7 +328,7 @@ pub struct DepositCacheUpdateOutcome { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Config { /// An Eth1 node (e.g., Geth) running a HTTP JSON-RPC endpoint. - pub endpoints: Vec, + pub endpoints: Vec, /// The address the `BlockCache` and `DepositCache` should assume is the canonical deposit contract. pub deposit_contract_address: String, /// The eth1 network id where the deposit contract is deployed (Goerli/Mainnet). @@ -383,7 +392,8 @@ impl Config { impl Default for Config { fn default() -> Self { Self { - endpoints: vec!["http://localhost:8545".into()], + endpoints: vec![SensitiveUrl::parse(DEFAULT_ETH1_ENDPOINT) + .expect("The default Eth1 endpoint must always be a valid URL.")], deposit_contract_address: "0x0000000000000000000000000000000000000000".into(), network_id: DEFAULT_NETWORK_ID, chain_id: DEFAULT_CHAIN_ID, @@ -1137,7 +1147,7 @@ fn relevant_block_range( /// /// Performs three async calls to an Eth1 HTTP JSON RPC endpoint. async fn download_eth1_block( - endpoint: &str, + endpoint: &SensitiveUrl, cache: Arc, block_number_opt: Option, ) -> Result { @@ -1182,6 +1192,12 @@ mod tests { use super::*; use types::MainnetEthSpec; + #[test] + // Ensures the default config does not panic. + fn default_config() { + Config::default(); + } + #[test] fn serde_serialize() { let serialized = diff --git a/beacon_node/eth1/tests/test.rs b/beacon_node/eth1/tests/test.rs index db9b79f8e..3a503e78b 100644 --- a/beacon_node/eth1/tests/test.rs +++ b/beacon_node/eth1/tests/test.rs @@ -5,6 +5,7 @@ use eth1::{Config, Service}; use eth1::{DepositCache, DEFAULT_CHAIN_ID, DEFAULT_NETWORK_ID}; use eth1_test_rig::GanacheEth1Instance; use merkle_proof::verify_merkle_proof; +use sensitive_url::SensitiveUrl; use slog::Logger; use sloggers::{null::NullLoggerBuilder, Build}; use std::ops::Range; @@ -53,7 +54,7 @@ fn random_deposit_data() -> DepositData { /// Blocking operation to get the deposit logs from the `deposit_contract`. async fn blocking_deposit_logs(eth1: &GanacheEth1Instance, range: Range) -> Vec { get_deposit_logs_in_range( - ð1.endpoint(), + &SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap(), ð1.deposit_contract.address(), range, timeout(), @@ -65,7 +66,7 @@ async fn blocking_deposit_logs(eth1: &GanacheEth1Instance, range: Range) -> /// Blocking operation to get the deposit root from the `deposit_contract`. async fn blocking_deposit_root(eth1: &GanacheEth1Instance, block_number: u64) -> Option { get_deposit_root( - ð1.endpoint(), + &SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap(), ð1.deposit_contract.address(), block_number, timeout(), @@ -77,7 +78,7 @@ async fn blocking_deposit_root(eth1: &GanacheEth1Instance, block_number: u64) -> /// Blocking operation to get the deposit count from the `deposit_contract`. async fn blocking_deposit_count(eth1: &GanacheEth1Instance, block_number: u64) -> Option { get_deposit_count( - ð1.endpoint(), + &SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap(), ð1.deposit_contract.address(), block_number, timeout(), @@ -119,7 +120,7 @@ mod eth1_cache { let service = Service::new( Config { - endpoints: vec![eth1.endpoint()], + endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()], deposit_contract_address: deposit_contract.address(), lowest_cached_block_number: initial_block_number, follow_distance, @@ -200,7 +201,7 @@ mod eth1_cache { let service = Service::new( Config { - endpoints: vec![eth1.endpoint()], + endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()], deposit_contract_address: deposit_contract.address(), lowest_cached_block_number: get_block_number(&web3).await, follow_distance: 0, @@ -255,7 +256,7 @@ mod eth1_cache { let service = Service::new( Config { - endpoints: vec![eth1.endpoint()], + endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()], deposit_contract_address: deposit_contract.address(), lowest_cached_block_number: get_block_number(&web3).await, follow_distance: 0, @@ -306,7 +307,7 @@ mod eth1_cache { let service = Service::new( Config { - endpoints: vec![eth1.endpoint()], + endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()], deposit_contract_address: deposit_contract.address(), lowest_cached_block_number: get_block_number(&web3).await, follow_distance: 0, @@ -359,7 +360,7 @@ mod deposit_tree { let service = Service::new( Config { - endpoints: vec![eth1.endpoint()], + endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()], deposit_contract_address: deposit_contract.address(), deposit_contract_deploy_block: start_block, follow_distance: 0, @@ -440,7 +441,7 @@ mod deposit_tree { let service = Service::new( Config { - endpoints: vec![eth1.endpoint()], + endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()], deposit_contract_address: deposit_contract.address(), deposit_contract_deploy_block: start_block, lowest_cached_block_number: start_block, @@ -582,7 +583,7 @@ mod http { async fn get_block(eth1: &GanacheEth1Instance, block_number: u64) -> Block { eth1::http::get_block( - ð1.endpoint(), + &SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap(), BlockQuery::Number(block_number), timeout(), ) @@ -698,7 +699,7 @@ mod fast { let now = get_block_number(&web3).await; let service = Service::new( Config { - endpoints: vec![eth1.endpoint()], + endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()], deposit_contract_address: deposit_contract.address(), deposit_contract_deploy_block: now, lowest_cached_block_number: now, @@ -775,7 +776,7 @@ mod persist { let now = get_block_number(&web3).await; let config = Config { - endpoints: vec![eth1.endpoint()], + endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()], deposit_contract_address: deposit_contract.address(), deposit_contract_deploy_block: now, lowest_cached_block_number: now, @@ -885,7 +886,10 @@ mod fallbacks { let service = Service::new( Config { - endpoints: vec![endpoint1.endpoint(), endpoint2.endpoint()], + endpoints: vec![ + SensitiveUrl::parse(endpoint1.endpoint().as_str()).unwrap(), + SensitiveUrl::parse(endpoint2.endpoint().as_str()).unwrap(), + ], deposit_contract_address: deposit_contract.address(), lowest_cached_block_number: initial_block_number, follow_distance: 0, @@ -961,7 +965,10 @@ mod fallbacks { let service = Service::new( Config { - endpoints: vec![endpoint2.endpoint(), endpoint1.endpoint()], + endpoints: vec![ + SensitiveUrl::parse(endpoint2.endpoint().as_str()).unwrap(), + SensitiveUrl::parse(endpoint1.endpoint().as_str()).unwrap(), + ], deposit_contract_address: deposit_contract.address(), lowest_cached_block_number: initial_block_number, follow_distance: 0, @@ -1028,7 +1035,10 @@ mod fallbacks { let service = Service::new( Config { - endpoints: vec![endpoint2.endpoint(), endpoint1.endpoint()], + endpoints: vec![ + SensitiveUrl::parse(endpoint2.endpoint().as_str()).unwrap(), + SensitiveUrl::parse(endpoint1.endpoint().as_str()).unwrap(), + ], deposit_contract_address: deposit_contract.address(), lowest_cached_block_number: initial_block_number, follow_distance: 0, @@ -1081,7 +1091,10 @@ mod fallbacks { let service = Service::new( Config { - endpoints: vec![endpoint1.endpoint(), endpoint2.endpoint()], + endpoints: vec![ + SensitiveUrl::parse(endpoint1.endpoint().as_str()).unwrap(), + SensitiveUrl::parse(endpoint2.endpoint().as_str()).unwrap(), + ], deposit_contract_address: deposit_contract.address(), lowest_cached_block_number: initial_block_number, follow_distance: 0, diff --git a/beacon_node/genesis/Cargo.toml b/beacon_node/genesis/Cargo.toml index ef9f21401..67bc30f1d 100644 --- a/beacon_node/genesis/Cargo.toml +++ b/beacon_node/genesis/Cargo.toml @@ -7,6 +7,7 @@ edition = "2018" [dev-dependencies] eth1_test_rig = { path = "../../testing/eth1_test_rig" } tokio-compat-02 = "0.1" +sensitive_url = { path = "../../common/sensitive_url" } [dependencies] futures = "0.3.7" diff --git a/beacon_node/genesis/tests/tests.rs b/beacon_node/genesis/tests/tests.rs index c5fd21928..d9ed18086 100644 --- a/beacon_node/genesis/tests/tests.rs +++ b/beacon_node/genesis/tests/tests.rs @@ -7,6 +7,7 @@ use environment::{Environment, EnvironmentBuilder}; use eth1::{DEFAULT_CHAIN_ID, DEFAULT_NETWORK_ID}; use eth1_test_rig::{DelayThenDeposit, GanacheEth1Instance}; use genesis::{Eth1Config, Eth1GenesisService}; +use sensitive_url::SensitiveUrl; use state_processing::is_valid_genesis_state; use std::time::Duration; use tokio_compat_02::FutureExt; @@ -46,7 +47,7 @@ fn basic() { let service = Eth1GenesisService::new( Eth1Config { - endpoints: vec![eth1.endpoint()], + endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()], deposit_contract_address: deposit_contract.address(), deposit_contract_deploy_block: now, lowest_cached_block_number: now, diff --git a/beacon_node/http_api/Cargo.toml b/beacon_node/http_api/Cargo.toml index 4d5d88d40..606fd7247 100644 --- a/beacon_node/http_api/Cargo.toml +++ b/beacon_node/http_api/Cargo.toml @@ -35,3 +35,4 @@ store = { path = "../store" } environment = { path = "../../lighthouse/environment" } tree_hash = "0.1.1" discv5 = { git = "https://github.com/sigp/discv5 ", rev = "02d2c896c66f8dc2b848c3996fedcd98e1dfec69", features = ["libp2p"] } +sensitive_url = { path = "../../common/sensitive_url" } diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 6027d76a8..6d1dfbf55 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -8,7 +8,7 @@ use discv5::enr::{CombinedKey, EnrBuilder}; use environment::null_logger; use eth2::Error; use eth2::StatusCode; -use eth2::{types::*, BeaconNodeHttpClient, Url}; +use eth2::{types::*, BeaconNodeHttpClient}; use eth2_libp2p::{ rpc::methods::MetaData, types::{EnrBitfield, SyncState}, @@ -18,6 +18,7 @@ use futures::stream::{Stream, StreamExt}; use futures::FutureExt; use http_api::{Config, Context}; use network::NetworkMessage; +use sensitive_url::SensitiveUrl; use slot_clock::SlotClock; use state_processing::per_slot_processing; use std::convert::TryInto; @@ -200,7 +201,7 @@ impl ApiTester { tokio::spawn(async { server.await }); let client = BeaconNodeHttpClient::new( - Url::parse(&format!( + SensitiveUrl::parse(&format!( "http://{}:{}", listening_socket.ip(), listening_socket.port() @@ -307,7 +308,7 @@ impl ApiTester { tokio::spawn(async { server.await }); let client = BeaconNodeHttpClient::new( - Url::parse(&format!( + SensitiveUrl::parse(&format!( "http://{}:{}", listening_socket.ip(), listening_socket.port() diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 256f7e04f..20e8e64d9 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -4,6 +4,7 @@ use client::{ClientConfig, ClientGenesis}; use directory::{DEFAULT_BEACON_NODE_DIR, DEFAULT_NETWORK_DIR, DEFAULT_ROOT_DIR}; use eth2_libp2p::{multiaddr::Protocol, Enr, Multiaddr, NetworkConfig, PeerIdSerialized}; use eth2_network_config::{Eth2NetworkConfig, DEFAULT_HARDCODED_NETWORK}; +use sensitive_url::SensitiveUrl; use slog::{info, warn, Logger}; use std::cmp; use std::cmp::max; @@ -163,17 +164,21 @@ pub fn get_config( } // Defines the URL to reach the eth1 node. - if let Some(val) = cli_args.value_of("eth1-endpoint") { + if let Some(endpoint) = cli_args.value_of("eth1-endpoint") { warn!( log, "The --eth1-endpoint flag is deprecated"; "msg" => "please use --eth1-endpoints instead" ); client_config.sync_eth1_chain = true; - client_config.eth1.endpoints = vec![val.to_string()]; - } else if let Some(val) = cli_args.value_of("eth1-endpoints") { - client_config.sync_eth1_chain = true; - client_config.eth1.endpoints = val.split(',').map(String::from).collect(); + client_config.eth1.endpoints = vec![SensitiveUrl::parse(endpoint) + .map_err(|e| format!("eth1-endpoint was an invalid URL: {:?}", e))?]; + } else if let Some(endpoints) = cli_args.value_of("eth1-endpoints") { + client_config.eth1.endpoints = endpoints + .split(',') + .map(|s| SensitiveUrl::parse(s)) + .collect::>() + .map_err(|e| format!("eth1-endpoints contains an invalid URL {:?}", e))?; } if let Some(val) = cli_args.value_of("eth1-blocks-per-log-query") { diff --git a/common/eth2/Cargo.toml b/common/eth2/Cargo.toml index 72b18d13f..e6b69c968 100644 --- a/common/eth2/Cargo.toml +++ b/common/eth2/Cargo.toml @@ -21,6 +21,7 @@ libsecp256k1 = "0.3.5" ring = "0.16.19" bytes = "1.0.1" account_utils = { path = "../../common/account_utils" } +sensitive_url = { path = "../../common/sensitive_url" } eth2_ssz = "0.1.2" eth2_ssz_derive = "0.1.0" futures-util = "0.3.8" diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 13c117b32..5e1b43460 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -19,6 +19,7 @@ use futures_util::StreamExt; pub use reqwest; use reqwest::{IntoUrl, Response}; pub use reqwest::{StatusCode, Url}; +use sensitive_url::SensitiveUrl; use serde::{de::DeserializeOwned, Serialize}; use ssz::Decode; use std::convert::TryFrom; @@ -36,7 +37,7 @@ pub enum Error { /// The server returned an error message where the body was unable to be parsed. StatusCode(StatusCode), /// The supplied URL is badly formatted. It should look something like `http://127.0.0.1:5052`. - InvalidUrl(Url), + InvalidUrl(SensitiveUrl), /// The supplied validator client secret is invalid. InvalidSecret(String), /// The server returned a response with an invalid signature. It may be an impostor. @@ -81,7 +82,7 @@ impl fmt::Display for Error { #[derive(Clone)] pub struct BeaconNodeHttpClient { client: reqwest::Client, - server: Url, + server: SensitiveUrl, } impl fmt::Display for BeaconNodeHttpClient { @@ -92,25 +93,25 @@ impl fmt::Display for BeaconNodeHttpClient { impl AsRef for BeaconNodeHttpClient { fn as_ref(&self) -> &str { - self.server.as_str() + self.server.as_ref() } } impl BeaconNodeHttpClient { - pub fn new(server: Url) -> Self { + pub fn new(server: SensitiveUrl) -> Self { Self { client: reqwest::Client::new(), server, } } - pub fn from_components(server: Url, client: reqwest::Client) -> Self { + pub fn from_components(server: SensitiveUrl, client: reqwest::Client) -> Self { Self { client, server } } /// Return the path with the standard `/eth1/v1` prefix applied. fn eth_path(&self) -> Result { - let mut path = self.server.clone(); + let mut path = self.server.full.clone(); path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? diff --git a/common/eth2/src/lighthouse.rs b/common/eth2/src/lighthouse.rs index a879b7c8d..7ea051e2e 100644 --- a/common/eth2/src/lighthouse.rs +++ b/common/eth2/src/lighthouse.rs @@ -214,7 +214,7 @@ impl BeaconNodeHttpClient { /// `GET lighthouse/health` pub async fn get_lighthouse_health(&self) -> Result, Error> { - let mut path = self.server.clone(); + let mut path = self.server.full.clone(); path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? @@ -226,7 +226,7 @@ impl BeaconNodeHttpClient { /// `GET lighthouse/syncing` pub async fn get_lighthouse_syncing(&self) -> Result, Error> { - let mut path = self.server.clone(); + let mut path = self.server.full.clone(); path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? @@ -246,7 +246,7 @@ impl BeaconNodeHttpClient { /// `GET lighthouse/proto_array` pub async fn get_lighthouse_proto_array(&self) -> Result, Error> { - let mut path = self.server.clone(); + let mut path = self.server.full.clone(); path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? @@ -261,7 +261,7 @@ impl BeaconNodeHttpClient { &self, epoch: Epoch, ) -> Result, Error> { - let mut path = self.server.clone(); + let mut path = self.server.full.clone(); path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? @@ -279,7 +279,7 @@ impl BeaconNodeHttpClient { epoch: Epoch, validator_id: ValidatorId, ) -> Result>, Error> { - let mut path = self.server.clone(); + let mut path = self.server.full.clone(); path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? @@ -295,7 +295,7 @@ impl BeaconNodeHttpClient { pub async fn get_lighthouse_eth1_syncing( &self, ) -> Result, Error> { - let mut path = self.server.clone(); + let mut path = self.server.full.clone(); path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? @@ -310,7 +310,7 @@ impl BeaconNodeHttpClient { pub async fn get_lighthouse_eth1_block_cache( &self, ) -> Result>, Error> { - let mut path = self.server.clone(); + let mut path = self.server.full.clone(); path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? @@ -325,7 +325,7 @@ impl BeaconNodeHttpClient { pub async fn get_lighthouse_eth1_deposit_cache( &self, ) -> Result>, Error> { - let mut path = self.server.clone(); + let mut path = self.server.full.clone(); path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? @@ -341,7 +341,7 @@ impl BeaconNodeHttpClient { &self, state_id: &StateId, ) -> Result>, Error> { - let mut path = self.server.clone(); + let mut path = self.server.full.clone(); path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? @@ -359,7 +359,7 @@ impl BeaconNodeHttpClient { /// `GET lighthouse/staking` pub async fn get_lighthouse_staking(&self) -> Result { - let mut path = self.server.clone(); + let mut path = self.server.full.clone(); path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? diff --git a/common/eth2/src/lighthouse_vc/http_client.rs b/common/eth2/src/lighthouse_vc/http_client.rs index 2258e93a2..6cb1bf338 100644 --- a/common/eth2/src/lighthouse_vc/http_client.rs +++ b/common/eth2/src/lighthouse_vc/http_client.rs @@ -8,6 +8,7 @@ use reqwest::{ }; use ring::digest::{digest, SHA256}; use secp256k1::{Message, PublicKey, Signature}; +use sensitive_url::SensitiveUrl; use serde::{de::DeserializeOwned, Serialize}; pub use reqwest; @@ -18,7 +19,7 @@ pub use reqwest::{Response, StatusCode, Url}; #[derive(Clone)] pub struct ValidatorClientHttpClient { client: reqwest::Client, - server: Url, + server: SensitiveUrl, secret: ZeroizeString, server_pubkey: PublicKey, } @@ -53,7 +54,7 @@ pub fn parse_pubkey(secret: &str) -> Result { } impl ValidatorClientHttpClient { - pub fn new(server: Url, secret: String) -> Result { + pub fn new(server: SensitiveUrl, secret: String) -> Result { Ok(Self { client: reqwest::Client::new(), server, @@ -63,7 +64,7 @@ impl ValidatorClientHttpClient { } pub fn from_components( - server: Url, + server: SensitiveUrl, client: reqwest::Client, secret: String, ) -> Result { @@ -187,7 +188,7 @@ impl ValidatorClientHttpClient { /// `GET lighthouse/version` pub async fn get_lighthouse_version(&self) -> Result, Error> { - let mut path = self.server.clone(); + let mut path = self.server.full.clone(); path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? @@ -199,7 +200,7 @@ impl ValidatorClientHttpClient { /// `GET lighthouse/health` pub async fn get_lighthouse_health(&self) -> Result, Error> { - let mut path = self.server.clone(); + let mut path = self.server.full.clone(); path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? @@ -211,7 +212,7 @@ impl ValidatorClientHttpClient { /// `GET lighthouse/spec` pub async fn get_lighthouse_spec(&self) -> Result, Error> { - let mut path = self.server.clone(); + let mut path = self.server.full.clone(); path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? @@ -225,7 +226,7 @@ impl ValidatorClientHttpClient { pub async fn get_lighthouse_validators( &self, ) -> Result>, Error> { - let mut path = self.server.clone(); + let mut path = self.server.full.clone(); path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? @@ -240,7 +241,7 @@ impl ValidatorClientHttpClient { &self, validator_pubkey: &PublicKeyBytes, ) -> Result>, Error> { - let mut path = self.server.clone(); + let mut path = self.server.full.clone(); path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? @@ -256,7 +257,7 @@ impl ValidatorClientHttpClient { &self, validators: Vec, ) -> Result, Error> { - let mut path = self.server.clone(); + let mut path = self.server.full.clone(); path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? @@ -271,7 +272,7 @@ impl ValidatorClientHttpClient { &self, request: &CreateValidatorsMnemonicRequest, ) -> Result>, Error> { - let mut path = self.server.clone(); + let mut path = self.server.full.clone(); path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? @@ -287,7 +288,7 @@ impl ValidatorClientHttpClient { &self, request: &KeystoreValidatorsPostRequest, ) -> Result, Error> { - let mut path = self.server.clone(); + let mut path = self.server.full.clone(); path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? @@ -304,7 +305,7 @@ impl ValidatorClientHttpClient { voting_pubkey: &PublicKeyBytes, enabled: bool, ) -> Result<(), Error> { - let mut path = self.server.clone(); + let mut path = self.server.full.clone(); path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? diff --git a/common/remote_signer_consumer/Cargo.toml b/common/remote_signer_consumer/Cargo.toml index 67279715c..bda6264c6 100644 --- a/common/remote_signer_consumer/Cargo.toml +++ b/common/remote_signer_consumer/Cargo.toml @@ -13,3 +13,4 @@ reqwest = { version = "0.11.0", features = ["json"] } serde = { version = "1.0.116", features = ["derive"] } tokio = { version = "1.1.0", features = ["time"] } types = { path = "../../consensus/types" } +sensitive_url = { path = "../sensitive_url" } diff --git a/common/remote_signer_consumer/src/http_client.rs b/common/remote_signer_consumer/src/http_client.rs index 951fc5d09..b5b22265b 100644 --- a/common/remote_signer_consumer/src/http_client.rs +++ b/common/remote_signer_consumer/src/http_client.rs @@ -4,17 +4,18 @@ use crate::{ }; use reqwest::StatusCode; pub use reqwest::Url; +use sensitive_url::SensitiveUrl; use types::{Domain, Fork, Hash256}; /// A wrapper around `reqwest::Client` which provides convenience methods /// to interface with a BLS Remote Signer. pub struct RemoteSignerHttpConsumer { client: reqwest::Client, - server: Url, + server: SensitiveUrl, } impl RemoteSignerHttpConsumer { - pub fn from_components(server: Url, client: reqwest::Client) -> Self { + pub fn from_components(server: SensitiveUrl, client: reqwest::Client) -> Self { Self { client, server } } @@ -43,7 +44,7 @@ impl RemoteSignerHttpConsumer { )); } - let mut path = self.server.clone(); + let mut path = self.server.full.clone(); path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? .push("sign") diff --git a/common/remote_signer_consumer/src/lib.rs b/common/remote_signer_consumer/src/lib.rs index cef64d47a..b8ad33f98 100644 --- a/common/remote_signer_consumer/src/lib.rs +++ b/common/remote_signer_consumer/src/lib.rs @@ -20,10 +20,11 @@ //! //! ``` //! use remote_signer_consumer::RemoteSignerHttpConsumer; -//! use reqwest::{ClientBuilder, Url}; +//! use reqwest::ClientBuilder; +//! use sensitive_url::SensitiveUrl; //! use tokio::time::Duration; //! -//! let url: Url = "http://127.0.0.1:9000".parse().unwrap(); +//! let url = SensitiveUrl::parse("http://127.0.0.1:9000").unwrap(); //! let reqwest_client = ClientBuilder::new() //! .timeout(Duration::from_secs(2)) //! .build() @@ -115,6 +116,7 @@ mod http_client; pub use http_client::RemoteSignerHttpConsumer; pub use reqwest::Url; +use sensitive_url::SensitiveUrl; use serde::{Deserialize, Serialize}; use types::{AttestationData, BeaconBlock, Domain, Epoch, EthSpec, Fork, Hash256, SignedRoot}; @@ -125,7 +127,7 @@ pub enum Error { /// The server returned an error message where the body was able to be parsed. ServerMessage(String), /// The supplied URL is badly formatted. It should look something like `http://127.0.0.1:5052`. - InvalidUrl(Url), + InvalidUrl(SensitiveUrl), /// The supplied parameter is invalid. InvalidParameter(String), } diff --git a/common/remote_signer_consumer/tests/post.rs b/common/remote_signer_consumer/tests/post.rs index 68a45aac3..1667ee2c6 100644 --- a/common/remote_signer_consumer/tests/post.rs +++ b/common/remote_signer_consumer/tests/post.rs @@ -1,7 +1,8 @@ mod post { use remote_signer_consumer::{Error, RemoteSignerHttpConsumer}; use remote_signer_test::*; - use reqwest::{ClientBuilder, Url}; + use reqwest::ClientBuilder; + use sensitive_url::SensitiveUrl; use tokio::time::Duration; #[test] @@ -53,7 +54,7 @@ mod post { let (test_signer, _tmp_dir) = set_up_api_test_signer_to_sign_message(); let run_testcase = |u: &str| -> Result { - let url: Url = u.parse().map_err(|e| format!("[ParseError] {:?}", e))?; + let url = SensitiveUrl::parse(u).map_err(|e| format!("{:?}", e))?; let reqwest_client = ClientBuilder::new() .timeout(Duration::from_secs(12)) @@ -66,7 +67,7 @@ mod post { let signature = do_sign_request(&test_client, test_input); signature.map_err(|e| match e { - Error::InvalidUrl(message) => format!("[InvalidUrl] {:?}", message), + Error::InvalidUrl(message) => format!("{:?}", message), Error::Reqwest(re) => { if re.is_builder() { format!("[Reqwest - Builder] {:?}", re.url().unwrap()) @@ -84,25 +85,22 @@ mod post { // url::parser::ParseError. // These cases don't even make it to the step of building a RemoteSignerHttpConsumer. - testcase("", "[ParseError] RelativeUrlWithoutBase"); - testcase("/4/8/15/16/23/42", "[ParseError] RelativeUrlWithoutBase"); - testcase("localhost", "[ParseError] RelativeUrlWithoutBase"); - testcase(":", "[ParseError] RelativeUrlWithoutBase"); - testcase("0.0:0", "[ParseError] RelativeUrlWithoutBase"); - testcase(":aa", "[ParseError] RelativeUrlWithoutBase"); - testcase("0:", "[ParseError] RelativeUrlWithoutBase"); - testcase("ftp://", "[ParseError] EmptyHost"); - testcase("http://", "[ParseError] EmptyHost"); - testcase("http://127.0.0.1:abcd", "[ParseError] InvalidPort"); - testcase("http://280.0.0.1", "[ParseError] InvalidIpv4Address"); + testcase("", "ParseError(RelativeUrlWithoutBase)"); + testcase("/4/8/15/16/23/42", "ParseError(RelativeUrlWithoutBase)"); + testcase("localhost", "ParseError(RelativeUrlWithoutBase)"); + testcase(":", "ParseError(RelativeUrlWithoutBase)"); + testcase("0.0:0", "ParseError(RelativeUrlWithoutBase)"); + testcase(":aa", "ParseError(RelativeUrlWithoutBase)"); + testcase("0:", "ParseError(RelativeUrlWithoutBase)"); + testcase("ftp://", "ParseError(EmptyHost)"); + testcase("http://", "ParseError(EmptyHost)"); + testcase("http://127.0.0.1:abcd", "ParseError(InvalidPort)"); + testcase("http://280.0.0.1", "ParseError(InvalidIpv4Address)"); // `Error::InvalidUrl`. // The RemoteSignerHttpConsumer is created, but fails at `path_segments_mut()`. - testcase( - "localhost:abcd", - "[InvalidUrl] Url { scheme: \"localhost\", username: \"\", password: None, host: None, port: None, path: \"abcd\", query: None, fragment: None }", - ); - testcase("localhost:", "[InvalidUrl] Url { scheme: \"localhost\", username: \"\", password: None, host: None, port: None, path: \"\", query: None, fragment: None }"); + testcase("localhost:abcd", "InvalidUrl(\"URL cannot be a base.\")"); + testcase("localhost:", "InvalidUrl(\"URL cannot be a base.\")"); // `Reqwest::Error` of the `Builder` kind. // POST is not made. @@ -130,7 +128,7 @@ mod post { let (test_signer, _tmp_dir) = set_up_api_test_signer_to_sign_message(); let run_testcase = |u: &str| -> Result { - let url: Url = u.parse().unwrap(); + let url = SensitiveUrl::parse(u).unwrap(); let reqwest_client = ClientBuilder::new() .timeout(Duration::from_secs(12)) diff --git a/common/sensitive_url/Cargo.toml b/common/sensitive_url/Cargo.toml new file mode 100644 index 000000000..b6b0620a0 --- /dev/null +++ b/common/sensitive_url/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "sensitive_url" +version = "0.1.0" +authors = ["Mac L "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +url = "2.2.1" +serde = "1.0.116" diff --git a/common/sensitive_url/src/lib.rs b/common/sensitive_url/src/lib.rs new file mode 100644 index 000000000..b7e620485 --- /dev/null +++ b/common/sensitive_url/src/lib.rs @@ -0,0 +1,105 @@ +use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; +use std::fmt; +use url::Url; + +#[derive(Debug)] +pub enum SensitiveError { + InvalidUrl(String), + ParseError(url::ParseError), + RedactError(String), +} + +// Wrapper around Url which provides a custom `Display` implementation to protect user secrets. +#[derive(Clone)] +pub struct SensitiveUrl { + pub full: Url, + pub redacted: String, +} + +impl fmt::Display for SensitiveUrl { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.redacted.fmt(f) + } +} + +impl fmt::Debug for SensitiveUrl { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.redacted.fmt(f) + } +} + +impl AsRef for SensitiveUrl { + fn as_ref(&self) -> &str { + self.redacted.as_str() + } +} + +impl Serialize for SensitiveUrl { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&self.full.to_string()) + } +} + +impl<'de> Deserialize<'de> for SensitiveUrl { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s: String = Deserialize::deserialize(deserializer)?; + SensitiveUrl::parse(&s) + .map_err(|e| de::Error::custom(format!("Failed to deserialize sensitive URL {:?}", e))) + } +} + +impl SensitiveUrl { + pub fn parse(url: &str) -> Result { + let surl = Url::parse(url).map_err(SensitiveError::ParseError)?; + SensitiveUrl::new(surl) + } + + fn new(full: Url) -> Result { + let mut redacted = full.clone(); + redacted + .path_segments_mut() + .map_err(|_| SensitiveError::InvalidUrl("URL cannot be a base.".to_string()))? + .clear(); + redacted.set_query(None); + + if redacted.has_authority() { + redacted.set_username("").map_err(|_| { + SensitiveError::RedactError("Unable to redact username.".to_string()) + })?; + redacted.set_password(None).map_err(|_| { + SensitiveError::RedactError("Unable to redact password.".to_string()) + })?; + } + + Ok(Self { + full, + redacted: redacted.to_string(), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn redact_remote_url() { + let full = "https://project:secret@example.com/example?somequery"; + let surl = SensitiveUrl::parse(full).unwrap(); + assert_eq!(surl.to_string(), "https://example.com/"); + assert_eq!(surl.full.to_string(), full); + } + #[test] + fn redact_localhost_url() { + let full = "http://localhost:5052/"; + let surl = SensitiveUrl::parse(full).unwrap(); + assert_eq!(surl.to_string(), "http://localhost:5052/"); + assert_eq!(surl.full.to_string(), full); + } +} diff --git a/lcli/Cargo.toml b/lcli/Cargo.toml index 8141c509f..bc9c69c8b 100644 --- a/lcli/Cargo.toml +++ b/lcli/Cargo.toml @@ -39,3 +39,4 @@ account_utils = { path = "../common/account_utils" } eth2_wallet = { path = "../crypto/eth2_wallet" } web3 = "0.14.0" eth1_test_rig = { path = "../testing/eth1_test_rig" } +sensitive_url = { path = "../common/sensitive_url" } diff --git a/lcli/src/eth1_genesis.rs b/lcli/src/eth1_genesis.rs index 530295021..d929b1784 100644 --- a/lcli/src/eth1_genesis.rs +++ b/lcli/src/eth1_genesis.rs @@ -2,6 +2,7 @@ use clap::ArgMatches; use environment::Environment; use eth2_network_config::Eth2NetworkConfig; use genesis::{Eth1Config, Eth1GenesisService}; +use sensitive_url::SensitiveUrl; use ssz::Encode; use std::cmp::max; use std::path::PathBuf; @@ -50,7 +51,11 @@ pub fn run(mut env: Environment, matches: &ArgMatches<'_>) -> Res let mut config = Eth1Config::default(); if let Some(v) = endpoints.clone() { - config.endpoints = v; + config.endpoints = v + .iter() + .map(|s| SensitiveUrl::parse(s)) + .collect::>() + .map_err(|e| format!("Unable to parse eth1 endpoint URL: {:?}", e))?; } config.deposit_contract_address = format!("{:?}", spec.deposit_contract_address); config.deposit_contract_deploy_block = eth2_network_config.deposit_contract_deploy_block; diff --git a/testing/node_test_rig/Cargo.toml b/testing/node_test_rig/Cargo.toml index 0a7fd1a98..addd05e7e 100644 --- a/testing/node_test_rig/Cargo.toml +++ b/testing/node_test_rig/Cargo.toml @@ -18,3 +18,4 @@ genesis = { path = "../../beacon_node/genesis" } eth2 = { path = "../../common/eth2" } validator_client = { path = "../../validator_client" } validator_dir = { path = "../../common/validator_dir", features = ["insecure_keys"] } +sensitive_url = { path = "../../common/sensitive_url" } diff --git a/testing/node_test_rig/src/lib.rs b/testing/node_test_rig/src/lib.rs index ef84a54bd..436b852c2 100644 --- a/testing/node_test_rig/src/lib.rs +++ b/testing/node_test_rig/src/lib.rs @@ -4,10 +4,8 @@ use beacon_node::ProductionBeaconNode; use environment::RuntimeContext; -use eth2::{ - reqwest::{ClientBuilder, Url}, - BeaconNodeHttpClient, -}; +use eth2::{reqwest::ClientBuilder, BeaconNodeHttpClient}; +use sensitive_url::SensitiveUrl; use std::path::PathBuf; use std::time::Duration; use std::time::{SystemTime, UNIX_EPOCH}; @@ -68,9 +66,10 @@ impl LocalBeaconNode { .http_api_listen_addr() .ok_or("A remote beacon node must have a http server")?; - let beacon_node_url: Url = format!("http://{}:{}", listen_addr.ip(), listen_addr.port()) - .parse() - .map_err(|e| format!("Unable to parse beacon node URL: {:?}", e))?; + let beacon_node_url: SensitiveUrl = SensitiveUrl::parse( + format!("http://{}:{}", listen_addr.ip(), listen_addr.port()).as_str(), + ) + .map_err(|e| format!("Unable to parse beacon node URL: {:?}", e))?; let beacon_node_http_client = ClientBuilder::new() .timeout(HTTP_TIMEOUT) .build() diff --git a/testing/remote_signer_test/Cargo.toml b/testing/remote_signer_test/Cargo.toml index 08dd0b538..1daf8c2c5 100644 --- a/testing/remote_signer_test/Cargo.toml +++ b/testing/remote_signer_test/Cargo.toml @@ -17,3 +17,4 @@ serde_json = "1.0.58" tempfile = "3.1.0" tokio = { version = "1.1.0", features = ["time"] } types = { path = "../../consensus/types" } +sensitive_url = { path = "../../common/sensitive_url" } diff --git a/testing/remote_signer_test/src/consumer.rs b/testing/remote_signer_test/src/consumer.rs index cdca7ceca..377d0c489 100644 --- a/testing/remote_signer_test/src/consumer.rs +++ b/testing/remote_signer_test/src/consumer.rs @@ -1,7 +1,8 @@ use crate::*; use remote_signer_client::api_response::SignatureApiResponse; -use remote_signer_consumer::{Error, RemoteSignerHttpConsumer, RemoteSignerObject, Url}; +use remote_signer_consumer::{Error, RemoteSignerHttpConsumer, RemoteSignerObject}; use reqwest::ClientBuilder; +use sensitive_url::SensitiveUrl; use serde::Serialize; use tokio::runtime::Builder; use tokio::time::Duration; @@ -15,7 +16,7 @@ pub fn set_up_test_consumer_with_timeout( test_signer_address: &str, timeout: u64, ) -> RemoteSignerHttpConsumer { - let url: Url = test_signer_address.parse().unwrap(); + let url = SensitiveUrl::parse(test_signer_address).unwrap(); let reqwest_client = ClientBuilder::new() .timeout(Duration::from_secs(timeout)) .build() diff --git a/testing/simulator/Cargo.toml b/testing/simulator/Cargo.toml index 549520c5a..6733ee632 100644 --- a/testing/simulator/Cargo.toml +++ b/testing/simulator/Cargo.toml @@ -18,3 +18,4 @@ eth1_test_rig = { path = "../eth1_test_rig" } env_logger = "0.8.2" clap = "2.33.3" rayon = "1.4.1" +sensitive_url = { path = "../../common/sensitive_url" } diff --git a/testing/simulator/src/eth1_sim.rs b/testing/simulator/src/eth1_sim.rs index 2172aebcb..08e960bee 100644 --- a/testing/simulator/src/eth1_sim.rs +++ b/testing/simulator/src/eth1_sim.rs @@ -10,6 +10,7 @@ use node_test_rig::{ ClientGenesis, ValidatorFiles, }; use rayon::prelude::*; +use sensitive_url::SensitiveUrl; use std::cmp::max; use std::net::{IpAddr, Ipv4Addr}; use std::time::Duration; @@ -84,7 +85,8 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> { let network_id = ganache_eth1_instance.ganache.network_id(); let chain_id = ganache_eth1_instance.ganache.chain_id(); let ganache = ganache_eth1_instance.ganache; - let eth1_endpoint = ganache.endpoint(); + let eth1_endpoint = SensitiveUrl::parse(ganache.endpoint().as_str()) + .expect("Unable to parse ganache endpoint."); let deposit_contract_address = deposit_contract.address(); // Start a timer that produces eth1 blocks on an interval. @@ -133,7 +135,10 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> { for i in 0..node_count - 1 { let mut config = beacon_config.clone(); if i % 2 == 0 { - config.eth1.endpoints.insert(0, INVALID_ADDRESS.to_string()); + config.eth1.endpoints.insert( + 0, + SensitiveUrl::parse(INVALID_ADDRESS).expect("Unable to parse invalid address"), + ); } network.add_beacon_node(config).await?; } diff --git a/testing/simulator/src/local_network.rs b/testing/simulator/src/local_network.rs index 325487f49..50604040b 100644 --- a/testing/simulator/src/local_network.rs +++ b/testing/simulator/src/local_network.rs @@ -4,6 +4,7 @@ use node_test_rig::{ ClientConfig, LocalBeaconNode, LocalValidatorClient, ValidatorConfig, ValidatorFiles, }; use parking_lot::RwLock; +use sensitive_url::SensitiveUrl; use std::{ ops::Deref, time::{SystemTime, UNIX_EPOCH}, @@ -140,9 +141,12 @@ impl LocalNetwork { .expect("Must have http started") }; - let beacon_node = format!("http://{}:{}", socket_addr.ip(), socket_addr.port()); + let beacon_node = SensitiveUrl::parse( + format!("http://{}:{}", socket_addr.ip(), socket_addr.port()).as_str(), + ) + .unwrap(); validator_config.beacon_nodes = if invalid_first_beacon_node { - vec![INVALID_ADDRESS.to_string(), beacon_node] + vec![SensitiveUrl::parse(INVALID_ADDRESS).unwrap(), beacon_node] } else { vec![beacon_node] }; diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index 30b3ce29b..2ac165b16 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -63,3 +63,4 @@ scrypt = { version = "0.5.0", default-features = false } lighthouse_metrics = { path = "../common/lighthouse_metrics" } lazy_static = "1.4.0" fallback = { path = "../common/fallback" } +sensitive_url = { path = "../common/sensitive_url" } diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index 0506565bf..1c01cc983 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -7,6 +7,7 @@ use directory::{ DEFAULT_VALIDATOR_DIR, }; use eth2::types::Graffiti; +use sensitive_url::SensitiveUrl; use serde_derive::{Deserialize, Serialize}; use slog::{info, warn, Logger}; use std::fs; @@ -26,7 +27,7 @@ pub struct Config { /// The http endpoints of the beacon node APIs. /// /// Should be similar to `["http://localhost:8080"]` - pub beacon_nodes: Vec, + pub beacon_nodes: Vec, /// If true, the validator client will still poll for duties and produce blocks even if the /// beacon node is not synced at startup. pub allow_unsynced_beacon_node: bool, @@ -55,10 +56,13 @@ impl Default for Config { .join(DEFAULT_HARDCODED_NETWORK); let validator_dir = base_dir.join(DEFAULT_VALIDATOR_DIR); let secrets_dir = base_dir.join(DEFAULT_SECRET_DIR); + + let beacon_nodes = vec![SensitiveUrl::parse(DEFAULT_BEACON_NODE) + .expect("beacon_nodes must always be a valid url.")]; Self { validator_dir, secrets_dir, - beacon_nodes: vec![DEFAULT_BEACON_NODE.to_string()], + beacon_nodes, allow_unsynced_beacon_node: false, disable_auto_discover: false, init_slashing_protection: false, @@ -111,25 +115,31 @@ impl Config { } if let Some(beacon_nodes) = parse_optional::(cli_args, "beacon-nodes")? { - config.beacon_nodes = beacon_nodes.as_str().split(',').map(String::from).collect() + config.beacon_nodes = beacon_nodes + .split(',') + .map(|s| SensitiveUrl::parse(s)) + .collect::>() + .map_err(|e| format!("Unable to parse beacon node URL: {:?}", e))?; } // To be deprecated. - else if let Some(beacon_node) = parse_optional(cli_args, "beacon-node")? { + else if let Some(beacon_node) = parse_optional::(cli_args, "beacon-node")? { warn!( log, "The --beacon-node flag is deprecated"; "msg" => "please use --beacon-nodes instead" ); - config.beacon_nodes = vec![beacon_node]; + config.beacon_nodes = vec![SensitiveUrl::parse(&beacon_node) + .map_err(|e| format!("Unable to parse beacon node URL: {:?}", e))?]; } // To be deprecated. - else if let Some(server) = parse_optional(cli_args, "server")? { + else if let Some(server) = parse_optional::(cli_args, "server")? { warn!( log, "The --server flag is deprecated"; "msg" => "please use --beacon-nodes instead" ); - config.beacon_nodes = vec![server]; + config.beacon_nodes = vec![SensitiveUrl::parse(&server) + .map_err(|e| format!("Unable to parse beacon node URL: {:?}", e))?]; } if cli_args.is_present("delete-lockfiles") { @@ -227,3 +237,14 @@ impl Config { Ok(config) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + // Ensures the default config does not panic. + fn default_config() { + Config::default(); + } +} diff --git a/validator_client/src/fork_service.rs b/validator_client/src/fork_service.rs index a69edef26..487038f06 100644 --- a/validator_client/src/fork_service.rs +++ b/validator_client/src/fork_service.rs @@ -84,7 +84,7 @@ impl ForkServiceBuilder { std::time::Duration::from_secs(42), ); let candidates = vec![CandidateBeaconNode::new(eth2::BeaconNodeHttpClient::new( - eth2::Url::parse("http://127.0.0.1").unwrap(), + sensitive_url::SensitiveUrl::parse("http://127.0.0.1").unwrap(), ))]; let mut beacon_nodes = BeaconNodeFallback::new(candidates, spec, log.clone()); beacon_nodes.set_slot_clock(slot_clock); diff --git a/validator_client/src/http_api/tests.rs b/validator_client/src/http_api/tests.rs index d7fcc38c1..2a748059c 100644 --- a/validator_client/src/http_api/tests.rs +++ b/validator_client/src/http_api/tests.rs @@ -11,12 +11,10 @@ use account_utils::{ }; use deposit_contract::decode_eth1_tx_data; use environment::null_logger; -use eth2::{ - lighthouse_vc::{http_client::ValidatorClientHttpClient, types::*}, - Url, -}; +use eth2::lighthouse_vc::{http_client::ValidatorClientHttpClient, types::*}; use eth2_keystore::KeystoreBuilder; use parking_lot::RwLock; +use sensitive_url::SensitiveUrl; use slashing_protection::{SlashingDatabase, SLASHING_PROTECTION_FILENAME}; use slot_clock::TestingSlotClock; use std::marker::PhantomData; @@ -33,7 +31,7 @@ type E = MainnetEthSpec; struct ApiTester { client: ValidatorClientHttpClient, initialized_validators: Arc>, - url: Url, + url: SensitiveUrl, _server_shutdown: oneshot::Sender<()>, _validator_dir: TempDir, } @@ -117,7 +115,7 @@ impl ApiTester { tokio::spawn(async { server.await }); - let url = Url::parse(&format!( + let url = SensitiveUrl::parse(&format!( "http://{}:{}", listening_socket.ip(), listening_socket.port() diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 1541fce27..a1673146e 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -28,7 +28,7 @@ use clap::ArgMatches; use duties_service::DutiesService; use environment::RuntimeContext; use eth2::types::StateId; -use eth2::{reqwest::ClientBuilder, BeaconNodeHttpClient, StatusCode, Url}; +use eth2::{reqwest::ClientBuilder, BeaconNodeHttpClient, StatusCode}; use fork_service::{ForkService, ForkServiceBuilder}; use http_api::ApiSecret; use initialized_validators::InitializedValidators; @@ -209,13 +209,9 @@ impl ProductionValidatorClient { })?; } - let beacon_node_urls: Vec = config + let beacon_nodes: Vec = config .beacon_nodes - .iter() - .map(|s| s.parse()) - .collect::>() - .map_err(|e| format!("Unable to parse beacon node URL: {:?}", e))?; - let beacon_nodes: Vec = beacon_node_urls + .clone() .into_iter() .map(|url| { let beacon_node_http_client = ClientBuilder::new()