Add SensitiveUrl to redact user secrets from endpoints (#2326)

## Issue Addressed

#2276 

## Proposed Changes

Add the `SensitiveUrl` struct which wraps `Url` and implements custom `Display` and `Debug` traits to redact user secrets from being logged in eth1 endpoints, beacon node endpoints and metrics.

## Additional Info

This also includes a small rewrite of the eth1 crate to make requests using `Url` instead of `&str`. 
Some error messages have also been changed to remove `Url` data.
This commit is contained in:
Mac L
2021-05-04 01:59:51 +00:00
parent 2ccb358d87
commit 4cc613d644
38 changed files with 362 additions and 143 deletions
+1
View File
@@ -44,3 +44,4 @@ hyper = "0.14.4"
lighthouse_version = { path = "../common/lighthouse_version" }
hex = "0.4.2"
slasher = { path = "../slasher" }
sensitive_url = { path = "../common/sensitive_url" }
+1
View File
@@ -34,3 +34,4 @@ lazy_static = "1.4.0"
task_executor = { path = "../../common/task_executor" }
eth2 = { path = "../../common/eth2" }
fallback = { path = "../../common/fallback" }
sensitive_url = { path = "../../common/sensitive_url" }
+11 -10
View File
@@ -12,6 +12,7 @@
use futures::future::TryFutureExt;
use reqwest::{header::CONTENT_TYPE, ClientBuilder, StatusCode};
use sensitive_url::SensitiveUrl;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use std::ops::Range;
@@ -79,7 +80,7 @@ impl FromStr for Eth1Id {
}
/// Get the eth1 network id of the given endpoint.
pub async fn get_network_id(endpoint: &str, timeout: Duration) -> Result<Eth1Id, String> {
pub async fn get_network_id(endpoint: &SensitiveUrl, timeout: Duration) -> Result<Eth1Id, String> {
let response_body = send_rpc_request(endpoint, "net_version", json!([]), timeout).await?;
Eth1Id::from_str(
response_result(&response_body)?
@@ -90,7 +91,7 @@ pub async fn get_network_id(endpoint: &str, timeout: Duration) -> Result<Eth1Id,
}
/// Get the eth1 chain id of the given endpoint.
pub async fn get_chain_id(endpoint: &str, timeout: Duration) -> Result<Eth1Id, String> {
pub async fn get_chain_id(endpoint: &SensitiveUrl, timeout: Duration) -> Result<Eth1Id, String> {
let response_body = send_rpc_request(endpoint, "eth_chainId", json!([]), timeout).await?;
hex_to_u64_be(
response_result(&response_body)?
@@ -111,7 +112,7 @@ pub struct Block {
/// Returns the current block number.
///
/// Uses HTTP JSON RPC at `endpoint`. E.g., `http://localhost:8545`.
pub async fn get_block_number(endpoint: &str, timeout: Duration) -> Result<u64, String> {
pub async fn get_block_number(endpoint: &SensitiveUrl, timeout: Duration) -> Result<u64, String> {
let response_body = send_rpc_request(endpoint, "eth_blockNumber", json!([]), timeout).await?;
hex_to_u64_be(
response_result(&response_body)?
@@ -126,7 +127,7 @@ pub async fn get_block_number(endpoint: &str, timeout: Duration) -> Result<u64,
///
/// Uses HTTP JSON RPC at `endpoint`. E.g., `http://localhost:8545`.
pub async fn get_block(
endpoint: &str,
endpoint: &SensitiveUrl,
query: BlockQuery,
timeout: Duration,
) -> Result<Block, String> {
@@ -191,7 +192,7 @@ pub async fn get_block(
///
/// Uses HTTP JSON RPC at `endpoint`. E.g., `http://localhost:8545`.
pub async fn get_deposit_count(
endpoint: &str,
endpoint: &SensitiveUrl,
address: &str,
block_number: u64,
timeout: Duration,
@@ -229,7 +230,7 @@ pub async fn get_deposit_count(
///
/// Uses HTTP JSON RPC at `endpoint`. E.g., `http://localhost:8545`.
pub async fn get_deposit_root(
endpoint: &str,
endpoint: &SensitiveUrl,
address: &str,
block_number: u64,
timeout: Duration,
@@ -266,7 +267,7 @@ pub async fn get_deposit_root(
///
/// Uses HTTP JSON RPC at `endpoint`. E.g., `http://localhost:8545`.
async fn call(
endpoint: &str,
endpoint: &SensitiveUrl,
address: &str,
hex_data: &str,
block_number: u64,
@@ -308,7 +309,7 @@ pub struct Log {
///
/// Uses HTTP JSON RPC at `endpoint`. E.g., `http://localhost:8545`.
pub async fn get_deposit_logs_in_range(
endpoint: &str,
endpoint: &SensitiveUrl,
address: &str,
block_height_range: Range<u64>,
timeout: Duration,
@@ -353,7 +354,7 @@ pub async fn get_deposit_logs_in_range(
///
/// Tries to receive the response and parse the body as a `String`.
pub async fn send_rpc_request(
endpoint: &str,
endpoint: &SensitiveUrl,
method: &str,
params: Value,
timeout: Duration,
@@ -374,7 +375,7 @@ pub async fn send_rpc_request(
.timeout(timeout)
.build()
.expect("The builder should always build a client")
.post(endpoint)
.post(endpoint.full.clone())
.header(CONTENT_TYPE, "application/json")
.body(body)
.send()
+33 -17
View File
@@ -11,6 +11,7 @@ use crate::{
use fallback::{Fallback, FallbackError};
use futures::future::TryFutureExt;
use parking_lot::{RwLock, RwLockReadGuard};
use sensitive_url::SensitiveUrl;
use serde::{Deserialize, Serialize};
use slog::{crit, debug, error, info, trace, warn, Logger};
use std::fmt::Debug;
@@ -26,6 +27,8 @@ use types::{ChainSpec, EthSpec, Unsigned};
pub const DEFAULT_NETWORK_ID: Eth1Id = Eth1Id::Goerli;
/// Indicates the default eth1 chain id we use for the deposit contract.
pub const DEFAULT_CHAIN_ID: Eth1Id = Eth1Id::Goerli;
/// Indicates the default eth1 endpoint.
pub const DEFAULT_ETH1_ENDPOINT: &str = "http://localhost:8545";
const STANDARD_TIMEOUT_MILLIS: u64 = 15_000;
@@ -51,7 +54,7 @@ pub enum EndpointError {
type EndpointState = Result<(), EndpointError>;
type EndpointWithState = (String, TRwLock<Option<EndpointState>>);
type EndpointWithState = (SensitiveUrl, TRwLock<Option<EndpointState>>);
/// A cache structure to lazily check usability of endpoints. An endpoint is usable if it is
/// reachable and has the correct network id and chain id. Emits a `WARN` log if a checked endpoint
@@ -74,7 +77,10 @@ impl EndpointsCache {
if let Some(result) = *value {
return result;
}
crate::metrics::inc_counter_vec(&crate::metrics::ENDPOINT_REQUESTS, &[&endpoint.0]);
crate::metrics::inc_counter_vec(
&crate::metrics::ENDPOINT_REQUESTS,
&[&endpoint.0.to_string()],
);
let state = endpoint_state(
&endpoint.0,
&self.config_network_id,
@@ -84,7 +90,10 @@ impl EndpointsCache {
.await;
*value = Some(state);
if state.is_err() {
crate::metrics::inc_counter_vec(&crate::metrics::ENDPOINT_ERRORS, &[&endpoint.0]);
crate::metrics::inc_counter_vec(
&crate::metrics::ENDPOINT_ERRORS,
&[&endpoint.0.to_string()],
);
}
state
}
@@ -94,7 +103,7 @@ impl EndpointsCache {
func: F,
) -> Result<O, FallbackError<SingleEndpointError>>
where
F: Fn(&'a str) -> R,
F: Fn(&'a SensitiveUrl) -> R,
R: Future<Output = Result<O, SingleEndpointError>>,
{
let func = &func;
@@ -102,7 +111,7 @@ impl EndpointsCache {
.first_success(|endpoint| async move {
match self.state(endpoint).await {
Ok(()) => {
let endpoint_str = &endpoint.0;
let endpoint_str = &endpoint.0.to_string();
crate::metrics::inc_counter_vec(
&crate::metrics::ENDPOINT_REQUESTS,
&[endpoint_str],
@@ -131,7 +140,7 @@ impl EndpointsCache {
/// Returns `Ok` if the endpoint is usable, i.e. is reachable and has a correct network id and
/// chain id. Otherwise it returns `Err`.
async fn endpoint_state(
endpoint: &str,
endpoint: &SensitiveUrl,
config_network_id: &Eth1Id,
config_chain_id: &Eth1Id,
log: &Logger,
@@ -140,7 +149,7 @@ async fn endpoint_state(
warn!(
log,
"Error connecting to eth1 node endpoint";
"endpoint" => endpoint,
"endpoint" => %endpoint,
"action" => "trying fallbacks"
);
EndpointError::NotReachable
@@ -152,7 +161,7 @@ async fn endpoint_state(
warn!(
log,
"Invalid eth1 network id on endpoint. Please switch to correct network id";
"endpoint" => endpoint,
"endpoint" => %endpoint,
"action" => "trying fallbacks",
"expected" => format!("{:?}",config_network_id),
"received" => format!("{:?}",network_id),
@@ -168,7 +177,7 @@ async fn endpoint_state(
warn!(
log,
"Remote eth1 node is not synced";
"endpoint" => endpoint,
"endpoint" => %endpoint,
"action" => "trying fallbacks"
);
return Err(EndpointError::FarBehind);
@@ -177,7 +186,7 @@ async fn endpoint_state(
warn!(
log,
"Invalid eth1 chain id. Please switch to correct chain id on endpoint";
"endpoint" => endpoint,
"endpoint" => %endpoint,
"action" => "trying fallbacks",
"expected" => format!("{:?}",config_chain_id),
"received" => format!("{:?}", chain_id),
@@ -198,7 +207,7 @@ pub enum HeadType {
/// Returns the head block and the new block ranges relevant for deposits and the block cache
/// from the given endpoint.
async fn get_remote_head_and_new_block_ranges(
endpoint: &str,
endpoint: &SensitiveUrl,
service: &Service,
node_far_behind_seconds: u64,
) -> Result<
@@ -218,7 +227,7 @@ async fn get_remote_head_and_new_block_ranges(
warn!(
service.log,
"Eth1 endpoint is not synced";
"endpoint" => endpoint,
"endpoint" => %endpoint,
"last_seen_block_unix_timestamp" => remote_head_block.timestamp,
"action" => "trying fallback"
);
@@ -230,7 +239,7 @@ async fn get_remote_head_and_new_block_ranges(
warn!(
service.log,
"Eth1 endpoint is not synced";
"endpoint" => endpoint,
"endpoint" => %endpoint,
"action" => "trying fallbacks"
);
}
@@ -252,7 +261,7 @@ async fn get_remote_head_and_new_block_ranges(
/// Returns the range of new block numbers to be considered for the given head type from the given
/// endpoint.
async fn relevant_new_block_numbers_from_endpoint(
endpoint: &str,
endpoint: &SensitiveUrl,
service: &Service,
head_type: HeadType,
) -> Result<Option<RangeInclusive<u64>>, SingleEndpointError> {
@@ -319,7 +328,7 @@ pub struct DepositCacheUpdateOutcome {
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Config {
/// An Eth1 node (e.g., Geth) running a HTTP JSON-RPC endpoint.
pub endpoints: Vec<String>,
pub endpoints: Vec<SensitiveUrl>,
/// The address the `BlockCache` and `DepositCache` should assume is the canonical deposit contract.
pub deposit_contract_address: String,
/// The eth1 network id where the deposit contract is deployed (Goerli/Mainnet).
@@ -383,7 +392,8 @@ impl Config {
impl Default for Config {
fn default() -> Self {
Self {
endpoints: vec!["http://localhost:8545".into()],
endpoints: vec![SensitiveUrl::parse(DEFAULT_ETH1_ENDPOINT)
.expect("The default Eth1 endpoint must always be a valid URL.")],
deposit_contract_address: "0x0000000000000000000000000000000000000000".into(),
network_id: DEFAULT_NETWORK_ID,
chain_id: DEFAULT_CHAIN_ID,
@@ -1137,7 +1147,7 @@ fn relevant_block_range(
///
/// Performs three async calls to an Eth1 HTTP JSON RPC endpoint.
async fn download_eth1_block(
endpoint: &str,
endpoint: &SensitiveUrl,
cache: Arc<Inner>,
block_number_opt: Option<u64>,
) -> Result<Eth1Block, SingleEndpointError> {
@@ -1182,6 +1192,12 @@ mod tests {
use super::*;
use types::MainnetEthSpec;
#[test]
// Ensures the default config does not panic.
fn default_config() {
Config::default();
}
#[test]
fn serde_serialize() {
let serialized =
+29 -16
View File
@@ -5,6 +5,7 @@ use eth1::{Config, Service};
use eth1::{DepositCache, DEFAULT_CHAIN_ID, DEFAULT_NETWORK_ID};
use eth1_test_rig::GanacheEth1Instance;
use merkle_proof::verify_merkle_proof;
use sensitive_url::SensitiveUrl;
use slog::Logger;
use sloggers::{null::NullLoggerBuilder, Build};
use std::ops::Range;
@@ -53,7 +54,7 @@ fn random_deposit_data() -> DepositData {
/// Blocking operation to get the deposit logs from the `deposit_contract`.
async fn blocking_deposit_logs(eth1: &GanacheEth1Instance, range: Range<u64>) -> Vec<Log> {
get_deposit_logs_in_range(
&eth1.endpoint(),
&SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap(),
&eth1.deposit_contract.address(),
range,
timeout(),
@@ -65,7 +66,7 @@ async fn blocking_deposit_logs(eth1: &GanacheEth1Instance, range: Range<u64>) ->
/// Blocking operation to get the deposit root from the `deposit_contract`.
async fn blocking_deposit_root(eth1: &GanacheEth1Instance, block_number: u64) -> Option<Hash256> {
get_deposit_root(
&eth1.endpoint(),
&SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap(),
&eth1.deposit_contract.address(),
block_number,
timeout(),
@@ -77,7 +78,7 @@ async fn blocking_deposit_root(eth1: &GanacheEth1Instance, block_number: u64) ->
/// Blocking operation to get the deposit count from the `deposit_contract`.
async fn blocking_deposit_count(eth1: &GanacheEth1Instance, block_number: u64) -> Option<u64> {
get_deposit_count(
&eth1.endpoint(),
&SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap(),
&eth1.deposit_contract.address(),
block_number,
timeout(),
@@ -119,7 +120,7 @@ mod eth1_cache {
let service = Service::new(
Config {
endpoints: vec![eth1.endpoint()],
endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()],
deposit_contract_address: deposit_contract.address(),
lowest_cached_block_number: initial_block_number,
follow_distance,
@@ -200,7 +201,7 @@ mod eth1_cache {
let service = Service::new(
Config {
endpoints: vec![eth1.endpoint()],
endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()],
deposit_contract_address: deposit_contract.address(),
lowest_cached_block_number: get_block_number(&web3).await,
follow_distance: 0,
@@ -255,7 +256,7 @@ mod eth1_cache {
let service = Service::new(
Config {
endpoints: vec![eth1.endpoint()],
endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()],
deposit_contract_address: deposit_contract.address(),
lowest_cached_block_number: get_block_number(&web3).await,
follow_distance: 0,
@@ -306,7 +307,7 @@ mod eth1_cache {
let service = Service::new(
Config {
endpoints: vec![eth1.endpoint()],
endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()],
deposit_contract_address: deposit_contract.address(),
lowest_cached_block_number: get_block_number(&web3).await,
follow_distance: 0,
@@ -359,7 +360,7 @@ mod deposit_tree {
let service = Service::new(
Config {
endpoints: vec![eth1.endpoint()],
endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()],
deposit_contract_address: deposit_contract.address(),
deposit_contract_deploy_block: start_block,
follow_distance: 0,
@@ -440,7 +441,7 @@ mod deposit_tree {
let service = Service::new(
Config {
endpoints: vec![eth1.endpoint()],
endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()],
deposit_contract_address: deposit_contract.address(),
deposit_contract_deploy_block: start_block,
lowest_cached_block_number: start_block,
@@ -582,7 +583,7 @@ mod http {
async fn get_block(eth1: &GanacheEth1Instance, block_number: u64) -> Block {
eth1::http::get_block(
&eth1.endpoint(),
&SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap(),
BlockQuery::Number(block_number),
timeout(),
)
@@ -698,7 +699,7 @@ mod fast {
let now = get_block_number(&web3).await;
let service = Service::new(
Config {
endpoints: vec![eth1.endpoint()],
endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()],
deposit_contract_address: deposit_contract.address(),
deposit_contract_deploy_block: now,
lowest_cached_block_number: now,
@@ -775,7 +776,7 @@ mod persist {
let now = get_block_number(&web3).await;
let config = Config {
endpoints: vec![eth1.endpoint()],
endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()],
deposit_contract_address: deposit_contract.address(),
deposit_contract_deploy_block: now,
lowest_cached_block_number: now,
@@ -885,7 +886,10 @@ mod fallbacks {
let service = Service::new(
Config {
endpoints: vec![endpoint1.endpoint(), endpoint2.endpoint()],
endpoints: vec![
SensitiveUrl::parse(endpoint1.endpoint().as_str()).unwrap(),
SensitiveUrl::parse(endpoint2.endpoint().as_str()).unwrap(),
],
deposit_contract_address: deposit_contract.address(),
lowest_cached_block_number: initial_block_number,
follow_distance: 0,
@@ -961,7 +965,10 @@ mod fallbacks {
let service = Service::new(
Config {
endpoints: vec![endpoint2.endpoint(), endpoint1.endpoint()],
endpoints: vec![
SensitiveUrl::parse(endpoint2.endpoint().as_str()).unwrap(),
SensitiveUrl::parse(endpoint1.endpoint().as_str()).unwrap(),
],
deposit_contract_address: deposit_contract.address(),
lowest_cached_block_number: initial_block_number,
follow_distance: 0,
@@ -1028,7 +1035,10 @@ mod fallbacks {
let service = Service::new(
Config {
endpoints: vec![endpoint2.endpoint(), endpoint1.endpoint()],
endpoints: vec![
SensitiveUrl::parse(endpoint2.endpoint().as_str()).unwrap(),
SensitiveUrl::parse(endpoint1.endpoint().as_str()).unwrap(),
],
deposit_contract_address: deposit_contract.address(),
lowest_cached_block_number: initial_block_number,
follow_distance: 0,
@@ -1081,7 +1091,10 @@ mod fallbacks {
let service = Service::new(
Config {
endpoints: vec![endpoint1.endpoint(), endpoint2.endpoint()],
endpoints: vec![
SensitiveUrl::parse(endpoint1.endpoint().as_str()).unwrap(),
SensitiveUrl::parse(endpoint2.endpoint().as_str()).unwrap(),
],
deposit_contract_address: deposit_contract.address(),
lowest_cached_block_number: initial_block_number,
follow_distance: 0,
+1
View File
@@ -7,6 +7,7 @@ edition = "2018"
[dev-dependencies]
eth1_test_rig = { path = "../../testing/eth1_test_rig" }
tokio-compat-02 = "0.1"
sensitive_url = { path = "../../common/sensitive_url" }
[dependencies]
futures = "0.3.7"
+2 -1
View File
@@ -7,6 +7,7 @@ use environment::{Environment, EnvironmentBuilder};
use eth1::{DEFAULT_CHAIN_ID, DEFAULT_NETWORK_ID};
use eth1_test_rig::{DelayThenDeposit, GanacheEth1Instance};
use genesis::{Eth1Config, Eth1GenesisService};
use sensitive_url::SensitiveUrl;
use state_processing::is_valid_genesis_state;
use std::time::Duration;
use tokio_compat_02::FutureExt;
@@ -46,7 +47,7 @@ fn basic() {
let service = Eth1GenesisService::new(
Eth1Config {
endpoints: vec![eth1.endpoint()],
endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()],
deposit_contract_address: deposit_contract.address(),
deposit_contract_deploy_block: now,
lowest_cached_block_number: now,
+1
View File
@@ -35,3 +35,4 @@ store = { path = "../store" }
environment = { path = "../../lighthouse/environment" }
tree_hash = "0.1.1"
discv5 = { git = "https://github.com/sigp/discv5 ", rev = "02d2c896c66f8dc2b848c3996fedcd98e1dfec69", features = ["libp2p"] }
sensitive_url = { path = "../../common/sensitive_url" }
+4 -3
View File
@@ -8,7 +8,7 @@ use discv5::enr::{CombinedKey, EnrBuilder};
use environment::null_logger;
use eth2::Error;
use eth2::StatusCode;
use eth2::{types::*, BeaconNodeHttpClient, Url};
use eth2::{types::*, BeaconNodeHttpClient};
use eth2_libp2p::{
rpc::methods::MetaData,
types::{EnrBitfield, SyncState},
@@ -18,6 +18,7 @@ use futures::stream::{Stream, StreamExt};
use futures::FutureExt;
use http_api::{Config, Context};
use network::NetworkMessage;
use sensitive_url::SensitiveUrl;
use slot_clock::SlotClock;
use state_processing::per_slot_processing;
use std::convert::TryInto;
@@ -200,7 +201,7 @@ impl ApiTester {
tokio::spawn(async { server.await });
let client = BeaconNodeHttpClient::new(
Url::parse(&format!(
SensitiveUrl::parse(&format!(
"http://{}:{}",
listening_socket.ip(),
listening_socket.port()
@@ -307,7 +308,7 @@ impl ApiTester {
tokio::spawn(async { server.await });
let client = BeaconNodeHttpClient::new(
Url::parse(&format!(
SensitiveUrl::parse(&format!(
"http://{}:{}",
listening_socket.ip(),
listening_socket.port()
+10 -5
View File
@@ -4,6 +4,7 @@ use client::{ClientConfig, ClientGenesis};
use directory::{DEFAULT_BEACON_NODE_DIR, DEFAULT_NETWORK_DIR, DEFAULT_ROOT_DIR};
use eth2_libp2p::{multiaddr::Protocol, Enr, Multiaddr, NetworkConfig, PeerIdSerialized};
use eth2_network_config::{Eth2NetworkConfig, DEFAULT_HARDCODED_NETWORK};
use sensitive_url::SensitiveUrl;
use slog::{info, warn, Logger};
use std::cmp;
use std::cmp::max;
@@ -163,17 +164,21 @@ pub fn get_config<E: EthSpec>(
}
// Defines the URL to reach the eth1 node.
if let Some(val) = cli_args.value_of("eth1-endpoint") {
if let Some(endpoint) = cli_args.value_of("eth1-endpoint") {
warn!(
log,
"The --eth1-endpoint flag is deprecated";
"msg" => "please use --eth1-endpoints instead"
);
client_config.sync_eth1_chain = true;
client_config.eth1.endpoints = vec![val.to_string()];
} else if let Some(val) = cli_args.value_of("eth1-endpoints") {
client_config.sync_eth1_chain = true;
client_config.eth1.endpoints = val.split(',').map(String::from).collect();
client_config.eth1.endpoints = vec![SensitiveUrl::parse(endpoint)
.map_err(|e| format!("eth1-endpoint was an invalid URL: {:?}", e))?];
} else if let Some(endpoints) = cli_args.value_of("eth1-endpoints") {
client_config.eth1.endpoints = endpoints
.split(',')
.map(|s| SensitiveUrl::parse(s))
.collect::<Result<_, _>>()
.map_err(|e| format!("eth1-endpoints contains an invalid URL {:?}", e))?;
}
if let Some(val) = cli_args.value_of("eth1-blocks-per-log-query") {