cerc-io/lighthouse
16
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Add SensitiveUrl to redact user secrets from endpoints (#2326)

Browse Source 
## Issue Addressed

#2276 

## Proposed Changes

Add the `SensitiveUrl` struct which wraps `Url` and implements custom `Display` and `Debug` traits to redact user secrets from being logged in eth1 endpoints, beacon node endpoints and metrics.

## Additional Info

This also includes a small rewrite of the eth1 crate to make requests using `Url` instead of `&str`. 
Some error messages have also been changed to remove `Url` data.
This commit is contained in:
Mac L 2021-05-04 01:59:51 +00:00
parent 2ccb358d87
commit 4cc613d644
38 changed files with 362 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
Cargo.lock generated
View File

@ -25,6 +25,7 @@ dependencies = [
"rand 0.7.3",
"rayon",
"safe_arith",
"sensitive_url",
"slashing_protection",
"slog",
"slog-async",
@ -650,6 +651,7 @@ dependencies = [
"logging",
"node_test_rig",
"rand 0.7.3",
"sensitive_url",
"serde",
"slasher",
"slog",
@ -1896,6 +1898,7 @@ dependencies = [
"merkle_proof",
"parking_lot",
"reqwest",
"sensitive_url",
"serde",
"serde_json",
"slog",
@ -1942,6 +1945,7 @@ dependencies = [
"psutil",
"reqwest",
"ring",
"sensitive_url",
"serde",
"serde_json",
"serde_utils",
@ -2514,6 +2518,7 @@ dependencies = [
"merkle_proof",
"parking_lot",
"rayon",
"sensitive_url",
"serde",
"serde_derive",
"slog",
@ -2844,6 +2849,7 @@ dependencies = [
"lighthouse_version",
"network",
"parking_lot",
"sensitive_url",
"serde",
"slog",
"slot_clock",
@ -3347,6 +3353,7 @@ dependencies = [
"log",
"rand 0.7.3",
"regex",
"sensitive_url",
"serde",
"serde_yaml",
"simple_logger",
@ -4189,6 +4196,7 @@ dependencies = [
"futures 0.3.14",
"genesis",
"reqwest",
"sensitive_url",
"serde",
"tempfile",
"types",
@ -5155,6 +5163,7 @@ dependencies = [
"rand 0.7.3",
"remote_signer_test",
"reqwest",
"sensitive_url",
"serde",
"tokio 1.5.0",
"types",
@ -5171,6 +5180,7 @@ dependencies = [
"remote_signer_client",
"remote_signer_consumer",
"reqwest",
"sensitive_url",
"serde",
"serde_json",
"tempfile",
@ -5499,6 +5509,14 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
[[package]]
name = "sensitive_url"
version = "0.1.0"
dependencies = [
"serde",
"url",
]
[[package]]
name = "serde"
version = "1.0.125"
@ -5714,6 +5732,7 @@ dependencies = [
"node_test_rig",
"parking_lot",
"rayon",
"sensitive_url",
"tokio 1.5.0",
"types",
"validator_client",
@ -7052,6 +7071,7 @@ dependencies = [
"ring",
"safe_arith",
"scrypt",
"sensitive_url",
"serde",
"serde_derive",
"serde_json",

1
Cargo.toml
View File

@ -33,6 +33,7 @@ members = [
"common/logging",
"common/lru_cache",
"common/remote_signer_consumer",
"common/sensitive_url",
"common/slot_clock",
"common/task_executor",
"common/test_random_derive",

1
account_manager/Cargo.toml
View File

@ -34,6 +34,7 @@ slashing_protection = { path = "../validator_client/slashing_protection" }
eth2 = {path = "../common/eth2"}
safe_arith = {path = "../consensus/safe_arith"}
slot_clock = { path = "../common/slot_clock" }
sensitive_url = { path = "../common/sensitive_url" }
[dev-dependencies]
tempfile = "3.1.0"

5
account_manager/src/validator/exit.rs
View File

@ -4,11 +4,12 @@ use clap::{App, Arg, ArgMatches};
use environment::Environment;
use eth2::{
types::{GenesisData, StateId, ValidatorData, ValidatorId, ValidatorStatus},
BeaconNodeHttpClient, Url,
BeaconNodeHttpClient,
};
use eth2_keystore::Keystore;
use eth2_network_config::Eth2NetworkConfig;
use safe_arith::SafeArith;
use sensitive_url::SensitiveUrl;
use slot_clock::{SlotClock, SystemTimeSlotClock};
use std::path::{Path, PathBuf};
use std::time::Duration;
@ -75,7 +76,7 @@ pub fn cli_run<E: EthSpec>(matches: &ArgMatches, env: Environment<E>) -> Result<
let spec = env.eth2_config().spec.clone();
let server_url: String = clap_utils::parse_required(matches, BEACON_SERVER_FLAG)?;
let client = BeaconNodeHttpClient::new(
Url::parse(&server_url)
SensitiveUrl::parse(&server_url)
.map_err(|e| format!("Failed to parse beacon http server: {:?}", e))?,
);

1
beacon_node/Cargo.toml
View File

@ -44,3 +44,4 @@ hyper = "0.14.4"
lighthouse_version = { path = "../common/lighthouse_version" }
hex = "0.4.2"
slasher = { path = "../slasher" }
sensitive_url = { path = "../common/sensitive_url" }

1
beacon_node/eth1/Cargo.toml
View File

@ -34,3 +34,4 @@ lazy_static = "1.4.0"
task_executor = { path = "../../common/task_executor" }
eth2 = { path = "../../common/eth2" }
fallback = { path = "../../common/fallback" }
sensitive_url = { path = "../../common/sensitive_url" }

21
beacon_node/eth1/src/http.rs
View File

@ -12,6 +12,7 @@
use futures::future::TryFutureExt;
use reqwest::{header::CONTENT_TYPE, ClientBuilder, StatusCode};
use sensitive_url::SensitiveUrl;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use std::ops::Range;
@ -79,7 +80,7 @@ impl FromStr for Eth1Id {
}
/// Get the eth1 network id of the given endpoint.
pub async fn get_network_id(endpoint: &str, timeout: Duration) -> Result<Eth1Id, String> {
pub async fn get_network_id(endpoint: &SensitiveUrl, timeout: Duration) -> Result<Eth1Id, String> {
let response_body = send_rpc_request(endpoint, "net_version", json!([]), timeout).await?;
Eth1Id::from_str(
response_result(&response_body)?
@ -90,7 +91,7 @@ pub async fn get_network_id(endpoint: &str, timeout: Duration) -> Result<Eth1Id,
}
/// Get the eth1 chain id of the given endpoint.
pub async fn get_chain_id(endpoint: &str, timeout: Duration) -> Result<Eth1Id, String> {
pub async fn get_chain_id(endpoint: &SensitiveUrl, timeout: Duration) -> Result<Eth1Id, String> {
let response_body = send_rpc_request(endpoint, "eth_chainId", json!([]), timeout).await?;
hex_to_u64_be(
response_result(&response_body)?
@ -111,7 +112,7 @@ pub struct Block {
/// Returns the current block number.
///
/// Uses HTTP JSON RPC at `endpoint`. E.g., `http://localhost:8545`.
pub async fn get_block_number(endpoint: &str, timeout: Duration) -> Result<u64, String> {
pub async fn get_block_number(endpoint: &SensitiveUrl, timeout: Duration) -> Result<u64, String> {
let response_body = send_rpc_request(endpoint, "eth_blockNumber", json!([]), timeout).await?;
hex_to_u64_be(
response_result(&response_body)?
@ -126,7 +127,7 @@ pub async fn get_block_number(endpoint: &str, timeout: Duration) -> Result<u64,
///
/// Uses HTTP JSON RPC at `endpoint`. E.g., `http://localhost:8545`.
pub async fn get_block(
endpoint: &str,
endpoint: &SensitiveUrl,
query: BlockQuery,
timeout: Duration,
) -> Result<Block, String> {
@ -191,7 +192,7 @@ pub async fn get_block(
///
/// Uses HTTP JSON RPC at `endpoint`. E.g., `http://localhost:8545`.
pub async fn get_deposit_count(
endpoint: &str,
endpoint: &SensitiveUrl,
address: &str,
block_number: u64,
timeout: Duration,
@ -229,7 +230,7 @@ pub async fn get_deposit_count(
///
/// Uses HTTP JSON RPC at `endpoint`. E.g., `http://localhost:8545`.
pub async fn get_deposit_root(
endpoint: &str,
endpoint: &SensitiveUrl,
address: &str,
block_number: u64,
timeout: Duration,
@ -266,7 +267,7 @@ pub async fn get_deposit_root(
///
/// Uses HTTP JSON RPC at `endpoint`. E.g., `http://localhost:8545`.
async fn call(
endpoint: &str,
endpoint: &SensitiveUrl,
address: &str,
hex_data: &str,
block_number: u64,
@ -308,7 +309,7 @@ pub struct Log {
///
/// Uses HTTP JSON RPC at `endpoint`. E.g., `http://localhost:8545`.
pub async fn get_deposit_logs_in_range(
endpoint: &str,
endpoint: &SensitiveUrl,
address: &str,
block_height_range: Range<u64>,
timeout: Duration,
@ -353,7 +354,7 @@ pub async fn get_deposit_logs_in_range(
///
/// Tries to receive the response and parse the body as a `String`.
pub async fn send_rpc_request(
endpoint: &str,
endpoint: &SensitiveUrl,
method: &str,
params: Value,
timeout: Duration,
@ -374,7 +375,7 @@ pub async fn send_rpc_request(
.timeout(timeout)
.build()
.expect("The builder should always build a client")
.post(endpoint)
.post(endpoint.full.clone())
.header(CONTENT_TYPE, "application/json")
.body(body)
.send()

50
beacon_node/eth1/src/service.rs
View File

@ -11,6 +11,7 @@ use crate::{
use fallback::{Fallback, FallbackError};
use futures::future::TryFutureExt;
use parking_lot::{RwLock, RwLockReadGuard};
use sensitive_url::SensitiveUrl;
use serde::{Deserialize, Serialize};
use slog::{crit, debug, error, info, trace, warn, Logger};
use std::fmt::Debug;
@ -26,6 +27,8 @@ use types::{ChainSpec, EthSpec, Unsigned};
pub const DEFAULT_NETWORK_ID: Eth1Id = Eth1Id::Goerli;
/// Indicates the default eth1 chain id we use for the deposit contract.
pub const DEFAULT_CHAIN_ID: Eth1Id = Eth1Id::Goerli;
/// Indicates the default eth1 endpoint.
pub const DEFAULT_ETH1_ENDPOINT: &str = "http://localhost:8545";
const STANDARD_TIMEOUT_MILLIS: u64 = 15_000;
@ -51,7 +54,7 @@ pub enum EndpointError {
type EndpointState = Result<(), EndpointError>;
type EndpointWithState = (String, TRwLock<Option<EndpointState>>);
type EndpointWithState = (SensitiveUrl, TRwLock<Option<EndpointState>>);
/// A cache structure to lazily check usability of endpoints. An endpoint is usable if it is
/// reachable and has the correct network id and chain id. Emits a `WARN` log if a checked endpoint
@ -74,7 +77,10 @@ impl EndpointsCache {
if let Some(result) = *value {
return result;
}
crate::metrics::inc_counter_vec(&crate::metrics::ENDPOINT_REQUESTS, &[&endpoint.0]);
crate::metrics::inc_counter_vec(
&crate::metrics::ENDPOINT_REQUESTS,
&[&endpoint.0.to_string()],
);
let state = endpoint_state(
&endpoint.0,
&self.config_network_id,
@ -84,7 +90,10 @@ impl EndpointsCache {
.await;
*value = Some(state);
if state.is_err() {
crate::metrics::inc_counter_vec(&crate::metrics::ENDPOINT_ERRORS, &[&endpoint.0]);
crate::metrics::inc_counter_vec(
&crate::metrics::ENDPOINT_ERRORS,
&[&endpoint.0.to_string()],
);
}
state
}
@ -94,7 +103,7 @@ impl EndpointsCache {
func: F,
) -> Result<O, FallbackError<SingleEndpointError>>
where
F: Fn(&'a str) -> R,
F: Fn(&'a SensitiveUrl) -> R,
R: Future<Output = Result<O, SingleEndpointError>>,
{
let func = &func;
@ -102,7 +111,7 @@ impl EndpointsCache {
.first_success(|endpoint| async move {
match self.state(endpoint).await {
Ok(()) => {
let endpoint_str = &endpoint.0;
let endpoint_str = &endpoint.0.to_string();
crate::metrics::inc_counter_vec(
&crate::metrics::ENDPOINT_REQUESTS,
&[endpoint_str],
@ -131,7 +140,7 @@ impl EndpointsCache {
/// Returns `Ok` if the endpoint is usable, i.e. is reachable and has a correct network id and
/// chain id. Otherwise it returns `Err`.
async fn endpoint_state(
endpoint: &str,
endpoint: &SensitiveUrl,
config_network_id: &Eth1Id,
config_chain_id: &Eth1Id,
log: &Logger,
@ -140,7 +149,7 @@ async fn endpoint_state(
warn!(
log,
"Error connecting to eth1 node endpoint";
"endpoint" => endpoint,
"endpoint" => %endpoint,
"action" => "trying fallbacks"
);
EndpointError::NotReachable
@ -152,7 +161,7 @@ async fn endpoint_state(
warn!(
log,
"Invalid eth1 network id on endpoint. Please switch to correct network id";
"endpoint" => endpoint,
"endpoint" => %endpoint,
"action" => "trying fallbacks",
"expected" => format!("{:?}",config_network_id),
"received" => format!("{:?}",network_id),
@ -168,7 +177,7 @@ async fn endpoint_state(
warn!(
log,
"Remote eth1 node is not synced";
"endpoint" => endpoint,
"endpoint" => %endpoint,
"action" => "trying fallbacks"
);
return Err(EndpointError::FarBehind);
@ -177,7 +186,7 @@ async fn endpoint_state(
warn!(
log,
"Invalid eth1 chain id. Please switch to correct chain id on endpoint";
"endpoint" => endpoint,
"endpoint" => %endpoint,
"action" => "trying fallbacks",
"expected" => format!("{:?}",config_chain_id),
"received" => format!("{:?}", chain_id),
@ -198,7 +207,7 @@ pub enum HeadType {
/// Returns the head block and the new block ranges relevant for deposits and the block cache
/// from the given endpoint.
async fn get_remote_head_and_new_block_ranges(
endpoint: &str,
endpoint: &SensitiveUrl,
service: &Service,
node_far_behind_seconds: u64,
) -> Result<
@ -218,7 +227,7 @@ async fn get_remote_head_and_new_block_ranges(
warn!(
service.log,
"Eth1 endpoint is not synced";
"endpoint" => endpoint,
"endpoint" => %endpoint,
"last_seen_block_unix_timestamp" => remote_head_block.timestamp,
"action" => "trying fallback"
);
@ -230,7 +239,7 @@ async fn get_remote_head_and_new_block_ranges(
warn!(
service.log,
"Eth1 endpoint is not synced";
"endpoint" => endpoint,
"endpoint" => %endpoint,
"action" => "trying fallbacks"
);
}
@ -252,7 +261,7 @@ async fn get_remote_head_and_new_block_ranges(
/// Returns the range of new block numbers to be considered for the given head type from the given
/// endpoint.
async fn relevant_new_block_numbers_from_endpoint(
endpoint: &str,
endpoint: &SensitiveUrl,
service: &Service,
head_type: HeadType,
) -> Result<Option<RangeInclusive<u64>>, SingleEndpointError> {
@ -319,7 +328,7 @@ pub struct DepositCacheUpdateOutcome {
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Config {
/// An Eth1 node (e.g., Geth) running a HTTP JSON-RPC endpoint.
pub endpoints: Vec<String>,
pub endpoints: Vec<SensitiveUrl>,
/// The address the `BlockCache` and `DepositCache` should assume is the canonical deposit contract.
pub deposit_contract_address: String,
/// The eth1 network id where the deposit contract is deployed (Goerli/Mainnet).
@ -383,7 +392,8 @@ impl Config {
impl Default for Config {
fn default() -> Self {
Self {
endpoints: vec!["http://localhost:8545".into()],
endpoints: vec![SensitiveUrl::parse(DEFAULT_ETH1_ENDPOINT)
.expect("The default Eth1 endpoint must always be a valid URL.")],
deposit_contract_address: "0x0000000000000000000000000000000000000000".into(),
network_id: DEFAULT_NETWORK_ID,
chain_id: DEFAULT_CHAIN_ID,
@ -1137,7 +1147,7 @@ fn relevant_block_range(
///
/// Performs three async calls to an Eth1 HTTP JSON RPC endpoint.
async fn download_eth1_block(
endpoint: &str,
endpoint: &SensitiveUrl,
cache: Arc<Inner>,
block_number_opt: Option<u64>,
) -> Result<Eth1Block, SingleEndpointError> {
@ -1182,6 +1192,12 @@ mod tests {
use super::*;
use types::MainnetEthSpec;
#[test]
// Ensures the default config does not panic.
fn default_config() {
Config::default();
}
#[test]
fn serde_serialize() {
let serialized =

45
beacon_node/eth1/tests/test.rs
View File

@ -5,6 +5,7 @@ use eth1::{Config, Service};
use eth1::{DepositCache, DEFAULT_CHAIN_ID, DEFAULT_NETWORK_ID};
use eth1_test_rig::GanacheEth1Instance;
use merkle_proof::verify_merkle_proof;
use sensitive_url::SensitiveUrl;
use slog::Logger;
use sloggers::{null::NullLoggerBuilder, Build};
use std::ops::Range;
@ -53,7 +54,7 @@ fn random_deposit_data() -> DepositData {
/// Blocking operation to get the deposit logs from the `deposit_contract`.
async fn blocking_deposit_logs(eth1: &GanacheEth1Instance, range: Range<u64>) -> Vec<Log> {
get_deposit_logs_in_range(
&eth1.endpoint(),
&SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap(),
&eth1.deposit_contract.address(),
range,
timeout(),
@ -65,7 +66,7 @@ async fn blocking_deposit_logs(eth1: &GanacheEth1Instance, range: Range<u64>) ->
/// Blocking operation to get the deposit root from the `deposit_contract`.
async fn blocking_deposit_root(eth1: &GanacheEth1Instance, block_number: u64) -> Option<Hash256> {
get_deposit_root(
&eth1.endpoint(),
&SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap(),
&eth1.deposit_contract.address(),
block_number,
timeout(),
@ -77,7 +78,7 @@ async fn blocking_deposit_root(eth1: &GanacheEth1Instance, block_number: u64) ->
/// Blocking operation to get the deposit count from the `deposit_contract`.
async fn blocking_deposit_count(eth1: &GanacheEth1Instance, block_number: u64) -> Option<u64> {
get_deposit_count(
&eth1.endpoint(),
&SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap(),
&eth1.deposit_contract.address(),
block_number,
timeout(),
@ -119,7 +120,7 @@ mod eth1_cache {
let service = Service::new(
Config {
endpoints: vec![eth1.endpoint()],
endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()],
deposit_contract_address: deposit_contract.address(),
lowest_cached_block_number: initial_block_number,
follow_distance,
@ -200,7 +201,7 @@ mod eth1_cache {
let service = Service::new(
Config {
endpoints: vec![eth1.endpoint()],
endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()],
deposit_contract_address: deposit_contract.address(),
lowest_cached_block_number: get_block_number(&web3).await,
follow_distance: 0,
@ -255,7 +256,7 @@ mod eth1_cache {
let service = Service::new(
Config {
endpoints: vec![eth1.endpoint()],
endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()],
deposit_contract_address: deposit_contract.address(),
lowest_cached_block_number: get_block_number(&web3).await,
follow_distance: 0,
@ -306,7 +307,7 @@ mod eth1_cache {
let service = Service::new(
Config {
endpoints: vec![eth1.endpoint()],
endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()],
deposit_contract_address: deposit_contract.address(),
lowest_cached_block_number: get_block_number(&web3).await,
follow_distance: 0,
@ -359,7 +360,7 @@ mod deposit_tree {
let service = Service::new(
Config {
endpoints: vec![eth1.endpoint()],
endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()],
deposit_contract_address: deposit_contract.address(),
deposit_contract_deploy_block: start_block,
follow_distance: 0,
@ -440,7 +441,7 @@ mod deposit_tree {
let service = Service::new(
Config {
endpoints: vec![eth1.endpoint()],
endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()],
deposit_contract_address: deposit_contract.address(),
deposit_contract_deploy_block: start_block,
lowest_cached_block_number: start_block,
@ -582,7 +583,7 @@ mod http {
async fn get_block(eth1: &GanacheEth1Instance, block_number: u64) -> Block {
eth1::http::get_block(
&eth1.endpoint(),
&SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap(),
BlockQuery::Number(block_number),
timeout(),
)
@ -698,7 +699,7 @@ mod fast {
let now = get_block_number(&web3).await;
let service = Service::new(
Config {
endpoints: vec![eth1.endpoint()],
endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()],
deposit_contract_address: deposit_contract.address(),
deposit_contract_deploy_block: now,
lowest_cached_block_number: now,
@ -775,7 +776,7 @@ mod persist {
let now = get_block_number(&web3).await;
let config = Config {
endpoints: vec![eth1.endpoint()],
endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()],
deposit_contract_address: deposit_contract.address(),
deposit_contract_deploy_block: now,
lowest_cached_block_number: now,
@ -885,7 +886,10 @@ mod fallbacks {
let service = Service::new(
Config {
endpoints: vec![endpoint1.endpoint(), endpoint2.endpoint()],
endpoints: vec![
SensitiveUrl::parse(endpoint1.endpoint().as_str()).unwrap(),
SensitiveUrl::parse(endpoint2.endpoint().as_str()).unwrap(),
],
deposit_contract_address: deposit_contract.address(),
lowest_cached_block_number: initial_block_number,
follow_distance: 0,
@ -961,7 +965,10 @@ mod fallbacks {
let service = Service::new(
Config {
endpoints: vec![endpoint2.endpoint(), endpoint1.endpoint()],
endpoints: vec![
SensitiveUrl::parse(endpoint2.endpoint().as_str()).unwrap(),
SensitiveUrl::parse(endpoint1.endpoint().as_str()).unwrap(),
],
deposit_contract_address: deposit_contract.address(),
lowest_cached_block_number: initial_block_number,
follow_distance: 0,
@ -1028,7 +1035,10 @@ mod fallbacks {
let service = Service::new(
Config {
endpoints: vec![endpoint2.endpoint(), endpoint1.endpoint()],
endpoints: vec![
SensitiveUrl::parse(endpoint2.endpoint().as_str()).unwrap(),
SensitiveUrl::parse(endpoint1.endpoint().as_str()).unwrap(),
],
deposit_contract_address: deposit_contract.address(),
lowest_cached_block_number: initial_block_number,
follow_distance: 0,
@ -1081,7 +1091,10 @@ mod fallbacks {
let service = Service::new(
Config {
endpoints: vec![endpoint1.endpoint(), endpoint2.endpoint()],
endpoints: vec![
SensitiveUrl::parse(endpoint1.endpoint().as_str()).unwrap(),
SensitiveUrl::parse(endpoint2.endpoint().as_str()).unwrap(),
],
deposit_contract_address: deposit_contract.address(),
lowest_cached_block_number: initial_block_number,
follow_distance: 0,

1
beacon_node/genesis/Cargo.toml
View File

@ -7,6 +7,7 @@ edition = "2018"
[dev-dependencies]
eth1_test_rig = { path = "../../testing/eth1_test_rig" }
tokio-compat-02 = "0.1"
sensitive_url = { path = "../../common/sensitive_url" }
[dependencies]
futures = "0.3.7"

3
beacon_node/genesis/tests/tests.rs
View File

@ -7,6 +7,7 @@ use environment::{Environment, EnvironmentBuilder};
use eth1::{DEFAULT_CHAIN_ID, DEFAULT_NETWORK_ID};
use eth1_test_rig::{DelayThenDeposit, GanacheEth1Instance};
use genesis::{Eth1Config, Eth1GenesisService};
use sensitive_url::SensitiveUrl;
use state_processing::is_valid_genesis_state;
use std::time::Duration;
use tokio_compat_02::FutureExt;
@ -46,7 +47,7 @@ fn basic() {
let service = Eth1GenesisService::new(
Eth1Config {
endpoints: vec![eth1.endpoint()],
endpoints: vec![SensitiveUrl::parse(eth1.endpoint().as_str()).unwrap()],
deposit_contract_address: deposit_contract.address(),
deposit_contract_deploy_block: now,
lowest_cached_block_number: now,

1
beacon_node/http_api/Cargo.toml
View File

@ -35,3 +35,4 @@ store = { path = "../store" }
environment = { path = "../../lighthouse/environment" }
tree_hash = "0.1.1"
discv5 = { git = "https://github.com/sigp/discv5 ", rev = "02d2c896c66f8dc2b848c3996fedcd98e1dfec69", features = ["libp2p"] }
sensitive_url = { path = "../../common/sensitive_url" }

7
beacon_node/http_api/tests/tests.rs
View File

@ -8,7 +8,7 @@ use discv5::enr::{CombinedKey, EnrBuilder};
use environment::null_logger;
use eth2::Error;
use eth2::StatusCode;
use eth2::{types::*, BeaconNodeHttpClient, Url};
use eth2::{types::*, BeaconNodeHttpClient};
use eth2_libp2p::{
rpc::methods::MetaData,
types::{EnrBitfield, SyncState},
@ -18,6 +18,7 @@ use futures::stream::{Stream, StreamExt};
use futures::FutureExt;
use http_api::{Config, Context};
use network::NetworkMessage;
use sensitive_url::SensitiveUrl;
use slot_clock::SlotClock;
use state_processing::per_slot_processing;
use std::convert::TryInto;
@ -200,7 +201,7 @@ impl ApiTester {
tokio::spawn(async { server.await });
let client = BeaconNodeHttpClient::new(
Url::parse(&format!(
SensitiveUrl::parse(&format!(
"http://{}:{}",
listening_socket.ip(),
listening_socket.port()
@ -307,7 +308,7 @@ impl ApiTester {
tokio::spawn(async { server.await });
let client = BeaconNodeHttpClient::new(
Url::parse(&format!(
SensitiveUrl::parse(&format!(
"http://{}:{}",
listening_socket.ip(),
listening_socket.port()

15
beacon_node/src/config.rs
View File

@ -4,6 +4,7 @@ use client::{ClientConfig, ClientGenesis};
use directory::{DEFAULT_BEACON_NODE_DIR, DEFAULT_NETWORK_DIR, DEFAULT_ROOT_DIR};
use eth2_libp2p::{multiaddr::Protocol, Enr, Multiaddr, NetworkConfig, PeerIdSerialized};
use eth2_network_config::{Eth2NetworkConfig, DEFAULT_HARDCODED_NETWORK};
use sensitive_url::SensitiveUrl;
use slog::{info, warn, Logger};
use std::cmp;
use std::cmp::max;
@ -163,17 +164,21 @@ pub fn get_config<E: EthSpec>(
}
// Defines the URL to reach the eth1 node.
if let Some(val) = cli_args.value_of("eth1-endpoint") {
if let Some(endpoint) = cli_args.value_of("eth1-endpoint") {
warn!(
log,
"The --eth1-endpoint flag is deprecated";
"msg" => "please use --eth1-endpoints instead"
);
client_config.sync_eth1_chain = true;
client_config.eth1.endpoints = vec![val.to_string()];
} else if let Some(val) = cli_args.value_of("eth1-endpoints") {
client_config.sync_eth1_chain = true;
client_config.eth1.endpoints = val.split(',').map(String::from).collect();
client_config.eth1.endpoints = vec![SensitiveUrl::parse(endpoint)
.map_err(|e| format!("eth1-endpoint was an invalid URL: {:?}", e))?];
} else if let Some(endpoints) = cli_args.value_of("eth1-endpoints") {
client_config.eth1.endpoints = endpoints
.split(',')
.map(|s| SensitiveUrl::parse(s))
.collect::<Result<_, _>>()
.map_err(|e| format!("eth1-endpoints contains an invalid URL {:?}", e))?;
}
if let Some(val) = cli_args.value_of("eth1-blocks-per-log-query") {

1
common/eth2/Cargo.toml
View File

@ -21,6 +21,7 @@ libsecp256k1 = "0.3.5"
ring = "0.16.19"
bytes = "1.0.1"
account_utils = { path = "../../common/account_utils" }
sensitive_url = { path = "../../common/sensitive_url" }
eth2_ssz = "0.1.2"
eth2_ssz_derive = "0.1.0"
futures-util = "0.3.8"

13
common/eth2/src/lib.rs
View File

@ -19,6 +19,7 @@ use futures_util::StreamExt;
pub use reqwest;
use reqwest::{IntoUrl, Response};
pub use reqwest::{StatusCode, Url};
use sensitive_url::SensitiveUrl;
use serde::{de::DeserializeOwned, Serialize};
use ssz::Decode;
use std::convert::TryFrom;
@ -36,7 +37,7 @@ pub enum Error {
/// The server returned an error message where the body was unable to be parsed.
StatusCode(StatusCode),
/// The supplied URL is badly formatted. It should look something like `http://127.0.0.1:5052`.
InvalidUrl(Url),
InvalidUrl(SensitiveUrl),
/// The supplied validator client secret is invalid.
InvalidSecret(String),
/// The server returned a response with an invalid signature. It may be an impostor.
@ -81,7 +82,7 @@ impl fmt::Display for Error {
#[derive(Clone)]
pub struct BeaconNodeHttpClient {
client: reqwest::Client,
server: Url,
server: SensitiveUrl,
}
impl fmt::Display for BeaconNodeHttpClient {
@ -92,25 +93,25 @@ impl fmt::Display for BeaconNodeHttpClient {
impl AsRef<str> for BeaconNodeHttpClient {
fn as_ref(&self) -> &str {
self.server.as_str()
self.server.as_ref()
}
}
impl BeaconNodeHttpClient {
pub fn new(server: Url) -> Self {
pub fn new(server: SensitiveUrl) -> Self {
Self {
client: reqwest::Client::new(),
server,
}
}
pub fn from_components(server: Url, client: reqwest::Client) -> Self {
pub fn from_components(server: SensitiveUrl, client: reqwest::Client) -> Self {
Self { client, server }
}
/// Return the path with the standard `/eth1/v1` prefix applied.
fn eth_path(&self) -> Result<Url, Error> {
let mut path = self.server.clone();
let mut path = self.server.full.clone();
path.path_segments_mut()
.map_err(|()| Error::InvalidUrl(self.server.clone()))?

20
common/eth2/src/lighthouse.rs
View File

@ -214,7 +214,7 @@ impl BeaconNodeHttpClient {
/// `GET lighthouse/health`
pub async fn get_lighthouse_health(&self) -> Result<GenericResponse<Health>, Error> {
let mut path = self.server.clone();
let mut path = self.server.full.clone();
path.path_segments_mut()
.map_err(|()| Error::InvalidUrl(self.server.clone()))?
@ -226,7 +226,7 @@ impl BeaconNodeHttpClient {
/// `GET lighthouse/syncing`
pub async fn get_lighthouse_syncing(&self) -> Result<GenericResponse<SyncState>, Error> {
let mut path = self.server.clone();
let mut path = self.server.full.clone();
path.path_segments_mut()
.map_err(|()| Error::InvalidUrl(self.server.clone()))?
@ -246,7 +246,7 @@ impl BeaconNodeHttpClient {
/// `GET lighthouse/proto_array`
pub async fn get_lighthouse_proto_array(&self) -> Result<GenericResponse<ProtoArray>, Error> {
let mut path = self.server.clone();
let mut path = self.server.full.clone();
path.path_segments_mut()
.map_err(|()| Error::InvalidUrl(self.server.clone()))?
@ -261,7 +261,7 @@ impl BeaconNodeHttpClient {
&self,
epoch: Epoch,
) -> Result<GenericResponse<GlobalValidatorInclusionData>, Error> {
let mut path = self.server.clone();
let mut path = self.server.full.clone();
path.path_segments_mut()
.map_err(|()| Error::InvalidUrl(self.server.clone()))?
@ -279,7 +279,7 @@ impl BeaconNodeHttpClient {
epoch: Epoch,
validator_id: ValidatorId,
) -> Result<GenericResponse<Option<ValidatorInclusionData>>, Error> {
let mut path = self.server.clone();
let mut path = self.server.full.clone();
path.path_segments_mut()
.map_err(|()| Error::InvalidUrl(self.server.clone()))?
@ -295,7 +295,7 @@ impl BeaconNodeHttpClient {
pub async fn get_lighthouse_eth1_syncing(
&self,
) -> Result<GenericResponse<Eth1SyncStatusData>, Error> {
let mut path = self.server.clone();
let mut path = self.server.full.clone();
path.path_segments_mut()
.map_err(|()| Error::InvalidUrl(self.server.clone()))?
@ -310,7 +310,7 @@ impl BeaconNodeHttpClient {
pub async fn get_lighthouse_eth1_block_cache(
&self,
) -> Result<GenericResponse<Vec<Eth1Block>>, Error> {
let mut path = self.server.clone();
let mut path = self.server.full.clone();
path.path_segments_mut()
.map_err(|()| Error::InvalidUrl(self.server.clone()))?
@ -325,7 +325,7 @@ impl BeaconNodeHttpClient {
pub async fn get_lighthouse_eth1_deposit_cache(
&self,
) -> Result<GenericResponse<Vec<DepositLog>>, Error> {
let mut path = self.server.clone();
let mut path = self.server.full.clone();
path.path_segments_mut()
.map_err(|()| Error::InvalidUrl(self.server.clone()))?
@ -341,7 +341,7 @@ impl BeaconNodeHttpClient {
&self,
state_id: &StateId,
) -> Result<Option<BeaconState<E>>, Error> {
let mut path = self.server.clone();
let mut path = self.server.full.clone();
path.path_segments_mut()
.map_err(|()| Error::InvalidUrl(self.server.clone()))?
@ -359,7 +359,7 @@ impl BeaconNodeHttpClient {
/// `GET lighthouse/staking`
pub async fn get_lighthouse_staking(&self) -> Result<bool, Error> {
let mut path = self.server.clone();
let mut path = self.server.full.clone();
path.path_segments_mut()
.map_err(|()| Error::InvalidUrl(self.server.clone()))?

25
common/eth2/src/lighthouse_vc/http_client.rs
View File

@ -8,6 +8,7 @@ use reqwest::{
};
use ring::digest::{digest, SHA256};
use secp256k1::{Message, PublicKey, Signature};
use sensitive_url::SensitiveUrl;
use serde::{de::DeserializeOwned, Serialize};
pub use reqwest;
@ -18,7 +19,7 @@ pub use reqwest::{Response, StatusCode, Url};
#[derive(Clone)]
pub struct ValidatorClientHttpClient {
client: reqwest::Client,
server: Url,
server: SensitiveUrl,
secret: ZeroizeString,
server_pubkey: PublicKey,
}
@ -53,7 +54,7 @@ pub fn parse_pubkey(secret: &str) -> Result<PublicKey, Error> {
}
impl ValidatorClientHttpClient {
pub fn new(server: Url, secret: String) -> Result<Self, Error> {
pub fn new(server: SensitiveUrl, secret: String) -> Result<Self, Error> {
Ok(Self {
client: reqwest::Client::new(),
server,
@ -63,7 +64,7 @@ impl ValidatorClientHttpClient {
}
pub fn from_components(
server: Url,
server: SensitiveUrl,
client: reqwest::Client,
secret: String,
) -> Result<Self, Error> {
@ -187,7 +188,7 @@ impl ValidatorClientHttpClient {
/// `GET lighthouse/version`
pub async fn get_lighthouse_version(&self) -> Result<GenericResponse<VersionData>, Error> {
let mut path = self.server.clone();
let mut path = self.server.full.clone();
path.path_segments_mut()
.map_err(|()| Error::InvalidUrl(self.server.clone()))?
@ -199,7 +200,7 @@ impl ValidatorClientHttpClient {
/// `GET lighthouse/health`
pub async fn get_lighthouse_health(&self) -> Result<GenericResponse<Health>, Error> {
let mut path = self.server.clone();
let mut path = self.server.full.clone();
path.path_segments_mut()
.map_err(|()| Error::InvalidUrl(self.server.clone()))?
@ -211,7 +212,7 @@ impl ValidatorClientHttpClient {
/// `GET lighthouse/spec`
pub async fn get_lighthouse_spec(&self) -> Result<GenericResponse<YamlConfig>, Error> {
let mut path = self.server.clone();
let mut path = self.server.full.clone();
path.path_segments_mut()
.map_err(|()| Error::InvalidUrl(self.server.clone()))?
@ -225,7 +226,7 @@ impl ValidatorClientHttpClient {
pub async fn get_lighthouse_validators(
&self,
) -> Result<GenericResponse<Vec<ValidatorData>>, Error> {
let mut path = self.server.clone();
let mut path = self.server.full.clone();
path.path_segments_mut()
.map_err(|()| Error::InvalidUrl(self.server.clone()))?
@ -240,7 +241,7 @@ impl ValidatorClientHttpClient {
&self,
validator_pubkey: &PublicKeyBytes,
) -> Result<Option<GenericResponse<ValidatorData>>, Error> {
let mut path = self.server.clone();
let mut path = self.server.full.clone();
path.path_segments_mut()
.map_err(|()| Error::InvalidUrl(self.server.clone()))?
@ -256,7 +257,7 @@ impl ValidatorClientHttpClient {
&self,
validators: Vec<ValidatorRequest>,
) -> Result<GenericResponse<PostValidatorsResponseData>, Error> {
let mut path = self.server.clone();
let mut path = self.server.full.clone();
path.path_segments_mut()
.map_err(|()| Error::InvalidUrl(self.server.clone()))?
@ -271,7 +272,7 @@ impl ValidatorClientHttpClient {
&self,
request: &CreateValidatorsMnemonicRequest,
) -> Result<GenericResponse<Vec<CreatedValidator>>, Error> {
let mut path = self.server.clone();
let mut path = self.server.full.clone();
path.path_segments_mut()
.map_err(|()| Error::InvalidUrl(self.server.clone()))?
@ -287,7 +288,7 @@ impl ValidatorClientHttpClient {
&self,
request: &KeystoreValidatorsPostRequest,
) -> Result<GenericResponse<ValidatorData>, Error> {
let mut path = self.server.clone();
let mut path = self.server.full.clone();
path.path_segments_mut()
.map_err(|()| Error::InvalidUrl(self.server.clone()))?
@ -304,7 +305,7 @@ impl ValidatorClientHttpClient {
voting_pubkey: &PublicKeyBytes,
enabled: bool,
) -> Result<(), Error> {
let mut path = self.server.clone();
let mut path = self.server.full.clone();
path.path_segments_mut()
.map_err(|()| Error::InvalidUrl(self.server.clone()))?

1
common/remote_signer_consumer/Cargo.toml
View File

@ -13,3 +13,4 @@ reqwest = { version = "0.11.0", features = ["json"] }
serde = { version = "1.0.116", features = ["derive"] }
tokio = { version = "1.1.0", features = ["time"] }
types = { path = "../../consensus/types" }
sensitive_url = { path = "../sensitive_url" }

7
common/remote_signer_consumer/src/http_client.rs
View File

@ -4,17 +4,18 @@ use crate::{
};
use reqwest::StatusCode;
pub use reqwest::Url;
use sensitive_url::SensitiveUrl;
use types::{Domain, Fork, Hash256};
/// A wrapper around `reqwest::Client` which provides convenience methods
/// to interface with a BLS Remote Signer.
pub struct RemoteSignerHttpConsumer {
client: reqwest::Client,
server: Url,
server: SensitiveUrl,
}
impl RemoteSignerHttpConsumer {
pub fn from_components(server: Url, client: reqwest::Client) -> Self {
pub fn from_components(server: SensitiveUrl, client: reqwest::Client) -> Self {
Self { client, server }
}
@ -43,7 +44,7 @@ impl RemoteSignerHttpConsumer {
));
}
let mut path = self.server.clone();
let mut path = self.server.full.clone();
path.path_segments_mut()
.map_err(|()| Error::InvalidUrl(self.server.clone()))?
.push("sign")

8
common/remote_signer_consumer/src/lib.rs
View File

@ -20,10 +20,11 @@
//!
//! ```
//! use remote_signer_consumer::RemoteSignerHttpConsumer;
//! use reqwest::{ClientBuilder, Url};
//! use reqwest::ClientBuilder;
//! use sensitive_url::SensitiveUrl;
//! use tokio::time::Duration;
//!
//! let url: Url = "http://127.0.0.1:9000".parse().unwrap();
//! let url = SensitiveUrl::parse("http://127.0.0.1:9000").unwrap();
//! let reqwest_client = ClientBuilder::new()
//! .timeout(Duration::from_secs(2))
//! .build()
@ -115,6 +116,7 @@ mod http_client;
pub use http_client::RemoteSignerHttpConsumer;
pub use reqwest::Url;
use sensitive_url::SensitiveUrl;
use serde::{Deserialize, Serialize};
use types::{AttestationData, BeaconBlock, Domain, Epoch, EthSpec, Fork, Hash256, SignedRoot};
@ -125,7 +127,7 @@ pub enum Error {
/// The server returned an error message where the body was able to be parsed.
ServerMessage(String),
/// The supplied URL is badly formatted. It should look something like `http://127.0.0.1:5052`.
InvalidUrl(Url),
InvalidUrl(SensitiveUrl),
/// The supplied parameter is invalid.
InvalidParameter(String),
}

38
common/remote_signer_consumer/tests/post.rs
View File

@ -1,7 +1,8 @@
mod post {
use remote_signer_consumer::{Error, RemoteSignerHttpConsumer};
use remote_signer_test::*;
use reqwest::{ClientBuilder, Url};
use reqwest::ClientBuilder;
use sensitive_url::SensitiveUrl;
use tokio::time::Duration;
#[test]
@ -53,7 +54,7 @@ mod post {
let (test_signer, _tmp_dir) = set_up_api_test_signer_to_sign_message();
let run_testcase = |u: &str| -> Result<String, String> {
let url: Url = u.parse().map_err(|e| format!("[ParseError] {:?}", e))?;
let url = SensitiveUrl::parse(u).map_err(|e| format!("{:?}", e))?;
let reqwest_client = ClientBuilder::new()
.timeout(Duration::from_secs(12))
@ -66,7 +67,7 @@ mod post {
let signature = do_sign_request(&test_client, test_input);
signature.map_err(|e| match e {
Error::InvalidUrl(message) => format!("[InvalidUrl] {:?}", message),
Error::InvalidUrl(message) => format!("{:?}", message),
Error::Reqwest(re) => {
if re.is_builder() {
format!("[Reqwest - Builder] {:?}", re.url().unwrap())
@ -84,25 +85,22 @@ mod post {
// url::parser::ParseError.
// These cases don't even make it to the step of building a RemoteSignerHttpConsumer.
testcase("", "[ParseError] RelativeUrlWithoutBase");
testcase("/4/8/15/16/23/42", "[ParseError] RelativeUrlWithoutBase");
testcase("localhost", "[ParseError] RelativeUrlWithoutBase");
testcase(":", "[ParseError] RelativeUrlWithoutBase");
testcase("0.0:0", "[ParseError] RelativeUrlWithoutBase");
testcase(":aa", "[ParseError] RelativeUrlWithoutBase");
testcase("0:", "[ParseError] RelativeUrlWithoutBase");
testcase("ftp://", "[ParseError] EmptyHost");
testcase("http://", "[ParseError] EmptyHost");
testcase("http://127.0.0.1:abcd", "[ParseError] InvalidPort");
testcase("http://280.0.0.1", "[ParseError] InvalidIpv4Address");
testcase("", "ParseError(RelativeUrlWithoutBase)");
testcase("/4/8/15/16/23/42", "ParseError(RelativeUrlWithoutBase)");
testcase("localhost", "ParseError(RelativeUrlWithoutBase)");
testcase(":", "ParseError(RelativeUrlWithoutBase)");
testcase("0.0:0", "ParseError(RelativeUrlWithoutBase)");
testcase(":aa", "ParseError(RelativeUrlWithoutBase)");
testcase("0:", "ParseError(RelativeUrlWithoutBase)");
testcase("ftp://", "ParseError(EmptyHost)");
testcase("http://", "ParseError(EmptyHost)");
testcase("http://127.0.0.1:abcd", "ParseError(InvalidPort)");
testcase("http://280.0.0.1", "ParseError(InvalidIpv4Address)");
// `Error::InvalidUrl`.
// The RemoteSignerHttpConsumer is created, but fails at `path_segments_mut()`.
testcase(
"localhost:abcd",
"[InvalidUrl] Url { scheme: \"localhost\", username: \"\", password: None, host: None, port: None, path: \"abcd\", query: None, fragment: None }",
);
testcase("localhost:", "[InvalidUrl] Url { scheme: \"localhost\", username: \"\", password: None, host: None, port: None, path: \"\", query: None, fragment: None }");
testcase("localhost:abcd", "InvalidUrl(\"URL cannot be a base.\")");
testcase("localhost:", "InvalidUrl(\"URL cannot be a base.\")");
// `Reqwest::Error` of the `Builder` kind.
// POST is not made.
@ -130,7 +128,7 @@ mod post {
let (test_signer, _tmp_dir) = set_up_api_test_signer_to_sign_message();
let run_testcase = |u: &str| -> Result<String, String> {
let url: Url = u.parse().unwrap();
let url = SensitiveUrl::parse(u).unwrap();
let reqwest_client = ClientBuilder::new()
.timeout(Duration::from_secs(12))

11
common/sensitive_url/Cargo.toml Normal file
View File

@ -0,0 +1,11 @@
[package]
name = "sensitive_url"
version = "0.1.0"
authors = ["Mac L <mjladson@pm.me>"]
edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
url = "2.2.1"
serde = "1.0.116"

105
common/sensitive_url/src/lib.rs Normal file
View File

@ -0,0 +1,105 @@
use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
use std::fmt;
use url::Url;
#[derive(Debug)]
pub enum SensitiveError {
InvalidUrl(String),
ParseError(url::ParseError),
RedactError(String),
}
// Wrapper around Url which provides a custom `Display` implementation to protect user secrets.
#[derive(Clone)]
pub struct SensitiveUrl {
pub full: Url,
pub redacted: String,
}
impl fmt::Display for SensitiveUrl {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.redacted.fmt(f)
}
}
impl fmt::Debug for SensitiveUrl {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.redacted.fmt(f)
}
}
impl AsRef<str> for SensitiveUrl {
fn as_ref(&self) -> &str {
self.redacted.as_str()
}
}
impl Serialize for SensitiveUrl {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(&self.full.to_string())
}
}
impl<'de> Deserialize<'de> for SensitiveUrl {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let s: String = Deserialize::deserialize(deserializer)?;
SensitiveUrl::parse(&s)
.map_err(|e| de::Error::custom(format!("Failed to deserialize sensitive URL {:?}", e)))
}
}
impl SensitiveUrl {
pub fn parse(url: &str) -> Result<Self, SensitiveError> {
let surl = Url::parse(url).map_err(SensitiveError::ParseError)?;
SensitiveUrl::new(surl)
}
fn new(full: Url) -> Result<Self, SensitiveError> {
let mut redacted = full.clone();
redacted
.path_segments_mut()
.map_err(|_| SensitiveError::InvalidUrl("URL cannot be a base.".to_string()))?
.clear();
redacted.set_query(None);
if redacted.has_authority() {
redacted.set_username("").map_err(|_| {
SensitiveError::RedactError("Unable to redact username.".to_string())
})?;
redacted.set_password(None).map_err(|_| {
SensitiveError::RedactError("Unable to redact password.".to_string())
})?;
}
Ok(Self {
full,
redacted: redacted.to_string(),
})
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn redact_remote_url() {
let full = "https://project:secret@example.com/example?somequery";
let surl = SensitiveUrl::parse(full).unwrap();
assert_eq!(surl.to_string(), "https://example.com/");
assert_eq!(surl.full.to_string(), full);
}
#[test]
fn redact_localhost_url() {
let full = "http://localhost:5052/";
let surl = SensitiveUrl::parse(full).unwrap();
assert_eq!(surl.to_string(), "http://localhost:5052/");
assert_eq!(surl.full.to_string(), full);
}
}

1
lcli/Cargo.toml
View File

@ -39,3 +39,4 @@ account_utils = { path = "../common/account_utils" }
eth2_wallet = { path = "../crypto/eth2_wallet" }
web3 = "0.14.0"
eth1_test_rig = { path = "../testing/eth1_test_rig" }
sensitive_url = { path = "../common/sensitive_url" }

7
lcli/src/eth1_genesis.rs
View File

@ -2,6 +2,7 @@ use clap::ArgMatches;
use environment::Environment;
use eth2_network_config::Eth2NetworkConfig;
use genesis::{Eth1Config, Eth1GenesisService};
use sensitive_url::SensitiveUrl;
use ssz::Encode;
use std::cmp::max;
use std::path::PathBuf;
@ -50,7 +51,11 @@ pub fn run<T: EthSpec>(mut env: Environment<T>, matches: &ArgMatches<'_>) -> Res
let mut config = Eth1Config::default();
if let Some(v) = endpoints.clone() {
config.endpoints = v;
config.endpoints = v
.iter()
.map(|s| SensitiveUrl::parse(s))
.collect::<Result<_, _>>()
.map_err(|e| format!("Unable to parse eth1 endpoint URL: {:?}", e))?;
}
config.deposit_contract_address = format!("{:?}", spec.deposit_contract_address);
config.deposit_contract_deploy_block = eth2_network_config.deposit_contract_deploy_block;

1
testing/node_test_rig/Cargo.toml
View File

@ -18,3 +18,4 @@ genesis = { path = "../../beacon_node/genesis" }
eth2 = { path = "../../common/eth2" }
validator_client = { path = "../../validator_client" }
validator_dir = { path = "../../common/validator_dir", features = ["insecure_keys"] }
sensitive_url = { path = "../../common/sensitive_url" }

11
testing/node_test_rig/src/lib.rs
View File

@ -4,10 +4,8 @@
use beacon_node::ProductionBeaconNode;
use environment::RuntimeContext;
use eth2::{
reqwest::{ClientBuilder, Url},
BeaconNodeHttpClient,
};
use eth2::{reqwest::ClientBuilder, BeaconNodeHttpClient};
use sensitive_url::SensitiveUrl;
use std::path::PathBuf;
use std::time::Duration;
use std::time::{SystemTime, UNIX_EPOCH};
@ -68,8 +66,9 @@ impl<E: EthSpec> LocalBeaconNode<E> {
.http_api_listen_addr()
.ok_or("A remote beacon node must have a http server")?;
let beacon_node_url: Url = format!("http://{}:{}", listen_addr.ip(), listen_addr.port())
.parse()
let beacon_node_url: SensitiveUrl = SensitiveUrl::parse(
format!("http://{}:{}", listen_addr.ip(), listen_addr.port()).as_str(),
)
.map_err(|e| format!("Unable to parse beacon node URL: {:?}", e))?;
let beacon_node_http_client = ClientBuilder::new()
.timeout(HTTP_TIMEOUT)

1
testing/remote_signer_test/Cargo.toml
View File

@ -17,3 +17,4 @@ serde_json = "1.0.58"
tempfile = "3.1.0"
tokio = { version = "1.1.0", features = ["time"] }
types = { path = "../../consensus/types" }
sensitive_url = { path = "../../common/sensitive_url" }

5
testing/remote_signer_test/src/consumer.rs
View File

@ -1,7 +1,8 @@
use crate::*;
use remote_signer_client::api_response::SignatureApiResponse;
use remote_signer_consumer::{Error, RemoteSignerHttpConsumer, RemoteSignerObject, Url};
use remote_signer_consumer::{Error, RemoteSignerHttpConsumer, RemoteSignerObject};
use reqwest::ClientBuilder;
use sensitive_url::SensitiveUrl;
use serde::Serialize;
use tokio::runtime::Builder;
use tokio::time::Duration;
@ -15,7 +16,7 @@ pub fn set_up_test_consumer_with_timeout(
test_signer_address: &str,
timeout: u64,
) -> RemoteSignerHttpConsumer {
let url: Url = test_signer_address.parse().unwrap();
let url = SensitiveUrl::parse(test_signer_address).unwrap();
let reqwest_client = ClientBuilder::new()
.timeout(Duration::from_secs(timeout))
.build()

1
testing/simulator/Cargo.toml
View File

@ -18,3 +18,4 @@ eth1_test_rig = { path = "../eth1_test_rig" }
env_logger = "0.8.2"
clap = "2.33.3"
rayon = "1.4.1"
sensitive_url = { path = "../../common/sensitive_url" }

9
testing/simulator/src/eth1_sim.rs
View File

@ -10,6 +10,7 @@ use node_test_rig::{
ClientGenesis, ValidatorFiles,
};
use rayon::prelude::*;
use sensitive_url::SensitiveUrl;
use std::cmp::max;
use std::net::{IpAddr, Ipv4Addr};
use std::time::Duration;
@ -84,7 +85,8 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> {
let network_id = ganache_eth1_instance.ganache.network_id();
let chain_id = ganache_eth1_instance.ganache.chain_id();
let ganache = ganache_eth1_instance.ganache;
let eth1_endpoint = ganache.endpoint();
let eth1_endpoint = SensitiveUrl::parse(ganache.endpoint().as_str())
.expect("Unable to parse ganache endpoint.");
let deposit_contract_address = deposit_contract.address();
// Start a timer that produces eth1 blocks on an interval.
@ -133,7 +135,10 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> {
for i in 0..node_count - 1 {
let mut config = beacon_config.clone();
if i % 2 == 0 {
config.eth1.endpoints.insert(0, INVALID_ADDRESS.to_string());
config.eth1.endpoints.insert(
0,
SensitiveUrl::parse(INVALID_ADDRESS).expect("Unable to parse invalid address"),
);
}
network.add_beacon_node(config).await?;
}

8
testing/simulator/src/local_network.rs
View File

@ -4,6 +4,7 @@ use node_test_rig::{
ClientConfig, LocalBeaconNode, LocalValidatorClient, ValidatorConfig, ValidatorFiles,
};
use parking_lot::RwLock;
use sensitive_url::SensitiveUrl;
use std::{
ops::Deref,
time::{SystemTime, UNIX_EPOCH},
@ -140,9 +141,12 @@ impl<E: EthSpec> LocalNetwork<E> {
.expect("Must have http started")
};
let beacon_node = format!("http://{}:{}", socket_addr.ip(), socket_addr.port());
let beacon_node = SensitiveUrl::parse(
format!("http://{}:{}", socket_addr.ip(), socket_addr.port()).as_str(),
)
.unwrap();
validator_config.beacon_nodes = if invalid_first_beacon_node {
vec![INVALID_ADDRESS.to_string(), beacon_node]
vec![SensitiveUrl::parse(INVALID_ADDRESS).unwrap(), beacon_node]
} else {
vec![beacon_node]
};

1
validator_client/Cargo.toml
View File

@ -63,3 +63,4 @@ scrypt = { version = "0.5.0", default-features = false }
lighthouse_metrics = { path = "../common/lighthouse_metrics" }
lazy_static = "1.4.0"
fallback = { path = "../common/fallback" }
sensitive_url = { path = "../common/sensitive_url" }

35
validator_client/src/config.rs
View File

@ -7,6 +7,7 @@ use directory::{
DEFAULT_VALIDATOR_DIR,
};
use eth2::types::Graffiti;
use sensitive_url::SensitiveUrl;
use serde_derive::{Deserialize, Serialize};
use slog::{info, warn, Logger};
use std::fs;
@ -26,7 +27,7 @@ pub struct Config {
/// The http endpoints of the beacon node APIs.
///
/// Should be similar to `["http://localhost:8080"]`
pub beacon_nodes: Vec<String>,
pub beacon_nodes: Vec<SensitiveUrl>,
/// If true, the validator client will still poll for duties and produce blocks even if the
/// beacon node is not synced at startup.
pub allow_unsynced_beacon_node: bool,
@ -55,10 +56,13 @@ impl Default for Config {
.join(DEFAULT_HARDCODED_NETWORK);
let validator_dir = base_dir.join(DEFAULT_VALIDATOR_DIR);
let secrets_dir = base_dir.join(DEFAULT_SECRET_DIR);
let beacon_nodes = vec![SensitiveUrl::parse(DEFAULT_BEACON_NODE)
.expect("beacon_nodes must always be a valid url.")];
Self {
validator_dir,
secrets_dir,
beacon_nodes: vec![DEFAULT_BEACON_NODE.to_string()],
beacon_nodes,
allow_unsynced_beacon_node: false,
disable_auto_discover: false,
init_slashing_protection: false,
@ -111,25 +115,31 @@ impl Config {
}
if let Some(beacon_nodes) = parse_optional::<String>(cli_args, "beacon-nodes")? {
config.beacon_nodes = beacon_nodes.as_str().split(',').map(String::from).collect()
config.beacon_nodes = beacon_nodes
.split(',')
.map(|s| SensitiveUrl::parse(s))
.collect::<Result<_, _>>()
.map_err(|e| format!("Unable to parse beacon node URL: {:?}", e))?;
}
// To be deprecated.
else if let Some(beacon_node) = parse_optional(cli_args, "beacon-node")? {
else if let Some(beacon_node) = parse_optional::<String>(cli_args, "beacon-node")? {
warn!(
log,
"The --beacon-node flag is deprecated";
"msg" => "please use --beacon-nodes instead"
);
config.beacon_nodes = vec![beacon_node];
config.beacon_nodes = vec![SensitiveUrl::parse(&beacon_node)
.map_err(|e| format!("Unable to parse beacon node URL: {:?}", e))?];
}
// To be deprecated.
else if let Some(server) = parse_optional(cli_args, "server")? {
else if let Some(server) = parse_optional::<String>(cli_args, "server")? {
warn!(
log,
"The --server flag is deprecated";
"msg" => "please use --beacon-nodes instead"
);
config.beacon_nodes = vec![server];
config.beacon_nodes = vec![SensitiveUrl::parse(&server)
.map_err(|e| format!("Unable to parse beacon node URL: {:?}", e))?];
}
if cli_args.is_present("delete-lockfiles") {
@ -227,3 +237,14 @@ impl Config {
Ok(config)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
// Ensures the default config does not panic.
fn default_config() {
Config::default();
}
}

2
validator_client/src/fork_service.rs
View File

@ -84,7 +84,7 @@ impl<E: EthSpec> ForkServiceBuilder<slot_clock::TestingSlotClock, E> {
std::time::Duration::from_secs(42),
);
let candidates = vec![CandidateBeaconNode::new(eth2::BeaconNodeHttpClient::new(
eth2::Url::parse("http://127.0.0.1").unwrap(),
sensitive_url::SensitiveUrl::parse("http://127.0.0.1").unwrap(),
))];
let mut beacon_nodes = BeaconNodeFallback::new(candidates, spec, log.clone());
beacon_nodes.set_slot_clock(slot_clock);

10
validator_client/src/http_api/tests.rs
View File

@ -11,12 +11,10 @@ use account_utils::{
};
use deposit_contract::decode_eth1_tx_data;
use environment::null_logger;
use eth2::{
lighthouse_vc::{http_client::ValidatorClientHttpClient, types::*},
Url,
};
use eth2::lighthouse_vc::{http_client::ValidatorClientHttpClient, types::*};
use eth2_keystore::KeystoreBuilder;
use parking_lot::RwLock;
use sensitive_url::SensitiveUrl;
use slashing_protection::{SlashingDatabase, SLASHING_PROTECTION_FILENAME};
use slot_clock::TestingSlotClock;
use std::marker::PhantomData;
@ -33,7 +31,7 @@ type E = MainnetEthSpec;
struct ApiTester {
client: ValidatorClientHttpClient,
initialized_validators: Arc<RwLock<InitializedValidators>>,
url: Url,
url: SensitiveUrl,
_server_shutdown: oneshot::Sender<()>,
_validator_dir: TempDir,
}
@ -117,7 +115,7 @@ impl ApiTester {
tokio::spawn(async { server.await });
let url = Url::parse(&format!(
let url = SensitiveUrl::parse(&format!(
"http://{}:{}",
listening_socket.ip(),
listening_socket.port()

10
validator_client/src/lib.rs
View File

@ -28,7 +28,7 @@ use clap::ArgMatches;
use duties_service::DutiesService;
use environment::RuntimeContext;
use eth2::types::StateId;
use eth2::{reqwest::ClientBuilder, BeaconNodeHttpClient, StatusCode, Url};
use eth2::{reqwest::ClientBuilder, BeaconNodeHttpClient, StatusCode};
use fork_service::{ForkService, ForkServiceBuilder};
use http_api::ApiSecret;
use initialized_validators::InitializedValidators;
@ -209,13 +209,9 @@ impl<T: EthSpec> ProductionValidatorClient<T> {
})?;
}
let beacon_node_urls: Vec<Url> = config
let beacon_nodes: Vec<BeaconNodeHttpClient> = config
.beacon_nodes
.iter()
.map(|s| s.parse())
.collect::<Result<_, _>>()
.map_err(|e| format!("Unable to parse beacon node URL: {:?}", e))?;
let beacon_nodes: Vec<BeaconNodeHttpClient> = beacon_node_urls
.clone()
.into_iter()
.map(|url| {
let beacon_node_http_client = ClientBuilder::new()