Add /node/health endpoint (#1197)

* Start adding health endpoint

* Use psutil more

* Add get_health test

* Expose health to Prom

* Update comments

* Add /node/health to docs

* Update Prom naming
This commit is contained in:
Paul Hauner 2020-05-28 11:24:24 +10:00 committed by GitHub
parent 58a9f979e0
commit ea4a52984c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 181 additions and 4 deletions

View File

@ -3,6 +3,7 @@ use crate::{ApiError, ApiResult};
use beacon_chain::{BeaconChain, BeaconChainTypes};
use hyper::{Body, Request};
use lighthouse_metrics::{Encoder, TextEncoder};
use rest_types::Health;
use std::path::PathBuf;
use std::sync::Arc;
@ -36,6 +37,38 @@ lazy_static! {
"http_server_validator_duties_get_request_duration_seconds",
"Time taken to respond to GET /validator/duties"
);
pub static ref PROCESS_NUM_THREADS: Result<IntGauge> = try_create_int_gauge(
"process_num_threads",
"Number of threads used by the current process"
);
pub static ref PROCESS_RES_MEM: Result<IntGauge> = try_create_int_gauge(
"process_resident_memory_bytes",
"Resident memory used by the current process"
);
pub static ref PROCESS_VIRT_MEM: Result<IntGauge> = try_create_int_gauge(
"process_virtual_memory_bytes",
"Virtual memory used by the current process"
);
pub static ref SYSTEM_VIRT_MEM_TOTAL: Result<IntGauge> =
try_create_int_gauge("system_virt_mem_total_bytes", "Total system virtual memory");
pub static ref SYSTEM_VIRT_MEM_AVAILABLE: Result<IntGauge> = try_create_int_gauge(
"system_virt_mem_available_bytes",
"Available system virtual memory"
);
pub static ref SYSTEM_VIRT_MEM_USED: Result<IntGauge> =
try_create_int_gauge("system_virt_mem_used_bytes", "Used system virtual memory");
pub static ref SYSTEM_VIRT_MEM_FREE: Result<IntGauge> =
try_create_int_gauge("system_virt_mem_free_bytes", "Free system virtual memory");
pub static ref SYSTEM_VIRT_MEM_PERCENTAGE: Result<Gauge> = try_create_float_gauge(
"system_virt_mem_percentage",
"Percentage of used virtual memory"
);
pub static ref SYSTEM_LOADAVG_1: Result<Gauge> =
try_create_float_gauge("system_loadavg_1", "Loadavg over 1 minute");
pub static ref SYSTEM_LOADAVG_5: Result<Gauge> =
try_create_float_gauge("system_loadavg_5", "Loadavg over 5 minutes");
pub static ref SYSTEM_LOADAVG_15: Result<Gauge> =
try_create_float_gauge("system_loadavg_15", "Loadavg over 15 minutes");
}
/// Returns the full set of Prometheus metrics for the Beacon Node application.
@ -72,6 +105,26 @@ pub fn get_prometheus<T: BeaconChainTypes>(
store::scrape_for_metrics(&db_path, &freezer_db_path);
beacon_chain::scrape_for_metrics(&beacon_chain);
if let Ok(health) = Health::observe() {
set_gauge(&PROCESS_NUM_THREADS, health.pid_num_threads as i64);
set_gauge(&PROCESS_RES_MEM, health.pid_mem_resident_set_size as i64);
set_gauge(&PROCESS_VIRT_MEM, health.pid_mem_virtual_memory_size as i64);
set_gauge(&SYSTEM_VIRT_MEM_TOTAL, health.sys_virt_mem_total as i64);
set_gauge(
&SYSTEM_VIRT_MEM_AVAILABLE,
health.sys_virt_mem_available as i64,
);
set_gauge(&SYSTEM_VIRT_MEM_USED, health.sys_virt_mem_used as i64);
set_gauge(&SYSTEM_VIRT_MEM_FREE, health.sys_virt_mem_free as i64);
set_float_gauge(
&SYSTEM_VIRT_MEM_PERCENTAGE,
health.sys_virt_mem_percent as f64,
);
set_float_gauge(&SYSTEM_LOADAVG_1, health.sys_loadavg_1);
set_float_gauge(&SYSTEM_LOADAVG_5, health.sys_loadavg_5);
set_float_gauge(&SYSTEM_LOADAVG_15, health.sys_loadavg_15);
}
encoder
.encode(&lighthouse_metrics::gather(), &mut buffer)
.unwrap();

View File

@ -1,8 +1,8 @@
use crate::response_builder::ResponseBuilder;
use crate::ApiResult;
use crate::{ApiError, ApiResult};
use eth2_libp2p::{types::SyncState, NetworkGlobals};
use hyper::{Body, Request};
use rest_types::{SyncingResponse, SyncingStatus};
use rest_types::{Health, SyncingResponse, SyncingStatus};
use std::sync::Arc;
use types::{EthSpec, Slot};
use version;
@ -41,3 +41,9 @@ pub fn syncing<T: EthSpec>(
sync_status,
})
}
pub fn get_health(req: Request<Body>) -> ApiResult {
let health = Health::observe().map_err(|e| ApiError::ServerError(e))?;
ResponseBuilder::new(&req)?.body_no_ssz(&health)
}

View File

@ -33,6 +33,7 @@ pub async fn route<T: BeaconChainTypes>(
let log = local_log.clone();
let request_result = match (req.method(), path.as_ref()) {
// Methods for Client
(&Method::GET, "/node/health") => node::get_health(req),
(&Method::GET, "/node/version") => node::get_version(req),
(&Method::GET, "/node/syncing") => {
// inform the current slot, or set to 0

View File

@ -1252,3 +1252,15 @@ mod validator_attestation {
);
}
}
#[test]
fn get_health() {
let mut env = build_env();
let node = build_node(&mut env, testing_client_config());
let remote_node = node.remote_node().expect("should produce remote node");
env.runtime()
.block_on(remote_node.http.node().get_health())
.unwrap();
}

View File

@ -55,3 +55,36 @@ Typical Responses | 200
}
}
```
## `/node/health`
Requests information about the health of the beacon node.
### HTTP Specification
| Property | Specification |
| --- |--- |
Path | `/node/health`
Method | GET
JSON Encoding | Object
Query Parameters | None
Typical Responses | 200
### Example Response
```json
{
"pid": 96160,
"pid_num_threads": 30,
"pid_mem_resident_set_size": 55476224,
"pid_mem_virtual_memory_size": 2081382400,
"sys_virt_mem_total": 16721076224,
"sys_virt_mem_available": 7423197184,
"sys_virt_mem_used": 8450183168,
"sys_virt_mem_free": 3496345600,
"sys_virt_mem_percent": 55.605743,
"sys_loadavg_1": 1.56,
"sys_loadavg_5": 2.61,
"sys_loadavg_15": 2.43
}
```

View File

@ -19,7 +19,7 @@ use url::Url;
pub use operation_pool::PersistedOperationPool;
pub use proto_array_fork_choice::core::ProtoArray;
pub use rest_types::{
CanonicalHeadResponse, Committee, HeadBeaconBlock, IndividualVotesRequest,
CanonicalHeadResponse, Committee, HeadBeaconBlock, Health, IndividualVotesRequest,
IndividualVotesResponse, SyncingResponse, ValidatorDutiesRequest, ValidatorDutyBytes,
ValidatorRequest, ValidatorResponse, ValidatorSubscription,
};
@ -612,6 +612,12 @@ impl<E: EthSpec> Node<E> {
client.json_get(url, vec![]).await
}
pub async fn get_health(&self) -> Result<Health, Error> {
let client = self.0.clone();
let url = self.url("health")?;
client.json_get(url, vec![]).await
}
pub async fn syncing_status(&self) -> Result<SyncingResponse, Error> {
let client = self.0.clone();
let url = self.url("syncing")?;

View File

@ -14,3 +14,5 @@ state_processing = { path = "../../consensus/state_processing" }
bls = { path = "../../crypto/bls" }
serde = { version = "1.0.110", features = ["derive"] }
rayon = "1.3.0"
psutil = "3.1.0"
procinfo = "0.4.2"

View File

@ -18,4 +18,4 @@ pub use validator::{
pub use consensus::{IndividualVote, IndividualVotesRequest, IndividualVotesResponse};
pub use node::{SyncingResponse, SyncingStatus};
pub use node::{Health, SyncingResponse, SyncingStatus};

View File

@ -1,4 +1,6 @@
//! Collection of types for the /node HTTP
use procinfo::pid;
use psutil::process::Process;
use serde::{Deserialize, Serialize};
use ssz_derive::{Decode, Encode};
use types::Slot;
@ -30,3 +32,65 @@ pub struct SyncingResponse {
/// The current sync status.
pub sync_status: SyncingStatus,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
/// Reports on the health of the Lighthouse instance.
pub struct Health {
/// The pid of this process.
pub pid: u32,
/// The number of threads used by this pid.
pub pid_num_threads: i32,
/// The total resident memory used by this pid.
pub pid_mem_resident_set_size: u64,
/// The total virtual memory used by this pid.
pub pid_mem_virtual_memory_size: u64,
/// Total virtual memory on the system
pub sys_virt_mem_total: u64,
/// Total virtual memory available for new processes.
pub sys_virt_mem_available: u64,
/// Total virtual memory used on the system
pub sys_virt_mem_used: u64,
/// Total virtual memory not used on the system
pub sys_virt_mem_free: u64,
/// Percentage of virtual memory used on the system
pub sys_virt_mem_percent: f32,
/// System load average over 1 minute.
pub sys_loadavg_1: f64,
/// System load average over 5 minutes.
pub sys_loadavg_5: f64,
/// System load average over 15 minutes.
pub sys_loadavg_15: f64,
}
impl Health {
pub fn observe() -> Result<Self, String> {
let process =
Process::current().map_err(|e| format!("Unable to get current process: {:?}", e))?;
let process_mem = process
.memory_info()
.map_err(|e| format!("Unable to get process memory info: {:?}", e))?;
let stat = pid::stat_self().map_err(|e| format!("Unable to get stat: {:?}", e))?;
let vm = psutil::memory::virtual_memory()
.map_err(|e| format!("Unable to get virtual memory: {:?}", e))?;
let loadavg =
psutil::host::loadavg().map_err(|e| format!("Unable to get loadavg: {:?}", e))?;
Ok(Self {
pid: process.pid().into(),
pid_num_threads: stat.num_threads,
pid_mem_resident_set_size: process_mem.rss().into(),
pid_mem_virtual_memory_size: process_mem.vms().into(),
sys_virt_mem_total: vm.total().into(),
sys_virt_mem_available: vm.available().into(),
sys_virt_mem_used: vm.used().into(),
sys_virt_mem_free: vm.free().into(),
sys_virt_mem_percent: vm.percent().into(),
sys_loadavg_1: loadavg.one.into(),
sys_loadavg_5: loadavg.five.into(),
sys_loadavg_15: loadavg.fifteen.into(),
})
}
}