Expose certain validator_monitor metrics to the HTTP API (#3760)

## Issue Addressed

#3724 

## Proposed Changes

Exposes certain `validator_monitor` as an endpoint on the HTTP API. Will only return metrics for validators which are actively being monitored.

### Usage

```bash
curl -X GET "http://localhost:5052/lighthouse/ui/validator_metrics" -H "accept: application/json" | jq
```

```json
{
  "data": {
    "validators": {
      "12345": {
        "attestation_hits": 10,
        "attestation_misses": 0,
        "attestation_hit_percentage": 100,
        "attestation_head_hits": 10,
        "attestation_head_misses": 0,
        "attestation_head_hit_percentage": 100,
        "attestation_target_hits": 5,
        "attestation_target_misses": 5,
        "attestation_target_hit_percentage": 50 
      }
    }
  }
}
```

## Additional Info

Based on #3756 which should be merged first.

beacon_node/beacon_chain/src/lib.rs

@@ -22,7 +22,7 @@ pub mod fork_revert;
 mod head_tracker;
 pub mod historical_blocks;
 pub mod merge_readiness;
-mod metrics;
+pub mod metrics;
 pub mod migrate;
 mod naive_aggregation_pool;
 mod observed_aggregates;

beacon_node/beacon_chain/src/validator_monitor.rs

@@ -629,6 +629,14 @@ impl<T: EthSpec> ValidatorMonitor<T> {
         self.validators.len()
     }
 
+    // Return the `id`'s of all monitored validators.
+    pub fn get_all_monitored_validators(&self) -> Vec<String> {
+        self.validators
+            .iter()
+            .map(|(_, val)| val.id.clone())
+            .collect()
+    }
+
     /// If `self.auto_register == true`, add the `validator_index` to `self.monitored_validators`.
     /// Otherwise, do nothing.
     pub fn auto_register_local_validator(&mut self, validator_index: u64) {

beacon_node/http_api/src/lib.rs

@@ -2899,6 +2899,22 @@ pub fn serve<T: BeaconChainTypes>(
             })
         });
 
+    // POST lighthouse/ui/validator_metrics
+    let post_lighthouse_ui_validator_metrics = warp::path("lighthouse")
+        .and(warp::path("ui"))
+        .and(warp::path("validator_metrics"))
+        .and(warp::path::end())
+        .and(warp::body::json())
+        .and(chain_filter.clone())
+        .and_then(
+            |request_data: ui::ValidatorMetricsRequestData, chain: Arc<BeaconChain<T>>| {
+                blocking_json_task(move || {
+                    ui::post_validator_monitor_metrics(request_data, chain)
+                        .map(api_types::GenericResponse::from)
+                })
+            },
+        );
+
     // GET lighthouse/syncing
     let get_lighthouse_syncing = warp::path("lighthouse")
         .and(warp::path("syncing"))
@@ -3349,6 +3365,7 @@ pub fn serve<T: BeaconChainTypes>(
                 .or(get_validator_sync_committee_contribution.boxed())
                 .or(get_lighthouse_health.boxed())
                 .or(get_lighthouse_ui_health.boxed())
+                .or(get_lighthouse_ui_validator_count.boxed())
                 .or(get_lighthouse_syncing.boxed())
                 .or(get_lighthouse_nat.boxed())
                 .or(get_lighthouse_peers.boxed())
@@ -3366,7 +3383,6 @@ pub fn serve<T: BeaconChainTypes>(
                 .or(get_lighthouse_attestation_performance.boxed())
                 .or(get_lighthouse_block_packing_efficiency.boxed())
                 .or(get_lighthouse_merge_readiness.boxed())
-                .or(get_lighthouse_ui_validator_count.boxed())
                 .or(get_events.boxed()),
         )
         .boxed()
@@ -3390,7 +3406,8 @@ pub fn serve<T: BeaconChainTypes>(
                 .or(post_lighthouse_liveness.boxed())
                 .or(post_lighthouse_database_reconstruct.boxed())
                 .or(post_lighthouse_database_historical_blocks.boxed())
-                .or(post_lighthouse_block_rewards.boxed()),
+                .or(post_lighthouse_block_rewards.boxed())
+                .or(post_lighthouse_ui_validator_metrics.boxed()),
         ))
         .recover(warp_utils::reject::handle_rejection)
         .with(slog_logging(log.clone()))

beacon_node/http_api/src/ui.rs

@@ -1,10 +1,11 @@
-use beacon_chain::{BeaconChain, BeaconChainError, BeaconChainTypes};
+use beacon_chain::{metrics, BeaconChain, BeaconChainError, BeaconChainTypes};
 use eth2::types::ValidatorStatus;
 use serde::{Deserialize, Serialize};
+use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
 use warp_utils::reject::beacon_chain_error;
 
-#[derive(Debug, Default, PartialEq, Clone, Serialize, Deserialize)]
+#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
 pub struct ValidatorCountResponse {
     pub active_ongoing: u64,
     pub active_exiting: u64,
@@ -69,3 +70,126 @@ pub fn get_validator_count<T: BeaconChainTypes>(
         exited_slashed,
     })
 }
+
+#[derive(PartialEq, Serialize, Deserialize)]
+pub struct ValidatorMetricsRequestData {
+    indices: Vec<u64>,
+}
+
+#[derive(PartialEq, Serialize, Deserialize)]
+pub struct ValidatorMetrics {
+    attestation_hits: u64,
+    attestation_misses: u64,
+    attestation_hit_percentage: f64,
+    attestation_head_hits: u64,
+    attestation_head_misses: u64,
+    attestation_head_hit_percentage: f64,
+    attestation_target_hits: u64,
+    attestation_target_misses: u64,
+    attestation_target_hit_percentage: f64,
+}
+
+#[derive(PartialEq, Serialize, Deserialize)]
+pub struct ValidatorMetricsResponse {
+    validators: HashMap<String, ValidatorMetrics>,
+}
+
+pub fn post_validator_monitor_metrics<T: BeaconChainTypes>(
+    request_data: ValidatorMetricsRequestData,
+    chain: Arc<BeaconChain<T>>,
+) -> Result<ValidatorMetricsResponse, warp::Rejection> {
+    let validator_ids = chain
+        .validator_monitor
+        .read()
+        .get_all_monitored_validators()
+        .iter()
+        .cloned()
+        .collect::<HashSet<String>>();
+
+    let indices = request_data
+        .indices
+        .iter()
+        .map(|index| index.to_string())
+        .collect::<HashSet<String>>();
+
+    let ids = validator_ids
+        .intersection(&indices)
+        .collect::<HashSet<&String>>();
+
+    let mut validators = HashMap::new();
+
+    for id in ids {
+        let attestation_hits = metrics::get_int_counter(
+            &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_ATTESTER_HIT,
+            &[id],
+        )
+        .map(|counter| counter.get())
+        .unwrap_or(0);
+        let attestation_misses = metrics::get_int_counter(
+            &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_ATTESTER_MISS,
+            &[id],
+        )
+        .map(|counter| counter.get())
+        .unwrap_or(0);
+        let attestations = attestation_hits + attestation_misses;
+        let attestation_hit_percentage: f64 = if attestations == 0 {
+            0.0
+        } else {
+            (100 * attestation_hits / attestations) as f64
+        };
+
+        let attestation_head_hits = metrics::get_int_counter(
+            &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_HEAD_ATTESTER_HIT,
+            &[id],
+        )
+        .map(|counter| counter.get())
+        .unwrap_or(0);
+        let attestation_head_misses = metrics::get_int_counter(
+            &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_HEAD_ATTESTER_MISS,
+            &[id],
+        )
+        .map(|counter| counter.get())
+        .unwrap_or(0);
+        let head_attestations = attestation_head_hits + attestation_head_misses;
+        let attestation_head_hit_percentage: f64 = if head_attestations == 0 {
+            0.0
+        } else {
+            (100 * attestation_head_hits / head_attestations) as f64
+        };
+
+        let attestation_target_hits = metrics::get_int_counter(
+            &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_TARGET_ATTESTER_HIT,
+            &[id],
+        )
+        .map(|counter| counter.get())
+        .unwrap_or(0);
+        let attestation_target_misses = metrics::get_int_counter(
+            &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_TARGET_ATTESTER_MISS,
+            &[id],
+        )
+        .map(|counter| counter.get())
+        .unwrap_or(0);
+        let target_attestations = attestation_target_hits + attestation_target_misses;
+        let attestation_target_hit_percentage: f64 = if target_attestations == 0 {
+            0.0
+        } else {
+            (100 * attestation_target_hits / target_attestations) as f64
+        };
+
+        let metrics = ValidatorMetrics {
+            attestation_hits,
+            attestation_misses,
+            attestation_hit_percentage,
+            attestation_head_hits,
+            attestation_head_misses,
+            attestation_head_hit_percentage,
+            attestation_target_hits,
+            attestation_target_misses,
+            attestation_target_hit_percentage,
+        };
+
+        validators.insert(id.clone(), metrics);
+    }
+
+    Ok(ValidatorMetricsResponse { validators })
+}

book/src/api-lighthouse.md

@@ -121,6 +121,33 @@ curl -X GET "http://localhost:5052/lighthouse/ui/validator_count" -H "accept: ap
 }
 ```
 
+### `/lighthouse/ui/validator_metrics`
+Re-exposes certain metrics from the validator monitor to the HTTP API.
+Will only return metrics for the validators currently being monitored and are present in the POST data.
+```bash
+curl -X POST "http://localhost:5052/lighthouse/ui/validator_metrics" -d '{"indices": [12345]}' -H "Content-Type: application/json" | jq
+```
+
+```json
+{
+  "data": {
+    "validators": {
+      "12345": {
+        "attestation_hits": 10,
+        "attestation_misses": 0,
+        "attestation_hit_percentage": 100,
+        "attestation_head_hits": 10,
+        "attestation_head_misses": 0,
+        "attestation_head_hit_percentage": 100,
+        "attestation_target_hits": 5,
+        "attestation_target_misses": 5,
+        "attestation_target_hit_percentage": 50 
+      }
+    }
+  }
+}
+```
+
 ### `/lighthouse/syncing`
 
 ```bash