Beacon state validator id filter (#1803)

## Issue Addressed

Michael's comment here: https://github.com/sigp/lighthouse/issues/1434#issuecomment-708834079
Resolves #1808

## Proposed Changes

- Add query param `id` and `status` to the `validators` endpoint
- Add string serialization and deserialization for `ValidatorStatus`
- Drop `Epoch` from `ValidatorStatus` variants

## Additional Info

Please provide any additional information. For example, future considerations
or information useful for reviewers.

beacon_node/http_api/src/lib.rs

@@ -421,40 +421,69 @@ pub fn serve<T: BeaconChainTypes>(
             })
         });
 
-    // GET beacon/states/{state_id}/validators
+    // GET beacon/states/{state_id}/validators?id,status
     let get_beacon_state_validators = beacon_states_path
         .clone()
         .and(warp::path("validators"))
+        .and(warp::query::<api_types::ValidatorsQuery>())
         .and(warp::path::end())
-        .and_then(|state_id: StateId, chain: Arc<BeaconChain<T>>| {
+        .and_then(
-            blocking_json_task(move || {
+            |state_id: StateId, chain: Arc<BeaconChain<T>>, query: api_types::ValidatorsQuery| {
-                state_id
+                blocking_json_task(move || {
-                    .map_state(&chain, |state| {
+                    state_id
-                        let epoch = state.current_epoch();
+                        .map_state(&chain, |state| {
-                        let finalized_epoch = state.finalized_checkpoint.epoch;
+                            let epoch = state.current_epoch();
-                        let far_future_epoch = chain.spec.far_future_epoch;
+                            let finalized_epoch = state.finalized_checkpoint.epoch;
+                            let far_future_epoch = chain.spec.far_future_epoch;
 
-                        Ok(state
+                            Ok(state
-                            .validators
+                                .validators
-                            .iter()
+                                .iter()
-                            .zip(state.balances.iter())
+                                .zip(state.balances.iter())
-                            .enumerate()
+                                .enumerate()
-                            .map(|(index, (validator, balance))| api_types::ValidatorData {
+                                // filter by validator id(s) if provided
-                                index: index as u64,
+                                .filter(|(index, (validator, _))| {
-                                balance: *balance,
+                                    query.id.as_ref().map_or(true, |ids| {
-                                status: api_types::ValidatorStatus::from_validator(
+                                        ids.0.iter().any(|id| match id {
-                                    Some(validator),
+                                            ValidatorId::PublicKey(pubkey) => {
-                                    epoch,
+                                                &validator.pubkey == pubkey
-                                    finalized_epoch,
+                                            }
-                                    far_future_epoch,
+                                            ValidatorId::Index(param_index) => {
-                                ),
+                                                *param_index == *index as u64
-                                validator: validator.clone(),
+                                            }
-                            })
+                                        })
-                            .collect::<Vec<_>>())
+                                    })
-                    })
+                                })
-                    .map(api_types::GenericResponse::from)
+                                // filter by status(es) if provided and map the result
-            })
+                                .filter_map(|(index, (validator, balance))| {
-        });
+                                    let status = api_types::ValidatorStatus::from_validator(
+                                        Some(validator),
+                                        epoch,
+                                        finalized_epoch,
+                                        far_future_epoch,
+                                    );
+
+                                    if query
+                                        .status
+                                        .as_ref()
+                                        .map_or(true, |statuses| statuses.0.contains(&status))
+                                    {
+                                        Some(api_types::ValidatorData {
+                                            index: index as u64,
+                                            balance: *balance,
+                                            status,
+                                            validator: validator.clone(),
+                                        })
+                                    } else {
+                                        None
+                                    }
+                                })
+                                .collect::<Vec<_>>())
+                        })
+                        .map(api_types::GenericResponse::from)
+                })
+            },
+        );
 
     // GET beacon/states/{state_id}/validators/{validator_id}
     let get_beacon_state_validators_id = beacon_states_path
@@ -537,8 +566,8 @@ pub fn serve<T: BeaconChainTypes>(
                         } else {
                             CommitteeCache::initialized(state, epoch, &chain.spec).map(Cow::Owned)
                         }
-                        .map_err(BeaconChainError::BeaconStateError)
+                            .map_err(BeaconChainError::BeaconStateError)
-                        .map_err(warp_utils::reject::beacon_chain_error)?;
+                            .map_err(warp_utils::reject::beacon_chain_error)?;
 
                         // Use either the supplied slot or all slots in the epoch.
                         let slots = query.slot.map(|slot| vec![slot]).unwrap_or_else(|| {
@@ -566,11 +595,11 @@ pub fn serve<T: BeaconChainTypes>(
                                 let committee = committee_cache
                                     .get_beacon_committee(slot, index)
                                     .ok_or_else(|| {
-                                    warp_utils::reject::custom_bad_request(format!(
+                                        warp_utils::reject::custom_bad_request(format!(
-                                        "committee index {} does not exist in epoch {}",
+                                            "committee index {} does not exist in epoch {}",
-                                        index, epoch
+                                            index, epoch
-                                    ))
+                                        ))
-                                })?;
+                                    })?;
 
                                 response.push(api_types::CommitteeData {
                                     index,
@@ -1605,7 +1634,7 @@ pub fn serve<T: BeaconChainTypes>(
                                 return Err(warp_utils::reject::object_invalid(format!(
                                     "gossip verification failed: {:?}",
                                     e
-                                )))
+                                )));
                             }
                         };
 

beacon_node/http_api/tests/tests.rs

@@ -411,40 +411,87 @@ impl ApiTester {
 
     pub async fn test_beacon_states_validators(self) -> Self {
         for state_id in self.interesting_state_ids() {
-            let result = self
+            for statuses in self.interesting_validator_statuses() {
-                .client
+                for validator_indices in self.interesting_validator_indices() {
-                .get_beacon_states_validators(state_id)
+                    let state_opt = self.get_state(state_id);
-                .await
+                    let validators: Vec<Validator> = match state_opt.as_ref() {
-                .unwrap()
+                        Some(state) => state.validators.clone().into(),
-                .map(|res| res.data);
+                        None => vec![],
+                    };
+                    let validator_index_ids = validator_indices
+                        .iter()
+                        .cloned()
+                        .map(|i| ValidatorId::Index(i))
+                        .collect::<Vec<ValidatorId>>();
+                    let validator_pubkey_ids = validator_indices
+                        .iter()
+                        .cloned()
+                        .map(|i| {
+                            ValidatorId::PublicKey(
+                                validators
+                                    .get(i as usize)
+                                    .map_or(PublicKeyBytes::empty(), |val| val.pubkey.clone()),
+                            )
+                        })
+                        .collect::<Vec<ValidatorId>>();
 
-            let expected = self.get_state(state_id).map(|state| {
+                    let result_index_ids = self
-                let epoch = state.current_epoch();
+                        .client
-                let finalized_epoch = state.finalized_checkpoint.epoch;
+                        .get_beacon_states_validators(
-                let far_future_epoch = self.chain.spec.far_future_epoch;
+                            state_id,
+                            Some(validator_index_ids.as_slice()),
+                            None,
+                        )
+                        .await
+                        .unwrap()
+                        .map(|res| res.data);
 
-                let mut validators = Vec::with_capacity(state.validators.len());
+                    let result_pubkey_ids = self
+                        .client
+                        .get_beacon_states_validators(
+                            state_id,
+                            Some(validator_pubkey_ids.as_slice()),
+                            None,
+                        )
+                        .await
+                        .unwrap()
+                        .map(|res| res.data);
 
-                for i in 0..state.validators.len() {
+                    let expected = state_opt.map(|state| {
-                    let validator = state.validators[i].clone();
+                        let epoch = state.current_epoch();
+                        let finalized_epoch = state.finalized_checkpoint.epoch;
+                        let far_future_epoch = self.chain.spec.far_future_epoch;
 
-                    validators.push(ValidatorData {
+                        let mut validators = Vec::with_capacity(validator_indices.len());
-                        index: i as u64,
+
-                        balance: state.balances[i],
+                        for i in validator_indices {
-                        status: ValidatorStatus::from_validator(
+                            if i >= state.validators.len() as u64 {
-                            Some(&validator),
+                                continue;
-                            epoch,
+                            }
-                            finalized_epoch,
+                            let validator = state.validators[i as usize].clone();
-                            far_future_epoch,
+                            let status = ValidatorStatus::from_validator(
-                        ),
+                                Some(&validator),
-                        validator,
+                                epoch,
-                    })
+                                finalized_epoch,
+                                far_future_epoch,
+                            );
+                            if statuses.contains(&status) || statuses.is_empty() {
+                                validators.push(ValidatorData {
+                                    index: i as u64,
+                                    balance: state.balances[i as usize],
+                                    status,
+                                    validator,
+                                });
+                            }
+                        }
+
+                        validators
+                    });
+
+                    assert_eq!(result_index_ids, expected, "{:?}", state_id);
+                    assert_eq!(result_pubkey_ids, expected, "{:?}", state_id);
                 }
-
+            }
-                validators
-            });
-
-            assert_eq!(result, expected, "{:?}", state_id);
         }
 
         self
@@ -1149,6 +1196,28 @@ impl ApiTester {
         interesting
     }
 
+    fn interesting_validator_statuses(&self) -> Vec<Vec<ValidatorStatus>> {
+        let interesting = vec![
+            vec![],
+            vec![ValidatorStatus::Active],
+            vec![
+                ValidatorStatus::Unknown,
+                ValidatorStatus::WaitingForEligibility,
+                ValidatorStatus::WaitingForFinality,
+                ValidatorStatus::WaitingInQueue,
+                ValidatorStatus::StandbyForActive,
+                ValidatorStatus::Active,
+                ValidatorStatus::ActiveAwaitingVoluntaryExit,
+                ValidatorStatus::ActiveAwaitingSlashedExit,
+                ValidatorStatus::ExitedVoluntarily,
+                ValidatorStatus::ExitedSlashed,
+                ValidatorStatus::Withdrawable,
+                ValidatorStatus::Withdrawn,
+            ],
+        ];
+        interesting
+    }
+
     pub async fn test_get_validator_duties_attester(self) -> Self {
         let current_epoch = self.chain.epoch().unwrap().as_u64();
 

common/eth2/src/lib.rs

@@ -210,12 +210,14 @@ impl BeaconNodeHttpClient {
         self.get_opt(path).await
     }
 
-    /// `GET beacon/states/{state_id}/validators`
+    /// `GET beacon/states/{state_id}/validators?id,status`
     ///
     /// Returns `Ok(None)` on a 404 error.
     pub async fn get_beacon_states_validators(
         &self,
         state_id: StateId,
+        ids: Option<&[ValidatorId]>,
+        statuses: Option<&[ValidatorStatus]>,
     ) -> Result<Option<GenericResponse<Vec<ValidatorData>>>, Error> {
         let mut path = self.eth_path()?;
 
@@ -226,6 +228,24 @@ impl BeaconNodeHttpClient {
             .push(&state_id.to_string())
             .push("validators");
 
+        if let Some(ids) = ids {
+            let id_string = ids
+                .iter()
+                .map(|i| i.to_string())
+                .collect::<Vec<_>>()
+                .join(",");
+            path.query_pairs_mut().append_pair("id", &id_string);
+        }
+
+        if let Some(statuses) = statuses {
+            let status_string = statuses
+                .iter()
+                .map(|i| i.to_string())
+                .collect::<Vec<_>>()
+                .join(",");
+            path.query_pairs_mut().append_pair("status", &status_string);
+        }
+
         self.get_opt(path).await
     }
 

common/eth2/src/types.rs

@@ -165,7 +165,7 @@ pub struct FinalityCheckpointsData {
     pub finalized: Checkpoint,
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum ValidatorId {
     PublicKey(PublicKeyBytes),
     Index(u64),
@@ -211,17 +211,18 @@ pub struct ValidatorData {
 //
 // https://hackmd.io/bQxMDRt1RbS1TLno8K4NPg?view
 #[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
 pub enum ValidatorStatus {
     Unknown,
     WaitingForEligibility,
     WaitingForFinality,
     WaitingInQueue,
-    StandbyForActive(Epoch),
+    StandbyForActive,
     Active,
-    ActiveAwaitingVoluntaryExit(Epoch),
+    ActiveAwaitingVoluntaryExit,
-    ActiveAwaitingSlashedExit(Epoch),
+    ActiveAwaitingSlashedExit,
-    ExitedVoluntarily(Epoch),
+    ExitedVoluntarily,
-    ExitedSlashed(Epoch),
+    ExitedSlashed,
     Withdrawable,
     Withdrawn,
 }
@@ -238,22 +239,22 @@ impl ValidatorStatus {
                 ValidatorStatus::Withdrawable
             } else if validator.is_exited_at(epoch) {
                 if validator.slashed {
-                    ValidatorStatus::ExitedSlashed(validator.withdrawable_epoch)
+                    ValidatorStatus::ExitedSlashed
                 } else {
-                    ValidatorStatus::ExitedVoluntarily(validator.withdrawable_epoch)
+                    ValidatorStatus::ExitedVoluntarily
                 }
             } else if validator.is_active_at(epoch) {
                 if validator.exit_epoch < far_future_epoch {
                     if validator.slashed {
-                        ValidatorStatus::ActiveAwaitingSlashedExit(validator.exit_epoch)
+                        ValidatorStatus::ActiveAwaitingSlashedExit
                     } else {
-                        ValidatorStatus::ActiveAwaitingVoluntaryExit(validator.exit_epoch)
+                        ValidatorStatus::ActiveAwaitingVoluntaryExit
                     }
                 } else {
                     ValidatorStatus::Active
                 }
             } else if validator.activation_epoch < far_future_epoch {
-                ValidatorStatus::StandbyForActive(validator.activation_epoch)
+                ValidatorStatus::StandbyForActive
             } else if validator.activation_eligibility_epoch < far_future_epoch {
                 if finalized_epoch < validator.activation_eligibility_epoch {
                     ValidatorStatus::WaitingForFinality
@@ -269,12 +270,61 @@ impl ValidatorStatus {
     }
 }
 
+impl FromStr for ValidatorStatus {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "unknown" => Ok(ValidatorStatus::Unknown),
+            "waiting_for_eligibility" => Ok(ValidatorStatus::WaitingForEligibility),
+            "waiting_for_finality" => Ok(ValidatorStatus::WaitingForFinality),
+            "waiting_in_queue" => Ok(ValidatorStatus::WaitingInQueue),
+            "standby_for_active" => Ok(ValidatorStatus::StandbyForActive),
+            "active" => Ok(ValidatorStatus::Active),
+            "active_awaiting_voluntary_exit" => Ok(ValidatorStatus::ActiveAwaitingVoluntaryExit),
+            "active_awaiting_slashed_exit" => Ok(ValidatorStatus::ActiveAwaitingSlashedExit),
+            "exited_voluntarily" => Ok(ValidatorStatus::ExitedVoluntarily),
+            "exited_slashed" => Ok(ValidatorStatus::ExitedSlashed),
+            "withdrawable" => Ok(ValidatorStatus::Withdrawable),
+            "withdrawn" => Ok(ValidatorStatus::Withdrawn),
+            _ => Err(format!("{} cannot be parsed as a validator status.", s)),
+        }
+    }
+}
+
+impl fmt::Display for ValidatorStatus {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            ValidatorStatus::Unknown => write!(f, "unknown"),
+            ValidatorStatus::WaitingForEligibility => write!(f, "waiting_for_eligibility"),
+            ValidatorStatus::WaitingForFinality => write!(f, "waiting_for_finality"),
+            ValidatorStatus::WaitingInQueue => write!(f, "waiting_in_queue"),
+            ValidatorStatus::StandbyForActive => write!(f, "standby_for_active"),
+            ValidatorStatus::Active => write!(f, "active"),
+            ValidatorStatus::ActiveAwaitingVoluntaryExit => {
+                write!(f, "active_awaiting_voluntary_exit")
+            }
+            ValidatorStatus::ActiveAwaitingSlashedExit => write!(f, "active_awaiting_slashed_exit"),
+            ValidatorStatus::ExitedVoluntarily => write!(f, "exited_voluntarily"),
+            ValidatorStatus::ExitedSlashed => write!(f, "exited_slashed"),
+            ValidatorStatus::Withdrawable => write!(f, "withdrawable"),
+            ValidatorStatus::Withdrawn => write!(f, "withdrawn"),
+        }
+    }
+}
+
 #[derive(Serialize, Deserialize)]
 pub struct CommitteesQuery {
     pub slot: Option<Slot>,
     pub index: Option<u64>,
 }
 
+#[derive(Deserialize)]
+pub struct ValidatorsQuery {
+    pub id: Option<QueryVec<ValidatorId>>,
+    pub status: Option<QueryVec<ValidatorStatus>>,
+}
+
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct CommitteeData {
     #[serde(with = "serde_utils::quoted_u64")]