From 328f11d564f32fc73680535e517d64d0e2a34ba9 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Wed, 28 Aug 2019 00:40:35 +1000 Subject: [PATCH] Validator API (#504) * Implemented more REST API endpoints. - Added many of the endpoints, which return 501 - Not Implemented - Created helper function to return a not implemented error - Created a parse_pubkey function to get a PublicKey from a hex string - Created a HTTP handler for the validator endpoints - Started implementing validator/duties endpoint. * Fleshed out get validator duties. - Re-implemented the get validator duties function for the REST API - Added an 'as_hex_string' function to FakePublicKey, beacuse it was missing. * Fixed small caching/state bug. * Extended to array of API inputs. - Created function for getting arrays from GET parameters. - Extended get validator duties function to support array of validator duties. * Tidy API to be more consistent with recent decisions * Addressing Paul's comments. - Cleaning up function to get list of proposers. - Removing unnecessary serde annotations - Clarifying error messages - Only accept pubkeys if they are '0x' prefixed. * Fixed formatting with rustfmt. --- beacon_node/rest_api/Cargo.toml | 2 + beacon_node/rest_api/src/helpers.rs | 30 +++++- beacon_node/rest_api/src/lib.rs | 46 +++++++- beacon_node/rest_api/src/url_query.rs | 23 +++- beacon_node/rest_api/src/validator.rs | 149 ++++++++++++++++++++++++++ eth2/utils/bls/src/fake_public_key.rs | 7 ++ eth2/utils/bls/src/public_key.rs | 2 +- 7 files changed, 253 insertions(+), 6 deletions(-) create mode 100644 beacon_node/rest_api/src/validator.rs diff --git a/beacon_node/rest_api/Cargo.toml b/beacon_node/rest_api/Cargo.toml index cac196d9c..cc69faec9 100644 --- a/beacon_node/rest_api/Cargo.toml +++ b/beacon_node/rest_api/Cargo.toml @@ -6,6 +6,7 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +bls = { path = "../../eth2/utils/bls" } beacon_chain = { path = "../beacon_chain" } network = { path = "../network" } eth2-libp2p = { path = "../eth2-libp2p" } @@ -29,3 +30,4 @@ url = "2.0" lazy_static = "1.3.0" lighthouse_metrics = { path = "../../eth2/utils/lighthouse_metrics" } slot_clock = { path = "../../eth2/utils/slot_clock" } +hex = "0.3.2" diff --git a/beacon_node/rest_api/src/helpers.rs b/beacon_node/rest_api/src/helpers.rs index 5365086df..88755fcde 100644 --- a/beacon_node/rest_api/src/helpers.rs +++ b/beacon_node/rest_api/src/helpers.rs @@ -1,5 +1,10 @@ -use crate::ApiError; +use crate::{ApiError, ApiResult}; use beacon_chain::{BeaconChain, BeaconChainTypes}; +use bls::PublicKey; +use hex; +use hyper::{Body, Request, StatusCode}; +use serde::de::value::StringDeserializer; +use serde_json::Deserializer; use store::{iter::AncestorIter, Store}; use types::{BeaconState, EthSpec, Hash256, RelativeEpoch, Slot}; @@ -31,6 +36,23 @@ pub fn parse_root(string: &str) -> Result { } } +/// Parse a PublicKey from a `0x` prefixed hex string +pub fn parse_pubkey(string: &str) -> Result { + const PREFIX: &str = "0x"; + if string.starts_with(PREFIX) { + let pubkey_bytes = hex::decode(string.trim_start_matches(PREFIX)) + .map_err(|e| ApiError::InvalidQueryParams(format!("Invalid hex string: {:?}", e)))?; + let pubkey = PublicKey::from_bytes(pubkey_bytes.as_slice()).map_err(|e| { + ApiError::InvalidQueryParams(format!("Unable to deserialize public key: {:?}.", e)) + })?; + return Ok(pubkey); + } else { + return Err(ApiError::InvalidQueryParams( + "Public key must have a '0x' prefix".to_string(), + )); + } +} + /// Returns the root of the `BeaconBlock` in the canonical chain of `beacon_chain` at the given /// `slot`, if possible. /// @@ -143,6 +165,12 @@ pub fn state_root_at_slot( } } +pub fn implementation_pending_response(_req: Request) -> ApiResult { + Err(ApiError::NotImplemented( + "API endpoint has not yet been implemented, but is planned to be soon.".to_owned(), + )) +} + #[cfg(test)] mod test { use super::*; diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index 964dd7998..b943a1d45 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -10,6 +10,7 @@ mod network; mod node; mod spec; mod url_query; +mod validator; use beacon_chain::{BeaconChain, BeaconChainTypes}; use client_network::Service as NetworkService; @@ -123,14 +124,26 @@ pub fn start_server( // Route the request to the correct handler. let result = match (req.method(), path.as_ref()) { + // Methods for Beacon Node + //TODO: Remove? + //(&Method::GET, "/beacon/best_slot") => beacon::get_best_slot::(req), (&Method::GET, "/beacon/head") => beacon::get_head::(req), (&Method::GET, "/beacon/block") => beacon::get_block::(req), + (&Method::GET, "/beacon/blocks") => helpers::implementation_pending_response(req), + //TODO Is the below replaced by finalized_checkpoint? + (&Method::GET, "/beacon/chainhead") => { + helpers::implementation_pending_response(req) + } (&Method::GET, "/beacon/block_root") => beacon::get_block_root::(req), (&Method::GET, "/beacon/latest_finalized_checkpoint") => { beacon::get_latest_finalized_checkpoint::(req) } (&Method::GET, "/beacon/state") => beacon::get_state::(req), (&Method::GET, "/beacon/state_root") => beacon::get_state_root::(req), + + //TODO: Add aggreggate/filtered state lookups here, e.g. /beacon/validators/balances + + // Methods for Client (&Method::GET, "/metrics") => metrics::get_prometheus::(req), (&Method::GET, "/network/enr") => network::get_enr::(req), (&Method::GET, "/network/peer_count") => network::get_peer_count::(req), @@ -142,9 +155,40 @@ pub fn start_server( } (&Method::GET, "/node/version") => node::get_version(req), (&Method::GET, "/node/genesis_time") => node::get_genesis_time::(req), + (&Method::GET, "/node/deposit_contract") => { + helpers::implementation_pending_response(req) + } + (&Method::GET, "/node/syncing") => helpers::implementation_pending_response(req), + (&Method::GET, "/node/fork") => helpers::implementation_pending_response(req), + + // Methods for Network + (&Method::GET, "/network/enr") => network::get_enr::(req), + (&Method::GET, "/network/peer_count") => network::get_peer_count::(req), + (&Method::GET, "/network/peer_id") => network::get_peer_id::(req), + (&Method::GET, "/network/peers") => network::get_peer_list::(req), + (&Method::GET, "/network/listen_addresses") => { + network::get_listen_addresses::(req) + } + + // Methods for Validator + (&Method::GET, "/validator/duties") => validator::get_validator_duties::(req), + (&Method::GET, "/validator/block") => helpers::implementation_pending_response(req), + (&Method::POST, "/validator/block") => { + helpers::implementation_pending_response(req) + } + (&Method::GET, "/validator/attestation") => { + helpers::implementation_pending_response(req) + } + (&Method::POST, "/validator/attestation") => { + helpers::implementation_pending_response(req) + } + (&Method::GET, "/spec") => spec::get_spec::(req), (&Method::GET, "/spec/slots_per_epoch") => spec::get_slots_per_epoch::(req), - _ => Err(ApiError::MethodNotAllowed(path.clone())), + + _ => Err(ApiError::NotFound( + "Request path and/or method not found.".to_owned(), + )), }; let response = match result { diff --git a/beacon_node/rest_api/src/url_query.rs b/beacon_node/rest_api/src/url_query.rs index d65312a9e..e39a9a449 100644 --- a/beacon_node/rest_api/src/url_query.rs +++ b/beacon_node/rest_api/src/url_query.rs @@ -2,6 +2,8 @@ use crate::ApiError; use hyper::Request; /// Provides handy functions for parsing the query parameters of a URL. + +#[derive(Clone, Copy)] pub struct UrlQuery<'a>(url::form_urlencoded::Parse<'a>); impl<'a> UrlQuery<'a> { @@ -11,9 +13,7 @@ impl<'a> UrlQuery<'a> { pub fn from_request(req: &'a Request) -> Result { let query_str = req.uri().query().ok_or_else(|| { ApiError::InvalidQueryParams( - "URL query must be valid and contain at least one - key." - .to_string(), + "URL query must be valid and contain at least one key.".to_string(), ) })?; @@ -60,6 +60,23 @@ impl<'a> UrlQuery<'a> { ))) } } + + /// Returns a vector of all values present where `key` is in `keys + /// + /// If no match is found, an `InvalidQueryParams` error is returned. + pub fn all_of(mut self, key: &str) -> Result, ApiError> { + let queries: Vec<_> = self + .0 + .filter_map(|(k, v)| { + if k.eq(key) { + Some(v.into_owned()) + } else { + None + } + }) + .collect(); + Ok(queries) + } } #[cfg(test)] diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs new file mode 100644 index 000000000..4294f9c20 --- /dev/null +++ b/beacon_node/rest_api/src/validator.rs @@ -0,0 +1,149 @@ +use super::{success_response, ApiResult}; +use crate::{helpers::*, ApiError, UrlQuery}; +use beacon_chain::{BeaconChain, BeaconChainTypes}; +use bls::PublicKey; +use hyper::{Body, Request}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use store::Store; +use types::beacon_state::EthSpec; +use types::{BeaconBlock, BeaconState, Epoch, RelativeEpoch, Shard, Slot}; + +#[derive(Debug, Serialize, Deserialize)] +pub struct ValidatorDuty { + /// The validator's BLS public key, uniquely identifying them. _48-bytes, hex encoded with 0x prefix, case insensitive._ + pub validator_pubkey: String, + /// The slot at which the validator must attest. + pub attestation_slot: Option, + /// The shard in which the validator must attest. + pub attestation_shard: Option, + /// The slot in which a validator must propose a block, or `null` if block production is not required. + pub block_proposal_slot: Option, +} + +impl ValidatorDuty { + pub fn new() -> ValidatorDuty { + ValidatorDuty { + validator_pubkey: "".to_string(), + attestation_slot: None, + attestation_shard: None, + block_proposal_slot: None, + } + } +} + +/// HTTP Handler to retrieve a the duties for a set of validators during a particular epoch +pub fn get_validator_duties(req: Request) -> ApiResult { + // Get beacon state + let beacon_chain = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; + let _ = beacon_chain + .ensure_state_caches_are_built() + .map_err(|e| ApiError::ServerError(format!("Unable to build state caches: {:?}", e)))?; + let head_state = beacon_chain + .speculative_state() + .expect("This is legacy code and should be removed."); + + // Parse and check query parameters + let query = UrlQuery::from_request(&req)?; + + let current_epoch = head_state.current_epoch(); + let epoch = match query.first_of(&["epoch"]) { + Ok((_, v)) => Epoch::new(v.parse::().map_err(|e| { + ApiError::InvalidQueryParams(format!("Invalid epoch parameter, must be a u64. {:?}", e)) + })?), + Err(_) => { + // epoch not supplied, use the current epoch + current_epoch + } + }; + let relative_epoch = RelativeEpoch::from_epoch(current_epoch, epoch).map_err(|e| { + ApiError::InvalidQueryParams(format!( + "Cannot get RelativeEpoch, epoch out of range: {:?}", + e + )) + })?; + //TODO: Handle an array of validators, currently only takes one + let mut validators: Vec = match query.all_of("validator_pubkeys") { + Ok(v) => v + .iter() + .map(|pk| parse_pubkey(pk)) + .collect::, _>>()?, + Err(e) => { + return Err(e); + } + }; + let mut duties: Vec = Vec::new(); + + // Get a list of all validators for this epoch + let validator_proposers: Vec = epoch + .slot_iter(T::EthSpec::slots_per_epoch()) + .map(|slot| { + head_state + .get_beacon_proposer_index(slot, relative_epoch, &beacon_chain.spec) + .map_err(|e| { + ApiError::ServerError(format!( + "Unable to get proposer index for validator: {:?}", + e + )) + }) + }) + .collect::, _>>()?; + + // Look up duties for each validator + for val_pk in validators { + let mut duty = ValidatorDuty::new(); + duty.validator_pubkey = val_pk.as_hex_string(); + + // Get the validator index + // If it does not exist in the index, just add a null duty and move on. + let val_index: usize = match head_state.get_validator_index(&val_pk) { + Ok(Some(i)) => i, + Ok(None) => { + duties.append(&mut vec![duty]); + continue; + } + Err(e) => { + return Err(ApiError::ServerError(format!( + "Unable to read validator index cache. {:?}", + e + ))); + } + }; + + // Set attestation duties + match head_state.get_attestation_duties(val_index, relative_epoch) { + Ok(Some(d)) => { + duty.attestation_slot = Some(d.slot); + duty.attestation_shard = Some(d.shard); + } + Ok(None) => {} + Err(e) => { + return Err(ApiError::ServerError(format!( + "unable to read cache for attestation duties: {:?}", + e + ))) + } + }; + + // If the validator is to propose a block, identify the slot + if let Some(slot) = validator_proposers.iter().position(|&v| val_index == v) { + duty.block_proposal_slot = Some(Slot::new( + relative_epoch + .into_epoch(current_epoch) + .start_slot(T::EthSpec::slots_per_epoch()) + .as_u64() + + slot as u64, + )); + } + + duties.append(&mut vec![duty]); + } + let body = Body::from( + serde_json::to_string(&duties) + .expect("We should always be able to serialize the duties we created."), + ); + Ok(success_response(body)) +} diff --git a/eth2/utils/bls/src/fake_public_key.rs b/eth2/utils/bls/src/fake_public_key.rs index b783aa0a6..4cd62a132 100644 --- a/eth2/utils/bls/src/fake_public_key.rs +++ b/eth2/utils/bls/src/fake_public_key.rs @@ -61,6 +61,13 @@ impl FakePublicKey { hex_encode(end_bytes) } + /// Returns the point as a hex string of the SSZ encoding. + /// + /// Note: the string is prefixed with `0x`. + pub fn as_hex_string(&self) -> String { + hex_encode(self.as_ssz_bytes()) + } + // Returns itself pub fn as_raw(&self) -> &Self { self diff --git a/eth2/utils/bls/src/public_key.rs b/eth2/utils/bls/src/public_key.rs index acf0139b2..e03b17686 100644 --- a/eth2/utils/bls/src/public_key.rs +++ b/eth2/utils/bls/src/public_key.rs @@ -28,7 +28,7 @@ impl PublicKey { /// Returns the underlying point as compressed bytes. /// /// Identical to `self.as_uncompressed_bytes()`. - fn as_bytes(&self) -> Vec { + pub fn as_bytes(&self) -> Vec { self.as_raw().as_bytes() }