First RESTful HTTP API (#399)

* Added generated code for REST API.
 - Created a new crate rest_api, which will adapt the openapi generated code to Lighthouse
 - Committed automatically generated code from openapi-generator-cli (via docker). Should hopfully not have to modify this at all, and do all changes in the rest_api crate.

* Removed openapi generated code, because it was the rust client, not the rust server.

* Added the correct rust-server code, automatically generated from openapi.

* Added generated code for REST API.
 - Created a new crate rest_api, which will adapt the openapi generated code to Lighthouse
 - Committed automatically generated code from openapi-generator-cli (via docker). Should hopfully not have to modify this at all, and do all changes in the rest_api crate.

* Removed openapi generated code, because it was the rust client, not the rust server.

* Added the correct rust-server code, automatically generated from openapi.

* Included REST API in configuratuion.
 - Started adding the rest_api into the beacon node's dependencies.
 - Set up configuration file for rest_api and integrated into main client config
 - Added CLI flags for REST API.

* Futher work on REST API.
 - Adding the dependencies to rest_api crate
 - Created a skeleton BeaconNodeService, which will handle /node requests.
 - Started the rest_api server definition, with the high level request handling logic.

* Added generated code for REST API.
 - Created a new crate rest_api, which will adapt the openapi generated code to Lighthouse
 - Committed automatically generated code from openapi-generator-cli (via docker). Should hopfully not have to modify this at all, and do all changes in the rest_api crate.

* Removed openapi generated code, because it was the rust client, not the rust server.

* Added the correct rust-server code, automatically generated from openapi.

* Included REST API in configuratuion.
 - Started adding the rest_api into the beacon node's dependencies.
 - Set up configuration file for rest_api and integrated into main client config
 - Added CLI flags for REST API.

* Futher work on REST API.
 - Adding the dependencies to rest_api crate
 - Created a skeleton BeaconNodeService, which will handle /node requests.
 - Started the rest_api server definition, with the high level request handling logic.

* WIP: Restructured REST API to use hyper_router and separate services.

* WIP: Fixing rust for REST API

* WIP: Fixed up many bugs in trying to get router to compile.

* WIP: Got the beacon_node to compile with the REST changes

* Basic API works!
 - Changed CLI flags from rest-api* to api*
 - Fixed port cli flag
 - Tested, works over HTTP

* WIP: Moved things around so that we can get state inside the handlers.

* WIP: Significant API updates.
 - Started writing a macro for getting the handler functions.
 - Added the BeaconChain into the type map, gives stateful access to the beacon state.
 - Created new generic error types (haven't figured out yet), to reduce code duplication.
 - Moved common stuff into lib.rs

* WIP: Factored macros, defined API result and error.
 - did more logging when creating HTTP responses
 - Tried moving stuff into macros, but can't get macros in macros to compile.
 - Pulled out a lot of placeholder code.

* Fixed macros so that things compile.

* Cleaned up code.
 - Removed unused imports
 - Removed comments
 - Addressed all compiler warnings.
 - Ran cargo fmt.

* Removed auto-generated OpenAPI code.

* Addressed Paul's suggestions.
 - Fixed spelling mistake
 - Moved the simple macros into functions, since it doesn't make sense for them to be macros.
 - Removed redundant code & inclusions.

* Removed redundant validate_request function.

* Included graceful shutdown in Hyper server.

* Fixing the dropped exit_signal, which prevented the API from starting.

* Wrapped the exit signal, to get an API shutdown log line.

Cargo.toml

@@ -26,6 +26,7 @@ members = [
 	"beacon_node/store",
 	"beacon_node/client",
 	"beacon_node/http_server",
+	"beacon_node/rest_api",
 	"beacon_node/network",
 	"beacon_node/eth2-libp2p",
     "beacon_node/rpc",

beacon_node/client/Cargo.toml

@@ -9,6 +9,7 @@ beacon_chain = { path = "../beacon_chain" }
 network = { path = "../network" }
 http_server = { path = "../http_server" }
 rpc = { path = "../rpc" }
+rest_api = { path = "../rest_api" }
 prometheus = "^0.6"
 types = { path = "../../eth2/types" }
 tree_hash = { path = "../../eth2/utils/tree_hash" }

beacon_node/client/src/config.rs

@@ -17,6 +17,7 @@ pub struct Config {
     pub network: network::NetworkConfig,
     pub rpc: rpc::RPCConfig,
     pub http: HttpServerConfig,
+    pub rest_api: rest_api::APIConfig,
 }
 
 impl Default for Config {
@@ -31,6 +32,7 @@ impl Default for Config {
             network: NetworkConfig::new(),
             rpc: rpc::RPCConfig::default(),
             http: HttpServerConfig::default(),
+            rest_api: rest_api::APIConfig::default(),
         }
     }
 }
@@ -101,6 +103,7 @@ impl Config {
         self.network.apply_cli_args(args)?;
         self.rpc.apply_cli_args(args)?;
         self.http.apply_cli_args(args)?;
+        self.rest_api.apply_cli_args(args)?;
 
         if let Some(log_file) = args.value_of("logfile") {
             self.log_file = PathBuf::from(log_file);

beacon_node/client/src/lib.rs

@@ -2,6 +2,7 @@ extern crate slog;
 
 mod beacon_chain_types;
 mod config;
+
 pub mod error;
 pub mod notifier;
 
@@ -39,6 +40,8 @@ pub struct Client<T: BeaconChainTypes> {
     pub http_exit_signal: Option<Signal>,
     /// Signal to terminate the slot timer.
     pub slot_timer_exit_signal: Option<Signal>,
+    /// Signal to terminate the API
+    pub api_exit_signal: Option<Signal>,
     /// The clients logger.
     log: slog::Logger,
     /// Marker to pin the beacon chain generics.
@@ -143,6 +146,24 @@ where
             None
         };
 
+        // Start the `rest_api` service
+        let api_exit_signal = if client_config.rest_api.enabled {
+            match rest_api::start_server(
+                &client_config.rest_api,
+                executor,
+                beacon_chain.clone(),
+                &log,
+            ) {
+                Ok(s) => Some(s),
+                Err(e) => {
+                    error!(log, "API service failed to start."; "error" => format!("{:?}",e));
+                    None
+                }
+            }
+        } else {
+            None
+        };
+
         let (slot_timer_exit_signal, exit) = exit_future::signal();
         if let Ok(Some(duration_to_next_slot)) = beacon_chain.slot_clock.duration_to_next_slot() {
             // set up the validator work interval - start at next slot and proceed every slot
@@ -175,6 +196,7 @@ where
             http_exit_signal,
             rpc_exit_signal,
             slot_timer_exit_signal: Some(slot_timer_exit_signal),
+            api_exit_signal,
             log,
             network,
             phantom: PhantomData,

beacon_node/rest_api/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "rest_api"
+version = "0.1.0"
+authors = ["Luke Anderson <luke@lukeanderson.com.au>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[dependencies]
+beacon_chain = { path = "../beacon_chain" }
+version = { path = "../version" }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "^1.0"
+slog = "^2.2.3"
+slog-term = "^2.4.0"
+slog-async = "^2.3.0"
+clap = "2.32.0"
+http = "^0.1.17"
+hyper = "0.12.32"
+hyper-router = "^0.5"
+futures = "0.1"
+exit-future = "0.1.3"
+tokio = "0.1.17"

beacon_node/rest_api/src/beacon_node.rs

@@ -0,0 +1,65 @@
+use beacon_chain::{BeaconChain, BeaconChainTypes};
+use serde::Serialize;
+use slog::info;
+use std::sync::Arc;
+use version;
+
+use super::{path_from_request, success_response, APIResult, APIService};
+
+use hyper::{Body, Request, Response};
+use hyper_router::{Route, RouterBuilder};
+
+#[derive(Clone)]
+pub struct BeaconNodeServiceInstance<T: BeaconChainTypes + 'static> {
+    pub marker: std::marker::PhantomData<T>,
+}
+
+/// A string which uniquely identifies the client implementation and its version; similar to [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3).
+#[derive(Serialize)]
+pub struct Version(String);
+impl From<String> for Version {
+    fn from(x: String) -> Self {
+        Version(x)
+    }
+}
+
+/// The genesis_time configured for the beacon node, which is the unix time at which the Eth2.0 chain began.
+#[derive(Serialize)]
+pub struct GenesisTime(u64);
+impl From<u64> for GenesisTime {
+    fn from(x: u64) -> Self {
+        GenesisTime(x)
+    }
+}
+
+impl<T: BeaconChainTypes + 'static> APIService for BeaconNodeServiceInstance<T> {
+    fn add_routes(&mut self, router_builder: RouterBuilder) -> Result<RouterBuilder, hyper::Error> {
+        let router_builder = router_builder
+            .add(Route::get("/version").using(result_to_response!(get_version)))
+            .add(Route::get("/genesis_time").using(result_to_response!(get_genesis_time::<T>)));
+        Ok(router_builder)
+    }
+}
+
+/// Read the version string from the current Lighthouse build.
+fn get_version(_req: Request<Body>) -> APIResult {
+    let ver = Version::from(version::version());
+    let body = Body::from(
+        serde_json::to_string(&ver).expect("Version should always be serialializable as JSON."),
+    );
+    Ok(success_response(body))
+}
+
+/// Read the genesis time from the current beacon chain state.
+fn get_genesis_time<T: BeaconChainTypes + 'static>(req: Request<Body>) -> APIResult {
+    let beacon_chain = req.extensions().get::<Arc<BeaconChain<T>>>().unwrap();
+    let gen_time = {
+        let state = beacon_chain.current_state();
+        state.genesis_time
+    };
+    let body = Body::from(
+        serde_json::to_string(&gen_time)
+            .expect("Genesis should time always have a valid JSON serialization."),
+    );
+    Ok(success_response(body))
+}

beacon_node/rest_api/src/config.rs

@@ -0,0 +1,46 @@
+use clap::ArgMatches;
+use serde::{Deserialize, Serialize};
+use std::net::Ipv4Addr;
+
+/// HTTP REST API Configuration
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Config {
+    /// Enable the REST API server.
+    pub enabled: bool,
+    /// The IPv4 address the REST API HTTP server will listen on.
+    pub listen_address: Ipv4Addr,
+    /// The port the REST API HTTP server will listen on.
+    pub port: u16,
+}
+
+impl Default for Config {
+    fn default() -> Self {
+        Config {
+            enabled: true, // rest_api enabled by default
+            listen_address: Ipv4Addr::new(127, 0, 0, 1),
+            port: 1248,
+        }
+    }
+}
+
+impl Config {
+    pub fn apply_cli_args(&mut self, args: &ArgMatches) -> Result<(), &'static str> {
+        if args.is_present("api") {
+            self.enabled = true;
+        }
+
+        if let Some(rpc_address) = args.value_of("api-address") {
+            self.listen_address = rpc_address
+                .parse::<Ipv4Addr>()
+                .map_err(|_| "api-address is not a valid IPv4 address.")?;
+        }
+
+        if let Some(rpc_port) = args.value_of("api-port") {
+            self.port = rpc_port
+                .parse::<u16>()
+                .map_err(|_| "api-port is not a valid u16.")?;
+        }
+
+        Ok(())
+    }
+}

beacon_node/rest_api/src/lib.rs

@@ -0,0 +1,132 @@
+extern crate futures;
+extern crate hyper;
+#[macro_use]
+mod macros;
+mod beacon_node;
+pub mod config;
+
+use beacon_chain::{BeaconChain, BeaconChainTypes};
+pub use config::Config as APIConfig;
+
+use slog::{info, o, warn};
+use std::sync::Arc;
+use tokio::runtime::TaskExecutor;
+
+use crate::beacon_node::BeaconNodeServiceInstance;
+use hyper::rt::Future;
+use hyper::service::{service_fn, Service};
+use hyper::{Body, Request, Response, Server, StatusCode};
+use hyper_router::{RouterBuilder, RouterService};
+
+pub enum APIError {
+    MethodNotAllowed { desc: String },
+    ServerError { desc: String },
+    NotImplemented { desc: String },
+}
+
+pub type APIResult = Result<Response<Body>, APIError>;
+
+impl Into<Response<Body>> for APIError {
+    fn into(self) -> Response<Body> {
+        let status_code: (StatusCode, String) = match self {
+            APIError::MethodNotAllowed { desc } => (StatusCode::METHOD_NOT_ALLOWED, desc),
+            APIError::ServerError { desc } => (StatusCode::INTERNAL_SERVER_ERROR, desc),
+            APIError::NotImplemented { desc } => (StatusCode::NOT_IMPLEMENTED, desc),
+        };
+        Response::builder()
+            .status(status_code.0)
+            .body(Body::from(status_code.1))
+            .expect("Response should always be created.")
+    }
+}
+
+pub trait APIService {
+    fn add_routes(&mut self, router_builder: RouterBuilder) -> Result<RouterBuilder, hyper::Error>;
+}
+
+pub fn start_server<T: BeaconChainTypes + Clone + 'static>(
+    config: &APIConfig,
+    executor: &TaskExecutor,
+    beacon_chain: Arc<BeaconChain<T>>,
+    log: &slog::Logger,
+) -> Result<exit_future::Signal, hyper::Error> {
+    let log = log.new(o!("Service" => "API"));
+
+    // build a channel to kill the HTTP server
+    let (exit_signal, exit) = exit_future::signal();
+
+    let exit_log = log.clone();
+    let server_exit = exit.and_then(move |_| {
+        info!(exit_log, "API service shutdown");
+        Ok(())
+    });
+
+    // Get the address to bind to
+    let bind_addr = (config.listen_address, config.port).into();
+
+    // Clone our stateful objects, for use in service closure.
+    let server_log = log.clone();
+    let server_bc = beacon_chain.clone();
+
+    // Create the service closure
+    let service = move || {
+        //TODO: This router must be moved out of this closure, so it isn't rebuilt for every connection.
+        let mut router = build_router_service::<T>();
+
+        // Clone our stateful objects, for use in handler closure
+        let service_log = server_log.clone();
+        let service_bc = server_bc.clone();
+
+        // Create a simple handler for the router, inject our stateful objects into the request.
+        service_fn(move |mut req| {
+            req.extensions_mut()
+                .insert::<slog::Logger>(service_log.clone());
+            req.extensions_mut()
+                .insert::<Arc<BeaconChain<T>>>(service_bc.clone());
+            router.call(req)
+        })
+    };
+
+    let server = Server::bind(&bind_addr)
+        .serve(service)
+        .with_graceful_shutdown(server_exit)
+        .map_err(move |e| {
+            warn!(
+                log,
+                "API failed to start, Unable to bind"; "address" => format!("{:?}", e)
+            )
+        });
+
+    executor.spawn(server);
+
+    Ok(exit_signal)
+}
+
+fn build_router_service<T: BeaconChainTypes + 'static>() -> RouterService {
+    let mut router_builder = RouterBuilder::new();
+
+    let mut bn_service: BeaconNodeServiceInstance<T> = BeaconNodeServiceInstance {
+        marker: std::marker::PhantomData,
+    };
+
+    router_builder = bn_service
+        .add_routes(router_builder)
+        .expect("The routes should always be made.");
+
+    RouterService::new(router_builder.build())
+}
+
+fn path_from_request(req: &Request<Body>) -> String {
+    req.uri()
+        .path_and_query()
+        .as_ref()
+        .map(|pq| String::from(pq.as_str()))
+        .unwrap_or(String::new())
+}
+
+fn success_response(body: Body) -> Response<Body> {
+    Response::builder()
+        .status(StatusCode::OK)
+        .body(body)
+        .expect("We should always be able to make response from the success body.")
+}

beacon_node/rest_api/src/macros.rs

@@ -0,0 +1,23 @@
+macro_rules! result_to_response {
+    ($handler: path) => {
+        |req: Request<Body>| -> Response<Body> {
+            let log = req
+                .extensions()
+                .get::<slog::Logger>()
+                .expect("Our logger should be on req.")
+                .clone();
+            let path = path_from_request(&req);
+            let result = $handler(req);
+            match result {
+                Ok(response) => {
+                    info!(log, "Request successful: {:?}", path);
+                    response
+                }
+                Err(e) => {
+                    info!(log, "Request failure: {:?}", path);
+                    e.into()
+                }
+            }
+        }
+    };
+}

beacon_node/src/main.rs

@@ -127,6 +127,29 @@ fn main() {
                 .help("Listen port for the HTTP server.")
                 .takes_value(true),
         )
+        // REST API related arguments
+        .arg(
+            Arg::with_name("api")
+                .long("api")
+                .value_name("API")
+                .help("Enable the RESTful HTTP API server.")
+                .takes_value(false),
+        )
+        .arg(
+            Arg::with_name("api-address")
+                .long("api-address")
+                .value_name("APIADDRESS")
+                .help("Set the listen address for the RESTful HTTP API server.")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("api-port")
+                .long("api-port")
+                .value_name("APIPORT")
+                .help("Set the listen TCP port for the RESTful HTTP API server.")
+                .takes_value(true),
+        )
+        // General arguments
         .arg(
             Arg::with_name("db")
                 .long("db")