Upgrade to tokio 0.3 (#1839)

## Description

This PR updates Lighthouse to tokio 0.3. It includes a number of dependency updates and some structural changes as to how we create and spawn tasks.

This also brings with it a number of various improvements:

- Discv5 update
- Libp2p update
- Fix for recompilation issues
- Improved UPnP port mapping handling
- Futures dependency update
- Log downgrade to traces for rejecting peers when we've reached our max



Co-authored-by: blacktemplar <blacktemplar@a1.net>

beacon_node/Cargo.toml

@@ -10,6 +10,7 @@ path = "src/lib.rs"
 
 [dev-dependencies]
 node_test_rig = { path = "../testing/node_test_rig" }
+tokio-compat-02 = "0.1"
 
 [features]
 write_ssz_files = ["beacon_chain/write_ssz_files"]  # Writes debugging .ssz files to /tmp during block processing.
@@ -26,12 +27,12 @@ slog = { version = "2.5.2", features = ["max_level_trace", "release_max_level_tr
 slog-term = "2.6.0"
 slog-async = "2.5.0"
 ctrlc = { version = "3.1.6", features = ["termination"] }
-tokio = { version = "0.2.22", features = ["time"] }
+tokio = { version = "0.3.2", features = ["time"] }
 exit-future = "0.2.0"
 dirs = "3.0.1"
 logging = { path = "../common/logging" }
 directory = {path = "../common/directory"}
-futures = "0.3.5"
+futures = "0.3.7"
 environment = { path = "../lighthouse/environment" }
 task_executor = { path = "../common/task_executor" }
 genesis = { path = "genesis" }

beacon_node/beacon_chain/Cargo.toml

@@ -40,10 +40,10 @@ eth2_ssz_derive = "0.1.0"
 state_processing = { path = "../../consensus/state_processing" }
 tree_hash = "0.1.1"
 types = { path = "../../consensus/types" }
-tokio = "0.2.22"
+tokio = "0.3.2"
 eth1 = { path = "../eth1" }
 websocket_server = { path = "../websocket_server" }
-futures = "0.3.5"
+futures = "0.3.7"
 genesis = { path = "../genesis" }
 integer-sqrt = "0.1.5"
 rand = "0.7.3"

beacon_node/beacon_chain/src/attestation_verification.rs

@@ -413,7 +413,7 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
 
         // Ensure there has been no other observed aggregate for the given `aggregator_index`.
         //
-        // Note: do not observe yet, only observe once the attestation has been verfied.
+        // Note: do not observe yet, only observe once the attestation has been verified.
         match chain
             .observed_aggregators
             .read()

beacon_node/client/Cargo.toml

@@ -27,9 +27,9 @@ error-chain = "0.12.4"
 serde_yaml = "0.8.13"
 slog = { version = "2.5.2", features = ["max_level_trace"] }
 slog-async = "2.5.0"
-tokio = "0.2.22"
+tokio = "0.3.2"
 dirs = "3.0.1"
-futures = "0.3.5"
+futures = "0.3.7"
 reqwest = { version = "0.10.8", features = ["native-tls-vendored"] }
 url = "2.1.1"
 eth1 = { path = "../eth1" }

beacon_node/client/src/builder.rs

@@ -254,10 +254,16 @@ where
                     let (listen_addr, server) = http_api::serve(ctx, exit_future)
                         .map_err(|e| format!("Unable to start HTTP API server: {:?}", e))?;
 
+                    let log_clone = context.log().clone();
+                    let http_api_task = async move {
+                        server.await;
+                        debug!(log_clone, "HTTP API server task ended");
+                    };
+
                     context
                         .clone()
                         .executor
-                        .spawn_without_exit(async move { server.await }, "http-api");
+                        .spawn_without_exit(http_api_task, "http-api");
 
                     Some(listen_addr)
                 } else {
@@ -283,7 +289,7 @@ where
                             "Waiting for HTTP server port to open";
                             "port" => http_listen
                         );
-                        tokio::time::delay_for(Duration::from_secs(1)).await;
+                        tokio::time::sleep(Duration::from_secs(1)).await;
                     }
                 }
 
@@ -442,10 +448,16 @@ where
             let (listen_addr, server) = http_api::serve(ctx, exit)
                 .map_err(|e| format!("Unable to start HTTP API server: {:?}", e))?;
 
+            let http_log = runtime_context.log().clone();
+            let http_api_task = async move {
+                server.await;
+                debug!(http_log, "HTTP API server task ended");
+            };
+
             runtime_context
                 .clone()
                 .executor
-                .spawn_without_exit(async move { server.await }, "http-api");
+                .spawn_without_exit(http_api_task, "http-api");
 
             Some(listen_addr)
         } else {

beacon_node/client/src/notifier.rs

@@ -7,7 +7,7 @@ use slog::{debug, error, info, warn};
 use slot_clock::SlotClock;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
-use tokio::time::delay_for;
+use tokio::time::sleep;
 use types::{EthSpec, Slot};
 
 /// Create a warning log whenever the peer count is at or below this value.
@@ -56,7 +56,7 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
                         "peers" => peer_count_pretty(network.connected_peers()),
                         "wait_time" => estimated_time_pretty(Some(next_slot.as_secs() as f64)),
                     );
-                    delay_for(slot_duration).await;
+                    sleep(slot_duration).await;
                 }
                 _ => break,
             }

beacon_node/eth1/Cargo.toml

@@ -10,10 +10,11 @@ toml = "0.5.6"
 web3 = "0.11.0"
 sloggers = "1.0.1"
 environment = { path = "../../lighthouse/environment" }
+tokio-compat-02 = "0.1"
 
 [dependencies]
 reqwest = { version = "0.10.8", features = ["native-tls-vendored"] }
-futures = { version = "0.3.5", features = ["compat"] }
+futures = { version = "0.3.7", features = ["compat"] }
 serde_json = "1.0.58"
 serde = { version = "1.0.116", features = ["derive"] }
 hex = "0.4.2"
@@ -25,7 +26,7 @@ tree_hash = "0.1.1"
 eth2_hashing = "0.1.0"
 parking_lot = "0.11.0"
 slog = "2.5.2"
-tokio = { version = "0.2.22", features = ["full"] }
+tokio = { version = "0.3.2", features = ["full"] }
 state_processing = { path = "../../consensus/state_processing" }
 libflate = "1.0.2"
 lighthouse_metrics = { path = "../../common/lighthouse_metrics"}

beacon_node/eth2_libp2p/Cargo.toml

@@ -5,7 +5,8 @@ authors = ["Sigma Prime <contact@sigmaprime.io>"]
 edition = "2018"
 
 [dependencies]
-discv5 = { git = "https://github.com/sigp/discv5", rev = "fba7ceb5cfebd219ebbad6ffdb5d8c31dc8e4bc0", features = ["libp2p"] }
+discv5 = { version = "0.1.0-beta.2", features = ["libp2p"] }
+unsigned-varint = { git = "https://github.com/sigp/unsigned-varint", branch = "dep-update", features = ["codec"] }
 types = { path =  "../../consensus/types" }
 hashset_delay = { path = "../../common/hashset_delay" }
 eth2_ssz_types = { path =  "../../consensus/ssz_types" }
@@ -15,15 +16,15 @@ eth2_ssz = "0.1.2"
 eth2_ssz_derive = "0.1.0"
 slog = { version = "2.5.2", features = ["max_level_trace"] }
 lighthouse_version = { path = "../../common/lighthouse_version" }
-tokio = { version = "0.2.22", features = ["time", "macros"] }
-futures = "0.3.5"
+tokio = { version = "0.3.2", features = ["time", "macros"] }
+futures = "0.3.7"
 error-chain = "0.12.4"
 dirs = "3.0.1"
 fnv = "1.0.7"
-unsigned-varint = { git = "https://github.com/sigp/unsigned-varint", branch = "latest-codecs", features = ["codec"] }
 lazy_static = "1.4.0"
 lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
 smallvec = "1.4.2"
+tokio-io-timeout = "0.5.0" 
 lru = "0.6.0"
 parking_lot = "0.11.0"
 sha2 = "0.9.1"
@@ -31,8 +32,7 @@ base64 = "0.13.0"
 snap = "1.0.1"
 void = "1.0.2"
 hex = "0.4.2"
-tokio-io-timeout = "0.4.0"
-tokio-util = { version = "0.3.1", features = ["codec", "compat"] }
+tokio-util = { version = "0.4.0", features = ["codec", "compat"] }
 tiny-keccak = "2.0.2"
 task_executor = { path = "../../common/task_executor" }
 rand = "0.7.3"
@@ -42,12 +42,12 @@ regex = "1.3.9"
 [dependencies.libp2p]
 #version = "0.23.0"
 git = "https://github.com/sigp/rust-libp2p"
-rev = "f53d02bc873fef2bf52cd31e3d5ce366a41d8a8c"
+rev = "e3caf9e0e5e78c9d51c6dccf0d6277cef553bb25"
 default-features = false
 features = ["websocket", "identify", "mplex", "yamux", "noise", "gossipsub", "dns", "tcp-tokio"]
 
 [dev-dependencies]
-tokio = { version = "0.2.22", features = ["full"] }
+tokio = { version = "0.3.2", features = ["full"] }
 slog-term = "2.6.0"
 slog-async = "2.5.0"
 tempdir = "0.3.7"

beacon_node/eth2_libp2p/src/behaviour/mod.rs

@@ -983,7 +983,14 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
         };
 
         if let Some(goodbye_reason) = goodbye_reason {
-            debug!(self.log, "Disconnecting newly connected peer"; "peer_id" => peer_id.to_string(), "reason" => goodbye_reason.to_string());
+            match goodbye_reason {
+                GoodbyeReason::Banned => {
+                    debug!(self.log, "Disconnecting newly connected peer"; "peer_id" => peer_id.to_string(), "reason" => goodbye_reason.to_string())
+                }
+                _ => {
+                    trace!(self.log, "Disconnecting newly connected peer"; "peer_id" => peer_id.to_string(), "reason" => goodbye_reason.to_string())
+                }
+            }
             self.peers_to_dc
                 .push_back((peer_id.clone(), Some(goodbye_reason)));
             // NOTE: We don't inform the peer manager that this peer is disconnecting. It is simply
@@ -1079,6 +1086,8 @@ impl<TSpec: EthSpec> NetworkBehaviour for Behaviour<TSpec> {
             // Inform the behaviour.
             delegate_to_behaviours!(self, inject_disconnected, peer_id);
 
+            debug!(self.log, "Peer disconnected"; "peer_id" => %peer_id);
+
             // Decrement the PEERS_PER_CLIENT metric
             if let Some(kind) = self
                 .network_globals

beacon_node/eth2_libp2p/src/discovery/mod.rs

@@ -212,7 +212,10 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
 
         // Start the discv5 service and obtain an event stream
         let event_stream = if !config.disable_discovery {
-            discv5.start(listen_socket).map_err(|e| e.to_string())?;
+            discv5
+                .start(listen_socket)
+                .map_err(|e| e.to_string())
+                .await?;
             debug!(log, "Discovery service started");
             EventStream::Awaiting(Box::pin(discv5.event_stream()))
         } else {
@@ -712,8 +715,10 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
                 return;
             }
         };
-        // predicate for finding nodes with a matching fork
-        let eth2_fork_predicate = move |enr: &Enr| enr.eth2() == Ok(enr_fork_id.clone());
+        // predicate for finding nodes with a matching fork and valid tcp port
+        let eth2_fork_predicate = move |enr: &Enr| {
+            enr.eth2() == Ok(enr_fork_id.clone()) && (enr.tcp().is_some() || enr.tcp6().is_some())
+        };
 
         // General predicate
         let predicate: Box<dyn Fn(&Enr) -> bool + Send> =
@@ -743,7 +748,7 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
                     }
                     Ok(r) => {
                         debug!(self.log, "Discovery query completed"; "peers_found" => r.len());
-                        let mut results: HashMap<PeerId, Option<Instant>> = HashMap::new();
+                        let mut results: HashMap<_, Option<Instant>> = HashMap::new();
                         r.iter().for_each(|enr| {
                             // cache the found ENR's
                             self.cached_enrs.put(enr.peer_id(), enr.clone());
@@ -766,7 +771,7 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
                     Ok(r) => {
                         debug!(self.log, "Peer grouped subnet discovery request completed"; "peers_found" => r.len(), "subnets_searched_for" => format!("{:?}",subnets_searched_for));
 
-                        let mut mapped_results: HashMap<PeerId, Option<Instant>> = HashMap::new();
+                        let mut mapped_results = HashMap::new();
 
                         // cache the found ENR's
                         for enr in r.iter().cloned() {

beacon_node/eth2_libp2p/src/lib.rs

@@ -7,6 +7,8 @@ extern crate lazy_static;
 
 pub mod behaviour;
 mod config;
+
+#[allow(clippy::mutable_key_type)] // PeerId in hashmaps are no longer permitted by clippy
 pub mod discovery;
 mod metrics;
 mod peer_manager;
@@ -64,6 +66,7 @@ pub use config::Config as NetworkConfig;
 pub use config::{GossipsubConfig, GossipsubConfigBuilder, GossipsubMessage};
 pub use discovery::{CombinedKeyExt, EnrExt, Eth2Enr};
 pub use discv5;
+pub use libp2p::bandwidth::BandwidthSinks;
 pub use libp2p::gossipsub::{MessageAcceptance, MessageId, Topic, TopicHash};
 pub use libp2p::{core::ConnectedPoint, PeerId, Swarm};
 pub use libp2p::{multiaddr, Multiaddr};

beacon_node/eth2_libp2p/src/peer_manager/mod.rs

@@ -27,6 +27,7 @@ pub use libp2p::core::{identity::Keypair, Multiaddr};
 pub mod client;
 mod peer_info;
 mod peer_sync_status;
+#[allow(clippy::mutable_key_type)] // PeerId in hashmaps are no longer permitted by clippy
 mod peerdb;
 pub(crate) mod score;
 
@@ -639,6 +640,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     /// with a new `PeerId` which involves a discovery routing table lookup. We could dial the
     /// multiaddr here, however this could relate to duplicate PeerId's etc. If the lookup
     /// proves resource constraining, we should switch to multiaddr dialling here.
+    #[allow(clippy::mutable_key_type)]
     fn peers_discovered(&mut self, results: HashMap<PeerId, Option<Instant>>) {
         let mut to_dial_peers = Vec::new();
 

beacon_node/eth2_libp2p/src/rpc/handler.rs

@@ -22,7 +22,8 @@ use std::{
     task::{Context, Poll},
     time::Duration,
 };
-use tokio::time::{delay_queue, delay_until, Delay, DelayQueue, Instant as TInstant};
+use tokio::time::{sleep_until, Instant as TInstant, Sleep};
+use tokio_util::time::{delay_queue, DelayQueue};
 use types::EthSpec;
 
 /// The time (in seconds) before a substream that is awaiting a response from the user times out.
@@ -132,7 +133,7 @@ enum HandlerState {
     ///
     /// While in this state the handler rejects new requests but tries to finish existing ones.
     /// Once the timer expires, all messages are killed.
-    ShuttingDown(Delay),
+    ShuttingDown(Sleep),
     /// The handler is deactivated. A goodbye has been sent and no more messages are sent or
     /// received.
     Deactivated,
@@ -255,7 +256,7 @@ where
                 self.dial_queue.push((id, req));
             }
 
-            self.state = HandlerState::ShuttingDown(delay_until(
+            self.state = HandlerState::ShuttingDown(sleep_until(
                 TInstant::now() + Duration::from_secs(SHUTDOWN_TIMEOUT_SECS as u64),
             ));
         }
@@ -540,7 +541,7 @@ where
 
         // purge expired inbound substreams and send an error
         loop {
-            match self.inbound_substreams_delay.poll_next_unpin(cx) {
+            match self.inbound_substreams_delay.poll_expired(cx) {
                 Poll::Ready(Some(Ok(inbound_id))) => {
                     // handle a stream timeout for various states
                     if let Some(info) = self.inbound_substreams.get_mut(inbound_id.get_ref()) {
@@ -574,7 +575,7 @@ where
 
         // purge expired outbound substreams
         loop {
-            match self.outbound_substreams_delay.poll_next_unpin(cx) {
+            match self.outbound_substreams_delay.poll_expired(cx) {
                 Poll::Ready(Some(Ok(outbound_id))) => {
                     if let Some(OutboundInfo { proto, req_id, .. }) =
                         self.outbound_substreams.remove(outbound_id.get_ref())
@@ -672,6 +673,7 @@ where
                                     if let Some(ref delay_key) = info.delay_key {
                                         self.inbound_substreams_delay.remove(delay_key);
                                     }
+                                    break;
                                 } else {
                                     // If we are not removing this substream, we reset the timer.
                                     // Each chunk is allowed RESPONSE_TIMEOUT to be sent.

beacon_node/eth2_libp2p/src/rpc/protocol.rs

@@ -503,8 +503,8 @@ impl From<ssz::DecodeError> for RPCError {
         RPCError::SSZDecodeError(err)
     }
 }
-impl From<tokio::time::Elapsed> for RPCError {
-    fn from(_: tokio::time::Elapsed) -> Self {
+impl From<tokio::time::error::Elapsed> for RPCError {
+    fn from(_: tokio::time::error::Elapsed) -> Self {
         RPCError::StreamTimeout
     }
 }

beacon_node/eth2_libp2p/src/service.rs

@@ -12,6 +12,7 @@ use libp2p::core::{
     identity::Keypair, multiaddr::Multiaddr, muxing::StreamMuxerBox, transport::Boxed,
 };
 use libp2p::{
+    bandwidth::{BandwidthLogging, BandwidthSinks},
     core, noise,
     swarm::{SwarmBuilder, SwarmEvent},
     PeerId, Swarm, Transport,
@@ -48,10 +49,10 @@ pub enum Libp2pEvent<TSpec: EthSpec> {
 pub struct Service<TSpec: EthSpec> {
     /// The libp2p Swarm handler.
     pub swarm: Swarm<Behaviour<TSpec>>,
-
+    /// The bandwidth logger for the underlying libp2p transport.
+    pub bandwidth: Arc<BandwidthSinks>,
     /// This node's PeerId.
     pub local_peer_id: PeerId,
-
     /// The libp2p logger handle.
     pub log: Logger,
 }
@@ -100,10 +101,11 @@ impl<TSpec: EthSpec> Service<TSpec> {
         };
         debug!(log, "Attempting to open listening ports"; "address" => format!("{}", config.listen_address), "tcp_port" => config.libp2p_port, "udp_port" => discovery_string);
 
-        let mut swarm = {
+        let (mut swarm, bandwidth) = {
             // Set up the transport - tcp/ws with noise and mplex
-            let transport = build_transport(local_keypair.clone())
+            let (transport, bandwidth) = build_transport(local_keypair.clone())
                 .map_err(|e| format!("Failed to build transport: {:?}", e))?;
+
             // Lighthouse network behaviour
             let behaviour = Behaviour::new(
                 &local_keypair,
@@ -121,14 +123,17 @@ impl<TSpec: EthSpec> Service<TSpec> {
                     self.0.spawn(f, "libp2p");
                 }
             }
-            SwarmBuilder::new(transport, behaviour, local_peer_id.clone())
-                .notify_handler_buffer_size(std::num::NonZeroUsize::new(32).expect("Not zero"))
-                .connection_event_buffer_size(64)
-                .incoming_connection_limit(10)
-                .outgoing_connection_limit(config.target_peers * 2)
-                .peer_connection_limit(MAX_CONNECTIONS_PER_PEER)
-                .executor(Box::new(Executor(executor)))
-                .build()
+            (
+                SwarmBuilder::new(transport, behaviour, local_peer_id.clone())
+                    .notify_handler_buffer_size(std::num::NonZeroUsize::new(32).expect("Not zero"))
+                    .connection_event_buffer_size(64)
+                    .incoming_connection_limit(10)
+                    .outgoing_connection_limit(config.target_peers * 2)
+                    .peer_connection_limit(MAX_CONNECTIONS_PER_PEER)
+                    .executor(Box::new(Executor(executor)))
+                    .build(),
+                bandwidth,
+            )
         };
 
         // listen on the specified address
@@ -221,6 +226,7 @@ impl<TSpec: EthSpec> Service<TSpec> {
 
         let service = Service {
             local_peer_id,
+            bandwidth,
             swarm,
             log,
         };
@@ -273,7 +279,7 @@ impl<TSpec: EthSpec> Service<TSpec> {
                     endpoint: _,
                     num_established,
                 } => {
-                    debug!(self.log, "Connection closed"; "peer_id"=> peer_id.to_string(), "cause" => format!("{:?}", cause), "connections" => num_established);
+                    trace!(self.log, "Connection closed"; "peer_id"=> peer_id.to_string(), "cause" => format!("{:?}", cause), "connections" => num_established);
                 }
                 SwarmEvent::NewListenAddr(multiaddr) => {
                     return Libp2pEvent::NewListenAddr(multiaddr)
@@ -282,7 +288,7 @@ impl<TSpec: EthSpec> Service<TSpec> {
                     local_addr,
                     send_back_addr,
                 } => {
-                    debug!(self.log, "Incoming connection"; "our_addr" => local_addr.to_string(), "from" => send_back_addr.to_string())
+                    trace!(self.log, "Incoming connection"; "our_addr" => local_addr.to_string(), "from" => send_back_addr.to_string())
                 }
                 SwarmEvent::IncomingConnectionError {
                     local_addr,
@@ -329,9 +335,13 @@ impl<TSpec: EthSpec> Service<TSpec> {
     }
 }
 
+type BoxedTransport = Boxed<(PeerId, StreamMuxerBox)>;
+
 /// The implementation supports TCP/IP, WebSockets over TCP/IP, noise as the encryption layer, and
 /// mplex as the multiplexing layer.
-fn build_transport(local_private_key: Keypair) -> std::io::Result<Boxed<(PeerId, StreamMuxerBox)>> {
+fn build_transport(
+    local_private_key: Keypair,
+) -> std::io::Result<(BoxedTransport, Arc<BandwidthSinks>)> {
     let transport = libp2p::tcp::TokioTcpConfig::new().nodelay(true);
     let transport = libp2p::dns::DnsConfig::new(transport)?;
     #[cfg(feature = "libp2p-websocket")]
@@ -340,21 +350,26 @@ fn build_transport(local_private_key: Keypair) -> std::io::Result<Boxed<(PeerId,
         transport.or_transport(libp2p::websocket::WsConfig::new(trans_clone))
     };
 
+    let (transport, bandwidth) = BandwidthLogging::new(transport);
+
     // mplex config
     let mut mplex_config = libp2p::mplex::MplexConfig::new();
-    mplex_config.max_buffer_len(256);
-    mplex_config.max_buffer_len_behaviour(libp2p::mplex::MaxBufferBehaviour::Block);
+    mplex_config.set_max_buffer_size(256);
+    mplex_config.set_max_buffer_behaviour(libp2p::mplex::MaxBufferBehaviour::Block);
 
     // Authentication
-    Ok(transport
-        .upgrade(core::upgrade::Version::V1)
-        .authenticate(generate_noise_config(&local_private_key))
-        .multiplex(core::upgrade::SelectUpgrade::new(
-            libp2p::yamux::Config::default(),
-            mplex_config,
-        ))
-        .timeout(Duration::from_secs(10))
-        .boxed())
+    Ok((
+        transport
+            .upgrade(core::upgrade::Version::V1)
+            .authenticate(generate_noise_config(&local_private_key))
+            .multiplex(core::upgrade::SelectUpgrade::new(
+                libp2p::yamux::YamuxConfig::default(),
+                mplex_config,
+            ))
+            .timeout(Duration::from_secs(10))
+            .boxed(),
+        bandwidth,
+    ))
 }
 
 // Useful helper functions for debugging. Currently not used in the client.

beacon_node/eth2_libp2p/tests/common/mod.rs

@@ -6,7 +6,9 @@ use eth2_libp2p::Service as LibP2PService;
 use eth2_libp2p::{GossipsubConfigBuilder, Libp2pEvent, NetworkConfig};
 use slog::{debug, error, o, Drain};
 use std::net::{TcpListener, UdpSocket};
+use std::sync::Weak;
 use std::time::Duration;
+use tokio::runtime::Runtime;
 use types::{ChainSpec, EnrForkId, MinimalEthSpec};
 
 type E = MinimalEthSpec;
@@ -91,19 +93,18 @@ pub fn build_config(port: u16, mut boot_nodes: Vec<Enr>) -> NetworkConfig {
     config
 }
 
-pub async fn build_libp2p_instance(boot_nodes: Vec<Enr>, log: slog::Logger) -> Libp2pInstance {
+pub async fn build_libp2p_instance(
+    rt: Weak<Runtime>,
+    boot_nodes: Vec<Enr>,
+    log: slog::Logger,
+) -> Libp2pInstance {
     let port = unused_port("tcp").unwrap();
     let config = build_config(port, boot_nodes);
     // launch libp2p service
 
     let (signal, exit) = exit_future::signal();
     let (shutdown_tx, _) = futures::channel::mpsc::channel(1);
-    let executor = task_executor::TaskExecutor::new(
-        tokio::runtime::Handle::current(),
-        exit,
-        log.clone(),
-        shutdown_tx,
-    );
+    let executor = task_executor::TaskExecutor::new(rt, exit, log.clone(), shutdown_tx);
     Libp2pInstance(
         LibP2PService::new(
             executor,
@@ -127,10 +128,14 @@ pub fn get_enr(node: &LibP2PService<E>) -> Enr {
 
 // Returns `n` libp2p peers in fully connected topology.
 #[allow(dead_code)]
-pub async fn build_full_mesh(log: slog::Logger, n: usize) -> Vec<Libp2pInstance> {
+pub async fn build_full_mesh(
+    rt: Weak<Runtime>,
+    log: slog::Logger,
+    n: usize,
+) -> Vec<Libp2pInstance> {
     let mut nodes = Vec::with_capacity(n);
     for _ in 0..n {
-        nodes.push(build_libp2p_instance(vec![], log.clone()).await);
+        nodes.push(build_libp2p_instance(rt.clone(), vec![], log.clone()).await);
     }
     let multiaddrs: Vec<Multiaddr> = nodes
         .iter()
@@ -153,12 +158,15 @@ pub async fn build_full_mesh(log: slog::Logger, n: usize) -> Vec<Libp2pInstance>
 // Constructs a pair of nodes with separate loggers. The sender dials the receiver.
 // This returns a (sender, receiver) pair.
 #[allow(dead_code)]
-pub async fn build_node_pair(log: &slog::Logger) -> (Libp2pInstance, Libp2pInstance) {
+pub async fn build_node_pair(
+    rt: Weak<Runtime>,
+    log: &slog::Logger,
+) -> (Libp2pInstance, Libp2pInstance) {
     let sender_log = log.new(o!("who" => "sender"));
     let receiver_log = log.new(o!("who" => "receiver"));
 
-    let mut sender = build_libp2p_instance(vec![], sender_log).await;
-    let mut receiver = build_libp2p_instance(vec![], receiver_log).await;
+    let mut sender = build_libp2p_instance(rt.clone(), vec![], sender_log).await;
+    let mut receiver = build_libp2p_instance(rt, vec![], receiver_log).await;
 
     let receiver_multiaddr = receiver.swarm.local_enr().multiaddr()[1].clone();
 
@@ -182,7 +190,7 @@ pub async fn build_node_pair(log: &slog::Logger) -> (Libp2pInstance, Libp2pInsta
 
     // wait for either both nodes to listen or a timeout
     tokio::select! {
-        _  = tokio::time::delay_for(Duration::from_millis(500)) => {}
+        _  = tokio::time::sleep(Duration::from_millis(500)) => {}
         _ = joined => {}
     }
 
@@ -197,10 +205,10 @@ pub async fn build_node_pair(log: &slog::Logger) -> (Libp2pInstance, Libp2pInsta
 
 // Returns `n` peers in a linear topology
 #[allow(dead_code)]
-pub async fn build_linear(log: slog::Logger, n: usize) -> Vec<Libp2pInstance> {
+pub async fn build_linear(rt: Weak<Runtime>, log: slog::Logger, n: usize) -> Vec<Libp2pInstance> {
     let mut nodes = Vec::with_capacity(n);
     for _ in 0..n {
-        nodes.push(build_libp2p_instance(vec![], log.clone()).await);
+        nodes.push(build_libp2p_instance(rt.clone(), vec![], log.clone()).await);
     }
 
     let multiaddrs: Vec<Multiaddr> = nodes

beacon_node/genesis/Cargo.toml

@@ -6,9 +6,10 @@ edition = "2018"
 
 [dev-dependencies]
 eth1_test_rig = { path = "../../testing/eth1_test_rig" }
+tokio-compat-02 = "0.1"
 
 [dependencies]
-futures = "0.3.5"
+futures = "0.3.7"
 types = { path = "../../consensus/types"}
 environment = { path = "../../lighthouse/environment"}
 eth1 = { path = "../eth1"}
@@ -18,7 +19,7 @@ merkle_proof = { path = "../../consensus/merkle_proof" }
 eth2_ssz = "0.1.2"
 eth2_hashing = "0.1.0"
 tree_hash = "0.1.1"
-tokio = { version = "0.2.22", features = ["full"] }
+tokio = { version = "0.3.2", features = ["full"] }
 parking_lot = "0.11.0"
 slog = "2.5.2"
 exit-future = "0.2.0"

beacon_node/genesis/src/eth1_genesis_service.rs

@@ -12,7 +12,7 @@ use std::sync::{
     Arc,
 };
 use std::time::Duration;
-use tokio::time::delay_for;
+use tokio::time::sleep;
 use types::{BeaconState, ChainSpec, Deposit, Eth1Data, EthSpec, Hash256};
 
 /// The number of blocks that are pulled per request whilst waiting for genesis.
@@ -151,7 +151,7 @@ impl Eth1GenesisService {
                         "valid_deposits" => eth1_service.get_raw_valid_signature_count(),
                     );
 
-                    delay_for(update_interval).await;
+                    sleep(update_interval).await;
 
                     continue;
                 }
@@ -231,9 +231,9 @@ impl Eth1GenesisService {
             // We assume that if we imported a large chunk of blocks then we're some distance from
             // the head and we should sync faster.
             if blocks_imported >= BLOCKS_PER_GENESIS_POLL {
-                delay_for(Duration::from_millis(50)).await;
+                sleep(Duration::from_millis(50)).await;
             } else {
-                delay_for(update_interval).await;
+                sleep(update_interval).await;
             }
         }
     }

beacon_node/genesis/tests/tests.rs

@@ -10,11 +10,12 @@ use futures::compat::Future01CompatExt;
 use genesis::{Eth1Config, Eth1GenesisService};
 use state_processing::is_valid_genesis_state;
 use std::time::Duration;
+use tokio_compat_02::FutureExt;
 use types::{test_utils::generate_deterministic_keypair, Hash256, MinimalEthSpec};
 
 pub fn new_env() -> Environment<MinimalEthSpec> {
     EnvironmentBuilder::minimal()
-        .single_thread_tokio_runtime()
+        .multi_threaded_tokio_runtime()
         .expect("should start tokio runtime")
         .null_logger()
         .expect("should start null logger")
@@ -28,83 +29,86 @@ fn basic() {
     let log = env.core_context().log().clone();
     let mut spec = env.eth2_config().spec.clone();
 
-    env.runtime().block_on(async {
-        let eth1 = GanacheEth1Instance::new(DEFAULT_NETWORK_ID.into(), DEFAULT_CHAIN_ID.into())
-            .await
-            .expect("should start eth1 environment");
-        let deposit_contract = &eth1.deposit_contract;
-        let web3 = eth1.web3();
+    env.runtime().block_on(
+        async {
+            let eth1 = GanacheEth1Instance::new(DEFAULT_NETWORK_ID.into(), DEFAULT_CHAIN_ID.into())
+                .await
+                .expect("should start eth1 environment");
+            let deposit_contract = &eth1.deposit_contract;
+            let web3 = eth1.web3();
 
-        let now = web3
-            .eth()
-            .block_number()
-            .compat()
-            .await
-            .map(|v| v.as_u64())
-            .expect("should get block number");
+            let now = web3
+                .eth()
+                .block_number()
+                .compat()
+                .await
+                .map(|v| v.as_u64())
+                .expect("should get block number");
 
-        let service = Eth1GenesisService::new(
-            Eth1Config {
-                endpoints: vec![eth1.endpoint()],
-                deposit_contract_address: deposit_contract.address(),
-                deposit_contract_deploy_block: now,
-                lowest_cached_block_number: now,
-                follow_distance: 0,
-                block_cache_truncation: None,
-                ..Eth1Config::default()
-            },
-            log,
-            spec.clone(),
-        );
+            let service = Eth1GenesisService::new(
+                Eth1Config {
+                    endpoints: vec![eth1.endpoint()],
+                    deposit_contract_address: deposit_contract.address(),
+                    deposit_contract_deploy_block: now,
+                    lowest_cached_block_number: now,
+                    follow_distance: 0,
+                    block_cache_truncation: None,
+                    ..Eth1Config::default()
+                },
+                log,
+                spec.clone(),
+            );
 
-        // NOTE: this test is sensitive to the response speed of the external web3 server. If
-        // you're experiencing failures, try increasing the update_interval.
-        let update_interval = Duration::from_millis(500);
+            // NOTE: this test is sensitive to the response speed of the external web3 server. If
+            // you're experiencing failures, try increasing the update_interval.
+            let update_interval = Duration::from_millis(500);
 
-        spec.min_genesis_time = 0;
-        spec.min_genesis_active_validator_count = 8;
+            spec.min_genesis_time = 0;
+            spec.min_genesis_active_validator_count = 8;
 
-        let deposits = (0..spec.min_genesis_active_validator_count + 2)
-            .map(|i| {
-                deposit_contract.deposit_helper::<MinimalEthSpec>(
-                    generate_deterministic_keypair(i as usize),
-                    Hash256::from_low_u64_le(i),
-                    32_000_000_000,
-                )
-            })
-            .map(|deposit| DelayThenDeposit {
-                delay: Duration::from_secs(0),
-                deposit,
-            })
-            .collect::<Vec<_>>();
+            let deposits = (0..spec.min_genesis_active_validator_count + 2)
+                .map(|i| {
+                    deposit_contract.deposit_helper::<MinimalEthSpec>(
+                        generate_deterministic_keypair(i as usize),
+                        Hash256::from_low_u64_le(i),
+                        32_000_000_000,
+                    )
+                })
+                .map(|deposit| DelayThenDeposit {
+                    delay: Duration::from_secs(0),
+                    deposit,
+                })
+                .collect::<Vec<_>>();
 
-        let deposit_future = deposit_contract.deposit_multiple(deposits);
+            let deposit_future = deposit_contract.deposit_multiple(deposits);
 
-        let wait_future =
-            service.wait_for_genesis_state::<MinimalEthSpec>(update_interval, spec.clone());
+            let wait_future =
+                service.wait_for_genesis_state::<MinimalEthSpec>(update_interval, spec.clone());
 
-        let state = futures::try_join!(deposit_future, wait_future)
-            .map(|(_, state)| state)
-            .expect("should finish waiting for genesis");
+            let state = futures::try_join!(deposit_future, wait_future)
+                .map(|(_, state)| state)
+                .expect("should finish waiting for genesis");
 
-        // Note: using ganache these deposits are 1-per-block, therefore we know there should only be
-        // the minimum number of validators.
-        assert_eq!(
-            state.validators.len(),
-            spec.min_genesis_active_validator_count as usize,
-            "should have expected validator count"
-        );
+            // Note: using ganache these deposits are 1-per-block, therefore we know there should only be
+            // the minimum number of validators.
+            assert_eq!(
+                state.validators.len(),
+                spec.min_genesis_active_validator_count as usize,
+                "should have expected validator count"
+            );
 
-        assert!(state.genesis_time > 0, "should have some genesis time");
+            assert!(state.genesis_time > 0, "should have some genesis time");
 
-        assert!(
-            is_valid_genesis_state(&state, &spec),
-            "should be valid genesis state"
-        );
+            assert!(
+                is_valid_genesis_state(&state, &spec),
+                "should be valid genesis state"
+            );
 
-        assert!(
-            is_valid_genesis_state(&state, &spec),
-            "should be valid genesis state"
-        );
-    });
+            assert!(
+                is_valid_genesis_state(&state, &spec),
+                "should be valid genesis state"
+            );
+        }
+        .compat(),
+    );
 }

beacon_node/http_api/Cargo.toml

@@ -5,9 +5,9 @@ authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2018"
 
 [dependencies]
-warp = { git = "https://github.com/paulhauner/warp", branch = "cors-wildcard" }
+warp = { git = "https://github.com/sigp/warp ", branch = "lighthouse" }
 serde = { version = "1.0.116", features = ["derive"] }
-tokio = { version = "0.2.22", features = ["macros"] }
+tokio = { version = "0.3.2", features = ["macros"] }
 parking_lot = "0.11.0"
 types = { path = "../../consensus/types" }
 hex = "0.4.2"
@@ -31,4 +31,5 @@ bs58 = "0.3.1"
 store = { path = "../store" }
 environment = { path = "../../lighthouse/environment" }
 tree_hash = "0.1.1"
-discv5 = { git = "https://github.com/sigp/discv5", rev = "fba7ceb5cfebd219ebbad6ffdb5d8c31dc8e4bc0", features = ["libp2p"] }
+discv5 = { git = "https://github.com/sigp/discv5", rev = "f117b3ca56fa3dca2317270434634ff7106d391a", features = ["libp2p"] }
+tokio-compat-02 = "0.1"

beacon_node/http_api/src/lib.rs

@@ -17,10 +17,7 @@ use beacon_chain::{
 };
 use beacon_proposer_cache::BeaconProposerCache;
 use block_id::BlockId;
-use eth2::{
-    types::{self as api_types, ValidatorId},
-    StatusCode,
-};
+use eth2::types::{self as api_types, ValidatorId};
 use eth2_libp2p::{types::SyncState, EnrExt, NetworkGlobals, PeerId, PubsubMessage};
 use lighthouse_version::version_with_platform;
 use network::NetworkMessage;
@@ -42,6 +39,7 @@ use types::{
     Hash256, ProposerSlashing, PublicKey, PublicKeyBytes, RelativeEpoch, SignedAggregateAndProof,
     SignedBeaconBlock, SignedVoluntaryExit, Slot, YamlConfig,
 };
+use warp::http::StatusCode;
 use warp::{http::Response, Filter};
 use warp_utils::task::{blocking_json_task, blocking_task};
 
@@ -2251,12 +2249,14 @@ pub fn serve<T: BeaconChainTypes>(
         .map(|reply| warp::reply::with_header(reply, "Server", &version_with_platform()))
         .with(cors_builder.build());
 
-    let (listening_socket, server) = warp::serve(routes).try_bind_with_graceful_shutdown(
-        SocketAddrV4::new(config.listen_addr, config.listen_port),
-        async {
-            shutdown.await;
-        },
-    )?;
+    let (listening_socket, server) = {
+        warp::serve(routes).try_bind_with_graceful_shutdown(
+            SocketAddrV4::new(config.listen_addr, config.listen_port),
+            async {
+                shutdown.await;
+            },
+        )?
+    };
 
     info!(
         log,

beacon_node/http_api/tests/tests.rs

@@ -7,6 +7,7 @@ use beacon_chain::{
 use discv5::enr::{CombinedKey, EnrBuilder};
 use environment::null_logger;
 use eth2::Error;
+use eth2::StatusCode;
 use eth2::{types::*, BeaconNodeHttpClient, Url};
 use eth2_libp2p::{
     rpc::methods::MetaData,
@@ -21,12 +22,12 @@ use std::net::Ipv4Addr;
 use std::sync::Arc;
 use tokio::sync::mpsc;
 use tokio::sync::oneshot;
+use tokio_compat_02::FutureExt;
 use tree_hash::TreeHash;
 use types::{
     test_utils::generate_deterministic_keypairs, AggregateSignature, BeaconState, BitList, Domain,
     EthSpec, Hash256, Keypair, MainnetEthSpec, RelativeEpoch, SelectionProof, SignedRoot, Slot,
 };
-use warp::http::StatusCode;
 
 type E = MainnetEthSpec;
 
@@ -1825,277 +1826,337 @@ impl ApiTester {
     }
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn beacon_get() {
+    async {
+        ApiTester::new()
+            .test_beacon_genesis()
+            .await
+            .test_beacon_states_root()
+            .await
+            .test_beacon_states_fork()
+            .await
+            .test_beacon_states_finality_checkpoints()
+            .await
+            .test_beacon_states_validators()
+            .await
+            .test_beacon_states_validator_balances()
+            .await
+            .test_beacon_states_committees()
+            .await
+            .test_beacon_states_validator_id()
+            .await
+            .test_beacon_headers_all_slots()
+            .await
+            .test_beacon_headers_all_parents()
+            .await
+            .test_beacon_headers_block_id()
+            .await
+            .test_beacon_blocks()
+            .await
+            .test_beacon_blocks_attestations()
+            .await
+            .test_beacon_blocks_root()
+            .await
+            .test_get_beacon_pool_attestations()
+            .await
+            .test_get_beacon_pool_attester_slashings()
+            .await
+            .test_get_beacon_pool_proposer_slashings()
+            .await
+            .test_get_beacon_pool_voluntary_exits()
+            .await;
+    }
+    .compat()
+    .await;
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn post_beacon_blocks_valid() {
     ApiTester::new()
-        .test_beacon_genesis()
-        .await
-        .test_beacon_states_root()
-        .await
-        .test_beacon_states_fork()
-        .await
-        .test_beacon_states_finality_checkpoints()
-        .await
-        .test_beacon_states_validators()
-        .await
-        .test_beacon_states_validator_balances()
-        .await
-        .test_beacon_states_committees()
-        .await
-        .test_beacon_states_validator_id()
-        .await
-        .test_beacon_headers_all_slots()
-        .await
-        .test_beacon_headers_all_parents()
-        .await
-        .test_beacon_headers_block_id()
-        .await
-        .test_beacon_blocks()
-        .await
-        .test_beacon_blocks_attestations()
-        .await
-        .test_beacon_blocks_root()
-        .await
-        .test_get_beacon_pool_attestations()
-        .await
-        .test_get_beacon_pool_attester_slashings()
-        .await
-        .test_get_beacon_pool_proposer_slashings()
-        .await
-        .test_get_beacon_pool_voluntary_exits()
+        .test_post_beacon_blocks_valid()
+        .compat()
         .await;
 }
 
-#[tokio::test(core_threads = 2)]
-async fn post_beacon_blocks_valid() {
-    ApiTester::new().test_post_beacon_blocks_valid().await;
-}
-
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn post_beacon_blocks_invalid() {
-    ApiTester::new().test_post_beacon_blocks_invalid().await;
+    ApiTester::new()
+        .test_post_beacon_blocks_invalid()
+        .compat()
+        .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn beacon_pools_post_attestations_valid() {
     ApiTester::new()
         .test_post_beacon_pool_attestations_valid()
+        .compat()
         .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn beacon_pools_post_attestations_invalid() {
     ApiTester::new()
         .test_post_beacon_pool_attestations_invalid()
+        .compat()
         .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn beacon_pools_post_attester_slashings_valid() {
     ApiTester::new()
         .test_post_beacon_pool_attester_slashings_valid()
+        .compat()
         .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn beacon_pools_post_attester_slashings_invalid() {
     ApiTester::new()
         .test_post_beacon_pool_attester_slashings_invalid()
+        .compat()
         .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn beacon_pools_post_proposer_slashings_valid() {
     ApiTester::new()
         .test_post_beacon_pool_proposer_slashings_valid()
+        .compat()
         .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn beacon_pools_post_proposer_slashings_invalid() {
     ApiTester::new()
         .test_post_beacon_pool_proposer_slashings_invalid()
+        .compat()
         .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn beacon_pools_post_voluntary_exits_valid() {
     ApiTester::new()
         .test_post_beacon_pool_voluntary_exits_valid()
+        .compat()
         .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn beacon_pools_post_voluntary_exits_invalid() {
     ApiTester::new()
         .test_post_beacon_pool_voluntary_exits_invalid()
+        .compat()
         .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn config_get() {
     ApiTester::new()
         .test_get_config_fork_schedule()
+        .compat()
         .await
         .test_get_config_spec()
+        .compat()
         .await
         .test_get_config_deposit_contract()
+        .compat()
         .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn debug_get() {
     ApiTester::new()
         .test_get_debug_beacon_states()
+        .compat()
         .await
         .test_get_debug_beacon_heads()
+        .compat()
         .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn node_get() {
     ApiTester::new()
         .test_get_node_version()
+        .compat()
         .await
         .test_get_node_syncing()
+        .compat()
         .await
         .test_get_node_identity()
+        .compat()
         .await
         .test_get_node_health()
+        .compat()
         .await
         .test_get_node_peers_by_id()
+        .compat()
         .await
         .test_get_node_peers()
+        .compat()
         .await
         .test_get_node_peer_count()
+        .compat()
         .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn get_validator_duties_attester() {
-    ApiTester::new().test_get_validator_duties_attester().await;
+    ApiTester::new()
+        .test_get_validator_duties_attester()
+        .compat()
+        .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn get_validator_duties_attester_with_skip_slots() {
     ApiTester::new()
         .skip_slots(E::slots_per_epoch() * 2)
         .test_get_validator_duties_attester()
+        .compat()
         .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn get_validator_duties_proposer() {
-    ApiTester::new().test_get_validator_duties_proposer().await;
+    ApiTester::new()
+        .test_get_validator_duties_proposer()
+        .compat()
+        .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn get_validator_duties_proposer_with_skip_slots() {
     ApiTester::new()
         .skip_slots(E::slots_per_epoch() * 2)
         .test_get_validator_duties_proposer()
+        .compat()
         .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn block_production() {
-    ApiTester::new().test_block_production().await;
+    ApiTester::new().test_block_production().compat().await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn block_production_with_skip_slots() {
     ApiTester::new()
         .skip_slots(E::slots_per_epoch() * 2)
         .test_block_production()
+        .compat()
         .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn get_validator_attestation_data() {
-    ApiTester::new().test_get_validator_attestation_data().await;
+    ApiTester::new()
+        .test_get_validator_attestation_data()
+        .compat()
+        .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn get_validator_attestation_data_with_skip_slots() {
     ApiTester::new()
         .skip_slots(E::slots_per_epoch() * 2)
         .test_get_validator_attestation_data()
+        .compat()
         .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn get_validator_aggregate_attestation() {
     ApiTester::new()
         .test_get_validator_aggregate_attestation()
+        .compat()
         .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn get_validator_aggregate_attestation_with_skip_slots() {
     ApiTester::new()
         .skip_slots(E::slots_per_epoch() * 2)
         .test_get_validator_aggregate_attestation()
+        .compat()
         .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn get_validator_aggregate_and_proofs_valid() {
     ApiTester::new()
         .test_get_validator_aggregate_and_proofs_valid()
+        .compat()
         .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn get_validator_aggregate_and_proofs_valid_with_skip_slots() {
     ApiTester::new()
         .skip_slots(E::slots_per_epoch() * 2)
         .test_get_validator_aggregate_and_proofs_valid()
+        .compat()
         .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn get_validator_aggregate_and_proofs_invalid() {
     ApiTester::new()
         .test_get_validator_aggregate_and_proofs_invalid()
+        .compat()
         .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn get_validator_aggregate_and_proofs_invalid_with_skip_slots() {
     ApiTester::new()
         .skip_slots(E::slots_per_epoch() * 2)
         .test_get_validator_aggregate_and_proofs_invalid()
+        .compat()
         .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn get_validator_beacon_committee_subscriptions() {
     ApiTester::new()
         .test_get_validator_beacon_committee_subscriptions()
+        .compat()
         .await;
 }
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn lighthouse_endpoints() {
     ApiTester::new()
         .test_get_lighthouse_health()
+        .compat()
         .await
         .test_get_lighthouse_syncing()
+        .compat()
         .await
         .test_get_lighthouse_proto_array()
+        .compat()
         .await
         .test_get_lighthouse_validator_inclusion()
+        .compat()
         .await
         .test_get_lighthouse_validator_inclusion_global()
+        .compat()
         .await
         .test_get_lighthouse_eth1_syncing()
+        .compat()
         .await
         .test_get_lighthouse_eth1_block_cache()
+        .compat()
         .await
         .test_get_lighthouse_eth1_deposit_cache()
+        .compat()
         .await
         .test_get_lighthouse_beacon_states_ssz()
+        .compat()
         .await
         .test_get_lighthouse_staking()
+        .compat()
         .await;
 }

beacon_node/http_metrics/Cargo.toml

@@ -8,7 +8,7 @@ edition = "2018"
 
 [dependencies]
 prometheus = "0.10.0"
-warp = { git = "https://github.com/paulhauner/warp", branch = "cors-wildcard" }
+warp = { git = "https://github.com/sigp/warp ", branch = "lighthouse" }
 serde = { version = "1.0.116", features = ["derive"] }
 slog = "2.5.2"
 beacon_chain = { path = "../beacon_chain" }
@@ -22,7 +22,8 @@ lighthouse_version = { path = "../../common/lighthouse_version" }
 warp_utils = { path = "../../common/warp_utils" }
 
 [dev-dependencies]
-tokio = { version = "0.2.22", features = ["sync"] }
+tokio = { version = "0.3.2", features = ["sync"] }
 reqwest = { version = "0.10.8", features = ["json"] }
 environment = { path = "../../lighthouse/environment" }
 types = { path = "../../consensus/types" }
+tokio-compat-02 = "0.1"

beacon_node/http_metrics/tests/tests.rs

@@ -5,42 +5,47 @@ use reqwest::StatusCode;
 use std::net::Ipv4Addr;
 use std::sync::Arc;
 use tokio::sync::oneshot;
+use tokio_compat_02::FutureExt;
 use types::MainnetEthSpec;
 
 type Context = http_metrics::Context<EphemeralHarnessType<MainnetEthSpec>>;
 
-#[tokio::test(core_threads = 2)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn returns_200_ok() {
-    let log = null_logger().unwrap();
+    async {
+        let log = null_logger().unwrap();
 
-    let context = Arc::new(Context {
-        config: Config {
-            enabled: true,
-            listen_addr: Ipv4Addr::new(127, 0, 0, 1),
-            listen_port: 0,
-            allow_origin: None,
-        },
-        chain: None,
-        db_path: None,
-        freezer_db_path: None,
-        log,
-    });
+        let context = Arc::new(Context {
+            config: Config {
+                enabled: true,
+                listen_addr: Ipv4Addr::new(127, 0, 0, 1),
+                listen_port: 0,
+                allow_origin: None,
+            },
+            chain: None,
+            db_path: None,
+            freezer_db_path: None,
+            log,
+        });
 
-    let ctx = context.clone();
-    let (_shutdown_tx, shutdown_rx) = oneshot::channel::<()>();
-    let server_shutdown = async {
-        // It's not really interesting why this triggered, just that it happened.
-        let _ = shutdown_rx.await;
-    };
-    let (listening_socket, server) = http_metrics::serve(ctx, server_shutdown).unwrap();
+        let ctx = context.clone();
+        let (_shutdown_tx, shutdown_rx) = oneshot::channel::<()>();
+        let server_shutdown = async {
+            // It's not really interesting why this triggered, just that it happened.
+            let _ = shutdown_rx.await;
+        };
+        let (listening_socket, server) = http_metrics::serve(ctx, server_shutdown).unwrap();
 
-    tokio::spawn(async { server.await });
+        tokio::spawn(async { server.await });
 
-    let url = format!(
-        "http://{}:{}/metrics",
-        listening_socket.ip(),
-        listening_socket.port()
-    );
+        let url = format!(
+            "http://{}:{}/metrics",
+            listening_socket.ip(),
+            listening_socket.port()
+        );
 
-    assert_eq!(reqwest::get(&url).await.unwrap().status(), StatusCode::OK);
+        assert_eq!(reqwest::get(&url).await.unwrap().status(), StatusCode::OK);
+    }
+    .compat()
+    .await
 }

beacon_node/network/Cargo.toml

@@ -11,6 +11,8 @@ lazy_static = "1.4.0"
 matches = "0.1.8"
 tempfile = "3.1.0"
 exit-future = "0.2.0"
+slog-term = "2.6.0"
+slog-async = "2.5.0"
 
 [dependencies]
 beacon_chain =  { path = "../beacon_chain" }
@@ -25,9 +27,9 @@ hex = "0.4.2"
 eth2_ssz = "0.1.2"
 eth2_ssz_types = { path =  "../../consensus/ssz_types" }
 tree_hash = "0.1.1"
-futures = "0.3.5"
+futures = "0.3.7"
 error-chain = "0.12.4"
-tokio = { version = "0.2.22", features = ["full"] }
+tokio = { version = "0.3.2", features = ["full"] }
 parking_lot = "0.11.0"
 smallvec = "1.4.2"
 rand = "0.7.3"

beacon_node/network/src/attestation_service/tests/mod.rs

@@ -156,7 +156,7 @@ mod tests {
 
         tokio::select! {
             _ = collect_stream_fut => {return events}
-            _ = tokio::time::delay_for(
+            _ = tokio::time::sleep(
             Duration::from_millis(SLOT_DURATION_MILLIS) * num_slots_before_timeout,
         ) => { return events; }
             }

beacon_node/network/src/beacon_processor/worker/gossip_methods.rs

@@ -233,6 +233,8 @@ impl<T: BeaconChainTypes> Worker<T> {
             | Err(e @ BlockError::BeaconChainError(_)) => {
                 debug!(self.log, "Could not verify block for gossip, ignoring the block";
                             "error" => e.to_string());
+                // Prevent recurring behaviour by penalizing the peer slightly.
+                self.penalize_peer(peer_id.clone(), PeerAction::HighToleranceError);
                 self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Ignore);
                 return;
             }
@@ -511,6 +513,12 @@ impl<T: BeaconChainTypes> Worker<T> {
                     "block" => %beacon_block_root,
                     "type" => ?attestation_type,
                 );
+
+                // Peers that are slow or not to spec can spam us with these messages draining our
+                // bandwidth. We therefore penalize these peers when they do this.
+                self.penalize_peer(peer_id.clone(), PeerAction::LowToleranceError);
+
+                // Do not propagate these messages.
                 self.propagate_validation_result(
                     message_id,
                     peer_id.clone(),
@@ -618,7 +626,12 @@ impl<T: BeaconChainTypes> Worker<T> {
                     "block" => %beacon_block_root,
                     "type" => ?attestation_type,
                 );
+                // We still penalize the peer slightly. We don't want this to be a recurring
+                // behaviour.
+                self.penalize_peer(peer_id.clone(), PeerAction::HighToleranceError);
+
                 self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Ignore);
+
                 return;
             }
             AttnError::PriorAttestationKnown { .. } => {
@@ -634,7 +647,12 @@ impl<T: BeaconChainTypes> Worker<T> {
                     "block" => %beacon_block_root,
                     "type" => ?attestation_type,
                 );
+                // We still penalize the peer slightly. We don't want this to be a recurring
+                // behaviour.
+                self.penalize_peer(peer_id.clone(), PeerAction::HighToleranceError);
+
                 self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Ignore);
+
                 return;
             }
             AttnError::ValidatorIndexTooHigh(_) => {
@@ -677,6 +695,10 @@ impl<T: BeaconChainTypes> Worker<T> {
                             "msg" => "UnknownBlockHash"
                         )
                     });
+                // We still penalize the peer slightly. We don't want this to be a recurring
+                // behaviour.
+                self.penalize_peer(peer_id.clone(), PeerAction::HighToleranceError);
+
                 self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Ignore);
                 return;
             }

beacon_node/network/src/lib.rs

@@ -3,15 +3,18 @@ extern crate lazy_static;
 
 /// This crate provides the network server for Lighthouse.
 pub mod error;
+#[allow(clippy::mutable_key_type)] // PeerId in hashmaps are no longer permitted by clippy
 pub mod service;
 
 mod attestation_service;
 mod beacon_processor;
+#[allow(clippy::mutable_key_type)] // PeerId in hashmaps are no longer permitted by clippy
 mod metrics;
 mod nat;
 mod persisted_dht;
 mod router;
 mod status;
+#[allow(clippy::mutable_key_type)] // PeerId in hashmaps are no longer permitted by clippy
 mod sync;
 
 pub use eth2_libp2p::NetworkConfig;

beacon_node/network/src/metrics.rs

@@ -1,5 +1,12 @@
 use beacon_chain::attestation_verification::Error as AttnError;
+use eth2_libp2p::PubsubMessage;
+use eth2_libp2p::{
+    types::GossipKind, BandwidthSinks, GossipTopic, Gossipsub, NetworkGlobals, TopicHash,
+};
+use fnv::FnvHashMap;
 pub use lighthouse_metrics::*;
+use std::{collections::HashMap, sync::Arc};
+use types::{subnet_id::subnet_id_to_string, EthSpec};
 
 lazy_static! {
 
@@ -404,6 +411,27 @@ lazy_static! {
         "gossipsub_attestation_error_beacon_chain_error",
         "Count of a specific error type (see metric name)"
     );
+
+    pub static ref INBOUND_LIBP2P_BYTES: Result<IntGauge> =
+        try_create_int_gauge("libp2p_inbound_bytes", "The inbound bandwidth over libp2p");
+
+    pub static ref OUTBOUND_LIBP2P_BYTES: Result<IntGauge> = try_create_int_gauge(
+        "libp2p_outbound_bytes",
+        "The outbound bandwidth over libp2p"
+    );
+    pub static ref TOTAL_LIBP2P_BANDWIDTH: Result<IntGauge> = try_create_int_gauge(
+        "libp2p_total_bandwidth",
+        "The total inbound/outbound bandwidth over libp2p"
+    );
+}
+
+pub fn update_bandwidth_metrics(bandwidth: Arc<BandwidthSinks>) {
+    set_gauge(&INBOUND_LIBP2P_BYTES, bandwidth.total_inbound() as i64);
+    set_gauge(&OUTBOUND_LIBP2P_BYTES, bandwidth.total_outbound() as i64);
+    set_gauge(
+        &TOTAL_LIBP2P_BANDWIDTH,
+        (bandwidth.total_inbound() + bandwidth.total_outbound()) as i64,
+    );
 }
 
 lazy_static! {
@@ -486,3 +514,359 @@ pub fn register_attestation_error(error: &AttnError) {
         AttnError::BeaconChainError(_) => inc_counter(&GOSSIP_ATTESTATION_ERROR_BEACON_CHAIN_ERROR),
     }
 }
+
+/// Inspects the `messages` that were being sent to the network and updates Prometheus metrics.
+pub fn expose_publish_metrics<T: EthSpec>(messages: &[PubsubMessage<T>]) {
+    for message in messages {
+        match message {
+            PubsubMessage::BeaconBlock(_) => inc_counter(&GOSSIP_BLOCKS_TX),
+            PubsubMessage::Attestation(subnet_id) => {
+                inc_counter_vec(
+                    &ATTESTATIONS_PUBLISHED_PER_SUBNET_PER_SLOT,
+                    &[&subnet_id.0.as_ref()],
+                );
+                inc_counter(&GOSSIP_UNAGGREGATED_ATTESTATIONS_TX)
+            }
+            PubsubMessage::AggregateAndProofAttestation(_) => {
+                inc_counter(&GOSSIP_AGGREGATED_ATTESTATIONS_TX)
+            }
+            _ => {}
+        }
+    }
+}
+
+/// Inspects a `message` received from the network and updates Prometheus metrics.
+pub fn expose_receive_metrics<T: EthSpec>(message: &PubsubMessage<T>) {
+    match message {
+        PubsubMessage::BeaconBlock(_) => inc_counter(&GOSSIP_BLOCKS_RX),
+        PubsubMessage::Attestation(_) => inc_counter(&GOSSIP_UNAGGREGATED_ATTESTATIONS_RX),
+        PubsubMessage::AggregateAndProofAttestation(_) => {
+            inc_counter(&GOSSIP_AGGREGATED_ATTESTATIONS_RX)
+        }
+        _ => {}
+    }
+}
+
+pub fn update_gossip_metrics<T: EthSpec>(
+    gossipsub: &Gossipsub,
+    network_globals: &Arc<NetworkGlobals<T>>,
+) {
+    // Clear the metrics
+    let _ = PEERS_PER_PROTOCOL.as_ref().map(|gauge| gauge.reset());
+    let _ = PEERS_PER_PROTOCOL.as_ref().map(|gauge| gauge.reset());
+    let _ = MESH_PEERS_PER_MAIN_TOPIC
+        .as_ref()
+        .map(|gauge| gauge.reset());
+    let _ = AVG_GOSSIPSUB_PEER_SCORE_PER_MAIN_TOPIC
+        .as_ref()
+        .map(|gauge| gauge.reset());
+    let _ = AVG_GOSSIPSUB_PEER_SCORE_PER_SUBNET_TOPIC
+        .as_ref()
+        .map(|gauge| gauge.reset());
+
+    let _ = SCORES_BELOW_ZERO_PER_CLIENT
+        .as_ref()
+        .map(|gauge| gauge.reset());
+    let _ = SCORES_BELOW_GOSSIP_THRESHOLD_PER_CLIENT
+        .as_ref()
+        .map(|gauge| gauge.reset());
+    let _ = SCORES_BELOW_PUBLISH_THRESHOLD_PER_CLIENT
+        .as_ref()
+        .map(|gauge| gauge.reset());
+    let _ = SCORES_BELOW_GREYLIST_THRESHOLD_PER_CLIENT
+        .as_ref()
+        .map(|gauge| gauge.reset());
+    let _ = MIN_SCORES_PER_CLIENT.as_ref().map(|gauge| gauge.reset());
+    let _ = MEDIAN_SCORES_PER_CLIENT.as_ref().map(|gauge| gauge.reset());
+    let _ = MEAN_SCORES_PER_CLIENT.as_ref().map(|gauge| gauge.reset());
+    let _ = MAX_SCORES_PER_CLIENT.as_ref().map(|gauge| gauge.reset());
+
+    let _ = BEACON_BLOCK_MESH_PEERS_PER_CLIENT
+        .as_ref()
+        .map(|gauge| gauge.reset());
+    let _ = BEACON_AGGREGATE_AND_PROOF_MESH_PEERS_PER_CLIENT
+        .as_ref()
+        .map(|gauge| gauge.reset());
+
+    // reset the mesh peers, showing all subnets
+    for subnet_id in 0..T::default_spec().attestation_subnet_count {
+        let _ = get_int_gauge(
+            &MESH_PEERS_PER_SUBNET_TOPIC,
+            &[subnet_id_to_string(subnet_id)],
+        )
+        .map(|v| v.set(0));
+
+        let _ = get_int_gauge(
+            &GOSSIPSUB_SUBSCRIBED_SUBNET_TOPIC,
+            &[subnet_id_to_string(subnet_id)],
+        )
+        .map(|v| v.set(0));
+
+        let _ = get_int_gauge(
+            &GOSSIPSUB_SUBSCRIBED_PEERS_SUBNET_TOPIC,
+            &[subnet_id_to_string(subnet_id)],
+        )
+        .map(|v| v.set(0));
+    }
+
+    // Subnet topics subscribed to
+    for topic_hash in gossipsub.topics() {
+        if let Ok(topic) = GossipTopic::decode(topic_hash.as_str()) {
+            if let GossipKind::Attestation(subnet_id) = topic.kind() {
+                let _ = get_int_gauge(
+                    &GOSSIPSUB_SUBSCRIBED_SUBNET_TOPIC,
+                    &[subnet_id_to_string(subnet_id.into())],
+                )
+                .map(|v| v.set(1));
+            }
+        }
+    }
+
+    // Peers per subscribed subnet
+    let mut peers_per_topic: HashMap<TopicHash, usize> = HashMap::new();
+    for (peer_id, topics) in gossipsub.all_peers() {
+        for topic_hash in topics {
+            *peers_per_topic.entry(topic_hash.clone()).or_default() += 1;
+
+            if let Ok(topic) = GossipTopic::decode(topic_hash.as_str()) {
+                match topic.kind() {
+                    GossipKind::Attestation(subnet_id) => {
+                        if let Some(v) = get_int_gauge(
+                            &GOSSIPSUB_SUBSCRIBED_PEERS_SUBNET_TOPIC,
+                            &[subnet_id_to_string(subnet_id.into())],
+                        ) {
+                            v.inc()
+                        };
+
+                        // average peer scores
+                        if let Some(score) = gossipsub.peer_score(peer_id) {
+                            if let Some(v) = get_gauge(
+                                &AVG_GOSSIPSUB_PEER_SCORE_PER_SUBNET_TOPIC,
+                                &[subnet_id_to_string(subnet_id.into())],
+                            ) {
+                                v.add(score)
+                            };
+                        }
+                    }
+                    kind => {
+                        // main topics
+                        if let Some(score) = gossipsub.peer_score(peer_id) {
+                            if let Some(v) = get_gauge(
+                                &AVG_GOSSIPSUB_PEER_SCORE_PER_MAIN_TOPIC,
+                                &[kind.as_ref()],
+                            ) {
+                                v.add(score)
+                            };
+                        }
+                    }
+                }
+            }
+        }
+    }
+    // adjust to average scores by dividing by number of peers
+    for (topic_hash, peers) in peers_per_topic.iter() {
+        if let Ok(topic) = GossipTopic::decode(topic_hash.as_str()) {
+            match topic.kind() {
+                GossipKind::Attestation(subnet_id) => {
+                    // average peer scores
+                    if let Some(v) = get_gauge(
+                        &AVG_GOSSIPSUB_PEER_SCORE_PER_SUBNET_TOPIC,
+                        &[subnet_id_to_string(subnet_id.into())],
+                    ) {
+                        v.set(v.get() / (*peers as f64))
+                    };
+                }
+                kind => {
+                    // main topics
+                    if let Some(v) =
+                        get_gauge(&AVG_GOSSIPSUB_PEER_SCORE_PER_MAIN_TOPIC, &[kind.as_ref()])
+                    {
+                        v.set(v.get() / (*peers as f64))
+                    };
+                }
+            }
+        }
+    }
+
+    // mesh peers
+    for topic_hash in gossipsub.topics() {
+        let peers = gossipsub.mesh_peers(&topic_hash).count();
+        if let Ok(topic) = GossipTopic::decode(topic_hash.as_str()) {
+            match topic.kind() {
+                GossipKind::Attestation(subnet_id) => {
+                    if let Some(v) = get_int_gauge(
+                        &MESH_PEERS_PER_SUBNET_TOPIC,
+                        &[subnet_id_to_string(subnet_id.into())],
+                    ) {
+                        v.set(peers as i64)
+                    };
+                }
+                kind => {
+                    // main topics
+                    if let Some(v) = get_int_gauge(&MESH_PEERS_PER_MAIN_TOPIC, &[kind.as_ref()]) {
+                        v.set(peers as i64)
+                    };
+                }
+            }
+        }
+    }
+
+    // protocol peers
+    let mut peers_per_protocol: HashMap<&'static str, i64> = HashMap::new();
+    for (_peer, protocol) in gossipsub.peer_protocol() {
+        *peers_per_protocol
+            .entry(protocol.as_static_ref())
+            .or_default() += 1;
+    }
+
+    for (protocol, peers) in peers_per_protocol.iter() {
+        if let Some(v) = get_int_gauge(&PEERS_PER_PROTOCOL, &[protocol]) {
+            v.set(*peers)
+        };
+    }
+
+    let mut peer_to_client = HashMap::new();
+    let mut scores_per_client: HashMap<&'static str, Vec<f64>> = HashMap::new();
+    {
+        let peers = network_globals.peers.read();
+        for (peer_id, _) in gossipsub.all_peers() {
+            let client = peers
+                .peer_info(peer_id)
+                .map(|peer_info| peer_info.client.kind.as_static_ref())
+                .unwrap_or_else(|| "Unknown");
+
+            peer_to_client.insert(peer_id, client);
+            let score = gossipsub.peer_score(peer_id).unwrap_or(0.0);
+            scores_per_client.entry(client).or_default().push(score);
+        }
+    }
+
+    // mesh peers per client
+    for topic_hash in gossipsub.topics() {
+        if let Ok(topic) = GossipTopic::decode(topic_hash.as_str()) {
+            match topic.kind() {
+                GossipKind::BeaconBlock => {
+                    for peer in gossipsub.mesh_peers(&topic_hash) {
+                        if let Some(client) = peer_to_client.get(peer) {
+                            if let Some(v) =
+                                get_int_gauge(&BEACON_BLOCK_MESH_PEERS_PER_CLIENT, &[client])
+                            {
+                                v.inc()
+                            };
+                        }
+                    }
+                }
+                GossipKind::BeaconAggregateAndProof => {
+                    for peer in gossipsub.mesh_peers(&topic_hash) {
+                        if let Some(client) = peer_to_client.get(peer) {
+                            if let Some(v) = get_int_gauge(
+                                &BEACON_AGGREGATE_AND_PROOF_MESH_PEERS_PER_CLIENT,
+                                &[client],
+                            ) {
+                                v.inc()
+                            };
+                        }
+                    }
+                }
+                _ => (),
+            }
+        }
+    }
+
+    for (client, scores) in scores_per_client.into_iter() {
+        let c = &[client];
+        let len = scores.len();
+        if len > 0 {
+            let mut below0 = 0;
+            let mut below_gossip_threshold = 0;
+            let mut below_publish_threshold = 0;
+            let mut below_greylist_threshold = 0;
+            let mut min = f64::INFINITY;
+            let mut sum = 0.0;
+            let mut max = f64::NEG_INFINITY;
+
+            let count = scores.len() as f64;
+
+            for &score in &scores {
+                if score < 0.0 {
+                    below0 += 1;
+                }
+                if score < -4000.0 {
+                    //TODO not hardcode
+                    below_gossip_threshold += 1;
+                }
+                if score < -8000.0 {
+                    //TODO not hardcode
+                    below_publish_threshold += 1;
+                }
+                if score < -16000.0 {
+                    //TODO not hardcode
+                    below_greylist_threshold += 1;
+                }
+                if score < min {
+                    min = score;
+                }
+                if score > max {
+                    max = score;
+                }
+                sum += score;
+            }
+
+            let median = if len == 0 {
+                0.0
+            } else if len % 2 == 0 {
+                (scores[len / 2 - 1] + scores[len / 2]) / 2.0
+            } else {
+                scores[len / 2]
+            };
+
+            set_gauge_entry(&SCORES_BELOW_ZERO_PER_CLIENT, c, below0 as f64 / count);
+            set_gauge_entry(
+                &SCORES_BELOW_GOSSIP_THRESHOLD_PER_CLIENT,
+                c,
+                below_gossip_threshold as f64 / count,
+            );
+            set_gauge_entry(
+                &SCORES_BELOW_PUBLISH_THRESHOLD_PER_CLIENT,
+                c,
+                below_publish_threshold as f64 / count,
+            );
+            set_gauge_entry(
+                &SCORES_BELOW_GREYLIST_THRESHOLD_PER_CLIENT,
+                c,
+                below_greylist_threshold as f64 / count,
+            );
+
+            set_gauge_entry(&MIN_SCORES_PER_CLIENT, c, min);
+            set_gauge_entry(&MEDIAN_SCORES_PER_CLIENT, c, median);
+            set_gauge_entry(&MEAN_SCORES_PER_CLIENT, c, sum / count);
+            set_gauge_entry(&MAX_SCORES_PER_CLIENT, c, max);
+        }
+    }
+}
+
+pub fn update_sync_metrics<T: EthSpec>(network_globals: &Arc<NetworkGlobals<T>>) {
+    // reset the counts
+    if PEERS_PER_SYNC_TYPE
+        .as_ref()
+        .map(|metric| metric.reset())
+        .is_err()
+    {
+        return;
+    };
+
+    // count per sync status, the number of connected peers
+    let mut peers_per_sync_type = FnvHashMap::default();
+    for sync_type in network_globals
+        .peers
+        .read()
+        .connected_peers()
+        .map(|(_peer_id, info)| info.sync_status.as_str())
+    {
+        *peers_per_sync_type.entry(sync_type).or_default() += 1;
+    }
+
+    for (sync_type, peer_count) in peers_per_sync_type {
+        set_gauge_entry(&PEERS_PER_SYNC_TYPE, &[sync_type], peer_count);
+    }
+}

beacon_node/network/src/nat.rs

@@ -5,7 +5,7 @@
 
 use crate::{NetworkConfig, NetworkMessage};
 use if_addrs::get_if_addrs;
-use slog::{debug, info, warn};
+use slog::{debug, info};
 use std::net::{IpAddr, SocketAddr, SocketAddrV4};
 use tokio::sync::mpsc;
 use types::EthSpec;
@@ -70,6 +70,8 @@ pub fn construct_upnp_mappings<T: EthSpec>(
                 Some(v) => v,
             };
 
+            debug!(log, "UPnP Local IP Discovered"; "ip" => ?local_ip);
+
             match local_ip {
                 IpAddr::V4(address) => {
                     let libp2p_socket = SocketAddrV4::new(address, config.tcp_port);
@@ -78,53 +80,39 @@ pub fn construct_upnp_mappings<T: EthSpec>(
                     // one.
                     // I've found this to be more reliable. If multiple users are behind a single
                     // router, they should ideally try to set different port numbers.
-                    let tcp_socket = match gateway.add_port(
+                    let tcp_socket = add_port_mapping(
+                        &gateway,
                         igd::PortMappingProtocol::TCP,
-                        libp2p_socket.port(),
                         libp2p_socket,
-                        0,
-                        "lighthouse-tcp",
-                    ) {
-                        Err(e) => {
-                            info!(log, "UPnP TCP route not set"; "error" => %e);
-                            None
-                        }
-                        Ok(_) => {
-                            info!(log, "UPnP TCP route established"; "external_socket" => format!("{}:{}", external_ip.as_ref().map(|ip| ip.to_string()).unwrap_or_else(|_| "".into()), config.tcp_port));
-                            external_ip
-                                .as_ref()
-                                .map(|ip| SocketAddr::new(ip.clone().into(), config.tcp_port))
-                                .ok()
-                        }
-                    };
+                        "tcp",
+                        &log,
+                    ).and_then(|_| {
+                        let external_socket = external_ip.as_ref().map(|ip| SocketAddr::new(ip.clone().into(), config.tcp_port)).map_err(|_| ());
+                        info!(log, "UPnP TCP route established"; "external_socket" => format!("{}:{}", external_socket.as_ref().map(|ip| ip.to_string()).unwrap_or_else(|_| "".into()), config.tcp_port));
+                        external_socket
+                    }).ok();
 
                     let udp_socket = if !config.disable_discovery {
                         let discovery_socket = SocketAddrV4::new(address, config.udp_port);
-                        match gateway.add_port(
+                        add_port_mapping(
+                            &gateway,
                             igd::PortMappingProtocol::UDP,
-                            discovery_socket.port(),
                             discovery_socket,
-                            0,
-                            "lighthouse-udp",
-                        ) {
-                            Err(e) => {
-                                info!(log, "UPnP UDP route not set"; "error" => %e);
-                                None
-                            }
-                            Ok(_) => {
-                                info!(log, "UPnP UDP route established"; "external_socket" => format!("{}:{}", external_ip.as_ref().map(|ip| ip.to_string()).unwrap_or_else(|_| "".into()), config.tcp_port));
-                                external_ip
-                                    .map(|ip| SocketAddr::new(ip.into(), config.tcp_port))
-                                    .ok()
-                            }
-                        }
+                            "udp",
+                            &log,
+                        ).and_then(|_| {
+                            let external_socket = external_ip
+                                    .map(|ip| SocketAddr::new(ip.into(), config.udp_port)).map_err(|_| ());
+                        info!(log, "UPnP UDP route established"; "external_socket" => format!("{}:{}", external_socket.as_ref().map(|ip| ip.to_string()).unwrap_or_else(|_| "".into()), config.udp_port));
+                        external_socket
+                    }).ok()
                     } else {
                         None
                     };
 
                     // report any updates to the network service.
                     network_send.send(NetworkMessage::UPnPMappingEstablished{ tcp_socket, udp_socket })
-            .unwrap_or_else(|e| warn!(log, "Could not send message to the network service"; "error" => %e));
+            .unwrap_or_else(|e| debug!(log, "Could not send message to the network service"; "error" => %e));
                 }
                 _ => debug!(log, "UPnP no routes constructed. IPv6 not supported"),
             }
@@ -132,6 +120,50 @@ pub fn construct_upnp_mappings<T: EthSpec>(
     };
 }
 
+/// Sets up a port mapping for a protocol returning the mapped port if successful.
+fn add_port_mapping(
+    gateway: &igd::Gateway,
+    protocol: igd::PortMappingProtocol,
+    socket: SocketAddrV4,
+    protocol_string: &'static str,
+    log: &slog::Logger,
+) -> Result<(), ()> {
+    // We add specific port mappings rather than getting the router to arbitrary assign
+    // one.
+    // I've found this to be more reliable. If multiple users are behind a single
+    // router, they should ideally try to set different port numbers.
+    let mapping_string = &format!("lighthouse-{}", protocol_string);
+    for _ in 0..2 {
+        match gateway.add_port(protocol, socket.port(), socket, 0, mapping_string) {
+            Err(e) => {
+                match e {
+                    igd::AddPortError::PortInUse => {
+                        // Try and remove and re-create
+                        debug!(log, "UPnP port in use, attempting to remap"; "protocol" => protocol_string, "port" => socket.port());
+                        match gateway.remove_port(protocol, socket.port()) {
+                            Ok(()) => {
+                                debug!(log, "UPnP Removed port mapping"; "protocol" => protocol_string,  "port" => socket.port())
+                            }
+                            Err(e) => {
+                                debug!(log, "UPnP Port remove failure"; "protocol" => protocol_string, "port" => socket.port(), "error" => %e);
+                                return Err(());
+                            }
+                        }
+                    }
+                    e => {
+                        info!(log, "UPnP TCP route not set"; "error" => %e);
+                        return Err(());
+                    }
+                }
+            }
+            Ok(_) => {
+                return Ok(());
+            }
+        }
+    }
+    Err(())
+}
+
 /// Removes the specified TCP and UDP port mappings.
 pub fn remove_mappings(tcp_port: Option<u16>, udp_port: Option<u16>, log: &slog::Logger) {
     if tcp_port.is_some() || udp_port.is_some() {

beacon_node/network/src/service.rs

@@ -8,20 +8,16 @@ use crate::{error, metrics};
 use beacon_chain::{BeaconChain, BeaconChainError, BeaconChainTypes};
 use eth2_libp2p::{
     rpc::{GoodbyeReason, RPCResponseErrorCode, RequestId},
-    Gossipsub, Libp2pEvent, PeerAction, PeerRequestId, PubsubMessage, Request, Response,
-};
-use eth2_libp2p::{
-    types::GossipKind, BehaviourEvent, GossipTopic, MessageId, NetworkGlobals, PeerId, TopicHash,
+    Libp2pEvent, PeerAction, PeerRequestId, PubsubMessage, Request, Response,
 };
+use eth2_libp2p::{types::GossipKind, BehaviourEvent, MessageId, NetworkGlobals, PeerId};
 use eth2_libp2p::{MessageAcceptance, Service as LibP2PService};
-use fnv::FnvHashMap;
 use futures::prelude::*;
 use slog::{debug, error, info, o, trace, warn};
-use std::{collections::HashMap, net::SocketAddr, sync::Arc, time::Duration};
+use std::{net::SocketAddr, sync::Arc, time::Duration};
 use store::HotColdDB;
 use tokio::sync::mpsc;
-use tokio::time::Delay;
-use types::subnet_id::subnet_id_to_string;
+use tokio::time::Sleep;
 use types::{EthSpec, RelativeEpoch, SubnetId, Unsigned, ValidatorSubscription};
 
 mod tests;
@@ -111,7 +107,7 @@ pub struct NetworkService<T: BeaconChainTypes> {
     /// update the UDP socket of discovery if the UPnP mappings get established.
     discovery_auto_update: bool,
     /// A delay that expires when a new fork takes place.
-    next_fork_update: Option<Delay>,
+    next_fork_update: Option<Sleep>,
     /// Subscribe to all the subnets once synced.
     subscribe_all_subnets: bool,
     /// A timer for updating various network metrics.
@@ -274,12 +270,12 @@ fn spawn_service<T: BeaconChainTypes>(
                             .as_ref()
                             .map(|gauge| gauge.reset());
                     }
-                    update_gossip_metrics::<T::EthSpec>(
+                    metrics::update_gossip_metrics::<T::EthSpec>(
                         &service.libp2p.swarm.gs(),
                         &service.network_globals,
                     );
                     // update sync metrics
-                    update_sync_metrics(&service.network_globals);
+                    metrics::update_sync_metrics(&service.network_globals);
 
                 }
                 _ = service.gossipsub_parameter_update.next() => {
@@ -382,7 +378,7 @@ fn spawn_service<T: BeaconChainTypes>(
                                     "count" => messages.len(),
                                     "topics" => format!("{:?}", topic_kinds)
                                 );
-                                expose_publish_metrics(&messages);
+                                metrics::expose_publish_metrics(&messages);
                                 service.libp2p.swarm.publish(messages);
                         }
                         NetworkMessage::ReportPeer { peer_id, action } => service.libp2p.report_peer(&peer_id, action),
@@ -512,7 +508,7 @@ fn spawn_service<T: BeaconChainTypes>(
                                 ..
                             } => {
                                 // Update prometheus metrics.
-                                expose_receive_metrics(&message);
+                                metrics::expose_receive_metrics(&message);
                                 match message {
                                     // attestation information gets processed in the attestation service
                                     PubsubMessage::Attestation(ref subnet_and_attestation) => {
@@ -566,399 +562,22 @@ fn spawn_service<T: BeaconChainTypes>(
                     service.next_fork_update = next_fork_delay(&service.beacon_chain);
                 }
             }
+
+            metrics::update_bandwidth_metrics(service.libp2p.bandwidth.clone());
         }
     }, "network");
 
     Ok(())
 }
 
-/// Returns a `Delay` that triggers shortly after the next change in the beacon chain fork version.
+/// Returns a `Sleep` that triggers shortly after the next change in the beacon chain fork version.
 /// If there is no scheduled fork, `None` is returned.
 fn next_fork_delay<T: BeaconChainTypes>(
     beacon_chain: &BeaconChain<T>,
-) -> Option<tokio::time::Delay> {
+) -> Option<tokio::time::Sleep> {
     beacon_chain.duration_to_next_fork().map(|until_fork| {
         // Add a short time-out to start within the new fork period.
         let delay = Duration::from_millis(200);
-        tokio::time::delay_until(tokio::time::Instant::now() + until_fork + delay)
+        tokio::time::sleep_until(tokio::time::Instant::now() + until_fork + delay)
     })
 }
-
-/// Inspects the `messages` that were being sent to the network and updates Prometheus metrics.
-fn expose_publish_metrics<T: EthSpec>(messages: &[PubsubMessage<T>]) {
-    for message in messages {
-        match message {
-            PubsubMessage::BeaconBlock(_) => metrics::inc_counter(&metrics::GOSSIP_BLOCKS_TX),
-            PubsubMessage::Attestation(subnet_id) => {
-                metrics::inc_counter_vec(
-                    &metrics::ATTESTATIONS_PUBLISHED_PER_SUBNET_PER_SLOT,
-                    &[&subnet_id.0.as_ref()],
-                );
-                metrics::inc_counter(&metrics::GOSSIP_UNAGGREGATED_ATTESTATIONS_TX)
-            }
-            PubsubMessage::AggregateAndProofAttestation(_) => {
-                metrics::inc_counter(&metrics::GOSSIP_AGGREGATED_ATTESTATIONS_TX)
-            }
-            _ => {}
-        }
-    }
-}
-
-/// Inspects a `message` received from the network and updates Prometheus metrics.
-fn expose_receive_metrics<T: EthSpec>(message: &PubsubMessage<T>) {
-    match message {
-        PubsubMessage::BeaconBlock(_) => metrics::inc_counter(&metrics::GOSSIP_BLOCKS_RX),
-        PubsubMessage::Attestation(_) => {
-            metrics::inc_counter(&metrics::GOSSIP_UNAGGREGATED_ATTESTATIONS_RX)
-        }
-        PubsubMessage::AggregateAndProofAttestation(_) => {
-            metrics::inc_counter(&metrics::GOSSIP_AGGREGATED_ATTESTATIONS_RX)
-        }
-        _ => {}
-    }
-}
-
-fn update_gossip_metrics<T: EthSpec>(
-    gossipsub: &Gossipsub,
-    network_globals: &Arc<NetworkGlobals<T>>,
-) {
-    // Clear the metrics
-    let _ = metrics::PEERS_PER_PROTOCOL
-        .as_ref()
-        .map(|gauge| gauge.reset());
-    let _ = metrics::PEERS_PER_PROTOCOL
-        .as_ref()
-        .map(|gauge| gauge.reset());
-    let _ = metrics::MESH_PEERS_PER_MAIN_TOPIC
-        .as_ref()
-        .map(|gauge| gauge.reset());
-    let _ = metrics::AVG_GOSSIPSUB_PEER_SCORE_PER_MAIN_TOPIC
-        .as_ref()
-        .map(|gauge| gauge.reset());
-    let _ = metrics::AVG_GOSSIPSUB_PEER_SCORE_PER_SUBNET_TOPIC
-        .as_ref()
-        .map(|gauge| gauge.reset());
-
-    let _ = metrics::SCORES_BELOW_ZERO_PER_CLIENT
-        .as_ref()
-        .map(|gauge| gauge.reset());
-    let _ = metrics::SCORES_BELOW_GOSSIP_THRESHOLD_PER_CLIENT
-        .as_ref()
-        .map(|gauge| gauge.reset());
-    let _ = metrics::SCORES_BELOW_PUBLISH_THRESHOLD_PER_CLIENT
-        .as_ref()
-        .map(|gauge| gauge.reset());
-    let _ = metrics::SCORES_BELOW_GREYLIST_THRESHOLD_PER_CLIENT
-        .as_ref()
-        .map(|gauge| gauge.reset());
-    let _ = metrics::MIN_SCORES_PER_CLIENT
-        .as_ref()
-        .map(|gauge| gauge.reset());
-    let _ = metrics::MEDIAN_SCORES_PER_CLIENT
-        .as_ref()
-        .map(|gauge| gauge.reset());
-    let _ = metrics::MEAN_SCORES_PER_CLIENT
-        .as_ref()
-        .map(|gauge| gauge.reset());
-    let _ = metrics::MAX_SCORES_PER_CLIENT
-        .as_ref()
-        .map(|gauge| gauge.reset());
-
-    let _ = metrics::BEACON_BLOCK_MESH_PEERS_PER_CLIENT
-        .as_ref()
-        .map(|gauge| gauge.reset());
-    let _ = metrics::BEACON_AGGREGATE_AND_PROOF_MESH_PEERS_PER_CLIENT
-        .as_ref()
-        .map(|gauge| gauge.reset());
-
-    // reset the mesh peers, showing all subnets
-    for subnet_id in 0..T::default_spec().attestation_subnet_count {
-        let _ = metrics::get_int_gauge(
-            &metrics::MESH_PEERS_PER_SUBNET_TOPIC,
-            &[subnet_id_to_string(subnet_id)],
-        )
-        .map(|v| v.set(0));
-
-        let _ = metrics::get_int_gauge(
-            &metrics::GOSSIPSUB_SUBSCRIBED_SUBNET_TOPIC,
-            &[subnet_id_to_string(subnet_id)],
-        )
-        .map(|v| v.set(0));
-
-        let _ = metrics::get_int_gauge(
-            &metrics::GOSSIPSUB_SUBSCRIBED_PEERS_SUBNET_TOPIC,
-            &[subnet_id_to_string(subnet_id)],
-        )
-        .map(|v| v.set(0));
-    }
-
-    // Subnet topics subscribed to
-    for topic_hash in gossipsub.topics() {
-        if let Ok(topic) = GossipTopic::decode(topic_hash.as_str()) {
-            if let GossipKind::Attestation(subnet_id) = topic.kind() {
-                let _ = metrics::get_int_gauge(
-                    &metrics::GOSSIPSUB_SUBSCRIBED_SUBNET_TOPIC,
-                    &[subnet_id_to_string(subnet_id.into())],
-                )
-                .map(|v| v.set(1));
-            }
-        }
-    }
-
-    // Peers per subscribed subnet
-    let mut peers_per_topic: HashMap<TopicHash, usize> = HashMap::new();
-    for (peer_id, topics) in gossipsub.all_peers() {
-        for topic_hash in topics {
-            *peers_per_topic.entry(topic_hash.clone()).or_default() += 1;
-
-            if let Ok(topic) = GossipTopic::decode(topic_hash.as_str()) {
-                match topic.kind() {
-                    GossipKind::Attestation(subnet_id) => {
-                        if let Some(v) = metrics::get_int_gauge(
-                            &metrics::GOSSIPSUB_SUBSCRIBED_PEERS_SUBNET_TOPIC,
-                            &[subnet_id_to_string(subnet_id.into())],
-                        ) {
-                            v.inc()
-                        };
-
-                        // average peer scores
-                        if let Some(score) = gossipsub.peer_score(peer_id) {
-                            if let Some(v) = metrics::get_gauge(
-                                &metrics::AVG_GOSSIPSUB_PEER_SCORE_PER_SUBNET_TOPIC,
-                                &[subnet_id_to_string(subnet_id.into())],
-                            ) {
-                                v.add(score)
-                            };
-                        }
-                    }
-                    kind => {
-                        // main topics
-                        if let Some(score) = gossipsub.peer_score(peer_id) {
-                            if let Some(v) = metrics::get_gauge(
-                                &metrics::AVG_GOSSIPSUB_PEER_SCORE_PER_MAIN_TOPIC,
-                                &[kind.as_ref()],
-                            ) {
-                                v.add(score)
-                            };
-                        }
-                    }
-                }
-            }
-        }
-    }
-    // adjust to average scores by dividing by number of peers
-    for (topic_hash, peers) in peers_per_topic.iter() {
-        if let Ok(topic) = GossipTopic::decode(topic_hash.as_str()) {
-            match topic.kind() {
-                GossipKind::Attestation(subnet_id) => {
-                    // average peer scores
-                    if let Some(v) = metrics::get_gauge(
-                        &metrics::AVG_GOSSIPSUB_PEER_SCORE_PER_SUBNET_TOPIC,
-                        &[subnet_id_to_string(subnet_id.into())],
-                    ) {
-                        v.set(v.get() / (*peers as f64))
-                    };
-                }
-                kind => {
-                    // main topics
-                    if let Some(v) = metrics::get_gauge(
-                        &metrics::AVG_GOSSIPSUB_PEER_SCORE_PER_MAIN_TOPIC,
-                        &[kind.as_ref()],
-                    ) {
-                        v.set(v.get() / (*peers as f64))
-                    };
-                }
-            }
-        }
-    }
-
-    // mesh peers
-    for topic_hash in gossipsub.topics() {
-        let peers = gossipsub.mesh_peers(&topic_hash).count();
-        if let Ok(topic) = GossipTopic::decode(topic_hash.as_str()) {
-            match topic.kind() {
-                GossipKind::Attestation(subnet_id) => {
-                    if let Some(v) = metrics::get_int_gauge(
-                        &metrics::MESH_PEERS_PER_SUBNET_TOPIC,
-                        &[subnet_id_to_string(subnet_id.into())],
-                    ) {
-                        v.set(peers as i64)
-                    };
-                }
-                kind => {
-                    // main topics
-                    if let Some(v) = metrics::get_int_gauge(
-                        &metrics::MESH_PEERS_PER_MAIN_TOPIC,
-                        &[kind.as_ref()],
-                    ) {
-                        v.set(peers as i64)
-                    };
-                }
-            }
-        }
-    }
-
-    // protocol peers
-    let mut peers_per_protocol: HashMap<&'static str, i64> = HashMap::new();
-    for (_peer, protocol) in gossipsub.peer_protocol() {
-        *peers_per_protocol
-            .entry(protocol.as_static_ref())
-            .or_default() += 1;
-    }
-
-    for (protocol, peers) in peers_per_protocol.iter() {
-        if let Some(v) = metrics::get_int_gauge(&metrics::PEERS_PER_PROTOCOL, &[protocol]) {
-            v.set(*peers)
-        };
-    }
-
-    let mut peer_to_client = HashMap::new();
-    let mut scores_per_client: HashMap<&'static str, Vec<f64>> = HashMap::new();
-    {
-        let peers = network_globals.peers.read();
-        for (peer_id, _) in gossipsub.all_peers() {
-            let client = peers
-                .peer_info(peer_id)
-                .map(|peer_info| peer_info.client.kind.as_static_ref())
-                .unwrap_or_else(|| "Unknown");
-
-            peer_to_client.insert(peer_id, client);
-            let score = gossipsub.peer_score(peer_id).unwrap_or(0.0);
-            scores_per_client.entry(client).or_default().push(score);
-        }
-    }
-
-    // mesh peers per client
-    for topic_hash in gossipsub.topics() {
-        if let Ok(topic) = GossipTopic::decode(topic_hash.as_str()) {
-            match topic.kind() {
-                GossipKind::BeaconBlock => {
-                    for peer in gossipsub.mesh_peers(&topic_hash) {
-                        if let Some(client) = peer_to_client.get(peer) {
-                            if let Some(v) = metrics::get_int_gauge(
-                                &metrics::BEACON_BLOCK_MESH_PEERS_PER_CLIENT,
-                                &[client],
-                            ) {
-                                v.inc()
-                            };
-                        }
-                    }
-                }
-                GossipKind::BeaconAggregateAndProof => {
-                    for peer in gossipsub.mesh_peers(&topic_hash) {
-                        if let Some(client) = peer_to_client.get(peer) {
-                            if let Some(v) = metrics::get_int_gauge(
-                                &metrics::BEACON_AGGREGATE_AND_PROOF_MESH_PEERS_PER_CLIENT,
-                                &[client],
-                            ) {
-                                v.inc()
-                            };
-                        }
-                    }
-                }
-                _ => (),
-            }
-        }
-    }
-
-    for (client, scores) in scores_per_client.into_iter() {
-        let c = &[client];
-        let len = scores.len();
-        if len > 0 {
-            let mut below0 = 0;
-            let mut below_gossip_threshold = 0;
-            let mut below_publish_threshold = 0;
-            let mut below_greylist_threshold = 0;
-            let mut min = f64::INFINITY;
-            let mut sum = 0.0;
-            let mut max = f64::NEG_INFINITY;
-
-            let count = scores.len() as f64;
-
-            for &score in &scores {
-                if score < 0.0 {
-                    below0 += 1;
-                }
-                if score < -4000.0 {
-                    //TODO not hardcode
-                    below_gossip_threshold += 1;
-                }
-                if score < -8000.0 {
-                    //TODO not hardcode
-                    below_publish_threshold += 1;
-                }
-                if score < -16000.0 {
-                    //TODO not hardcode
-                    below_greylist_threshold += 1;
-                }
-                if score < min {
-                    min = score;
-                }
-                if score > max {
-                    max = score;
-                }
-                sum += score;
-            }
-
-            let median = if len == 0 {
-                0.0
-            } else if len % 2 == 0 {
-                (scores[len / 2 - 1] + scores[len / 2]) / 2.0
-            } else {
-                scores[len / 2]
-            };
-
-            metrics::set_gauge_entry(
-                &metrics::SCORES_BELOW_ZERO_PER_CLIENT,
-                c,
-                below0 as f64 / count,
-            );
-            metrics::set_gauge_entry(
-                &metrics::SCORES_BELOW_GOSSIP_THRESHOLD_PER_CLIENT,
-                c,
-                below_gossip_threshold as f64 / count,
-            );
-            metrics::set_gauge_entry(
-                &metrics::SCORES_BELOW_PUBLISH_THRESHOLD_PER_CLIENT,
-                c,
-                below_publish_threshold as f64 / count,
-            );
-            metrics::set_gauge_entry(
-                &metrics::SCORES_BELOW_GREYLIST_THRESHOLD_PER_CLIENT,
-                c,
-                below_greylist_threshold as f64 / count,
-            );
-
-            metrics::set_gauge_entry(&metrics::MIN_SCORES_PER_CLIENT, c, min);
-            metrics::set_gauge_entry(&metrics::MEDIAN_SCORES_PER_CLIENT, c, median);
-            metrics::set_gauge_entry(&metrics::MEAN_SCORES_PER_CLIENT, c, sum / count);
-            metrics::set_gauge_entry(&metrics::MAX_SCORES_PER_CLIENT, c, max);
-        }
-    }
-}
-
-fn update_sync_metrics<T: EthSpec>(network_globals: &Arc<NetworkGlobals<T>>) {
-    // reset the counts
-    if metrics::PEERS_PER_SYNC_TYPE
-        .as_ref()
-        .map(|metric| metric.reset())
-        .is_err()
-    {
-        return;
-    };
-
-    // count per sync status, the number of connected peers
-    let mut peers_per_sync_type = FnvHashMap::default();
-    for sync_type in network_globals
-        .peers
-        .read()
-        .connected_peers()
-        .map(|(_peer_id, info)| info.sync_status.as_str())
-    {
-        *peers_per_sync_type.entry(sync_type).or_default() += 1;
-    }
-
-    for (sync_type, peer_count) in peers_per_sync_type {
-        metrics::set_gauge_entry(&metrics::PEERS_PER_SYNC_TYPE, &[sync_type], peer_count);
-    }
-}

beacon_node/network/src/service/tests.rs

@@ -5,6 +5,7 @@ mod tests {
     use crate::{NetworkConfig, NetworkService};
     use beacon_chain::test_utils::BeaconChainHarness;
     use eth2_libp2p::Enr;
+    //use slog::{o, Drain, Level, Logger};
     use slog::Logger;
     use sloggers::{null::NullLoggerBuilder, Build};
     use std::str::FromStr;
@@ -14,6 +15,18 @@ mod tests {
     use types::{test_utils::generate_deterministic_keypairs, MinimalEthSpec};
 
     fn get_logger() -> Logger {
+        /* For emitting logs during the tests
+        let drain = {
+            let decorator = slog_term::TermDecorator::new().build();
+            let decorator =
+                logging::AlignedTermDecorator::new(decorator, logging::MAX_MESSAGE_WIDTH);
+            let drain = slog_term::FullFormat::new(decorator).build().fuse();
+            let drain = slog_async::Async::new(drain).chan_size(2048).build();
+            drain.filter_level(Level::Debug)
+        };
+
+        Logger::root(drain.fuse(), o!())
+        */
         let builder = NullLoggerBuilder;
         builder.build().expect("should build logger")
     }
@@ -37,12 +50,12 @@ mod tests {
         let enr2 = Enr::from_str("enr:-IS4QJ2d11eu6dC7E7LoXeLMgMP3kom1u3SE8esFSWvaHoo0dP1jg8O3-nx9ht-EO3CmG7L6OkHcMmoIh00IYWB92QABgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQIB_c-jQMOXsbjWkbN-Oj99H57gfId5pfb4wa1qxwV4CIN1ZHCCIyk").unwrap();
         let enrs = vec![enr1, enr2];
 
-        let runtime = Runtime::new().unwrap();
+        let runtime = Arc::new(Runtime::new().unwrap());
 
         let (signal, exit) = exit_future::signal();
         let (shutdown_tx, _) = futures::channel::mpsc::channel(1);
         let executor = task_executor::TaskExecutor::new(
-            runtime.handle().clone(),
+            Arc::downgrade(&runtime),
             exit,
             log.clone(),
             shutdown_tx,
@@ -50,9 +63,10 @@ mod tests {
 
         let mut config = NetworkConfig::default();
         config.libp2p_port = 21212;
+        config.upnp_enabled = false;
         config.discovery_port = 21212;
         config.boot_nodes_enr = enrs.clone();
-        runtime.spawn(async move {
+        runtime.block_on(async move {
             // Create a new network service which implicitly gets dropped at the
             // end of the block.
 
@@ -61,7 +75,9 @@ mod tests {
                 .unwrap();
             drop(signal);
         });
-        runtime.shutdown_timeout(tokio::time::Duration::from_millis(300));
+
+        let raw_runtime = Arc::try_unwrap(runtime).unwrap();
+        raw_runtime.shutdown_timeout(tokio::time::Duration::from_secs(10));
 
         // Load the persisted dht from the store
         let persisted_enrs = load_dht(store);

beacon_node/src/lib.rs

@@ -72,10 +72,8 @@ impl<E: EthSpec> ProductionBeaconNode<E> {
         let client_genesis = client_config.genesis.clone();
         let store_config = client_config.store.clone();
         let log = context.log().clone();
-
         let db_path = client_config.create_db_path()?;
         let freezer_db_path_res = client_config.create_freezer_db_path();
-
         let executor = context.executor.clone();
 
         let builder = ClientBuilder::new(context.eth_spec_instance.clone())

beacon_node/tests/test.rs

@@ -1,11 +1,14 @@
 #![cfg(test)]
 
+//TODO: Drop compat library once reqwest and other libraries update to tokio 0.3
+
 use beacon_chain::StateSkipConfig;
 use node_test_rig::{
     environment::{Environment, EnvironmentBuilder},
     eth2::types::StateId,
     testing_client_config, LocalBeaconNode,
 };
+use tokio_compat_02::FutureExt;
 use types::{EthSpec, MinimalEthSpec, Slot};
 
 fn env_builder() -> EnvironmentBuilder<MinimalEthSpec> {
@@ -26,18 +29,26 @@ fn build_node<E: EthSpec>(env: &mut Environment<E>) -> LocalBeaconNode<E> {
 fn http_server_genesis_state() {
     let mut env = env_builder()
         .null_logger()
+        //.async_logger("debug", None)
         .expect("should build env logger")
         .multi_threaded_tokio_runtime()
         .expect("should start tokio runtime")
         .build()
         .expect("environment should build");
 
+    // build a runtime guard
+
     let node = build_node(&mut env);
+
     let remote_node = node.remote_node().expect("should produce remote node");
 
     let api_state = env
         .runtime()
-        .block_on(remote_node.get_debug_beacon_states(StateId::Slot(Slot::new(0))))
+        .block_on(
+            remote_node
+                .get_debug_beacon_states(StateId::Slot(Slot::new(0)))
+                .compat(),
+        )
         .expect("should fetch state from http api")
         .unwrap()
         .data;
@@ -54,5 +65,6 @@ fn http_server_genesis_state() {
         api_state, db_state,
         "genesis state from api should match that from the DB"
     );
+
     env.fire_signal();
 }

beacon_node/timer/Cargo.toml

@@ -8,8 +8,8 @@ edition = "2018"
 beacon_chain =  { path = "../beacon_chain" }
 types = { path = "../../consensus/types" }
 slot_clock = { path = "../../common/slot_clock" }
-tokio = { version = "0.2.22", features = ["full"] }
+tokio = { version = "0.3.2", features = ["full"] }
 slog = "2.5.2"
 parking_lot = "0.11.0"
-futures = "0.3.5"
+futures = "0.3.7"
 task_executor = { path = "../../common/task_executor" }

beacon_node/websocket_server/Cargo.toml

@@ -7,11 +7,11 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-futures = "0.3.5"
+futures = "0.3.7"
 serde = "1.0.116"
 serde_derive = "1.0.116"
 slog = "2.5.2"
-tokio = { version = "0.2.22", features = ["full"] }
+tokio = { version = "0.3.2", features = ["full"] }
 types = { path = "../../consensus/types" }
 ws = "0.9.1"
 task_executor = { path = "../../common/task_executor" }

beacon_node/websocket_server/src/lib.rs

@@ -79,11 +79,11 @@ pub fn start_server<T: EthSpec>(
 
     // Place a future on the handle that will shutdown the websocket server when the
     // application exits.
-    executor.runtime_handle().spawn(exit_future);
+
+    executor.spawn(exit_future, "Websocket exit");
 
     let log_inner = log.clone();
-
-    let _ = std::thread::spawn(move || match server.run() {
+    let server_future = move || match server.run() {
         Ok(_) => {
             debug!(
                 log_inner,
@@ -97,7 +97,9 @@ pub fn start_server<T: EthSpec>(
                 "error" => format!("{:?}", e)
             );
         }
-    });
+    };
+
+    executor.spawn_blocking(server_future, "Websocket server");
 
     info!(
         log,