Add DC/Shutdown capabilities to the behaviour handler (#1233)

* Remove ban event from the PM

* Fix dispatching of responses to peer's requests

* Disconnection logic

beacon_node/eth2-libp2p/src/behaviour/handler/delegate.rs

@@ -48,45 +48,20 @@ impl<TSpec: EthSpec> DelegatingHandler<TSpec> {
         }
     }
 
-    /// Gives access to the gossipsub handler.
-    pub fn _gossip_mut(&mut self) -> &mut GossipHandler {
-        &mut self.gossip_handler
-    }
-
     /// Gives mutable access to the rpc handler.
-    pub fn _rpc_mut(&mut self) -> &mut RPCHandler<TSpec> {
+    pub fn rpc_mut(&mut self) -> &mut RPCHandler<TSpec> {
         &mut self.rpc_handler
     }
 
-    /// Gives mutable access to identify's handler.
-    pub fn _identify_mut(&mut self) -> &mut IdentifyHandler {
-        &mut self.identify_handler
-    }
-
-    /// Gives mutable access to discovery's handler.
-    pub fn _discovery_mut(&mut self) -> &mut DiscoveryHandler<TSpec> {
-        &mut self.discovery_handler
-    }
-
-    /// Gives access to the gossipsub handler.
-    pub fn _gossip(&self) -> &GossipHandler {
-        &self.gossip_handler
-    }
-
     /// Gives access to the rpc handler.
-    pub fn _rpc(&self) -> &RPCHandler<TSpec> {
+    pub fn rpc(&self) -> &RPCHandler<TSpec> {
         &self.rpc_handler
     }
 
     /// Gives access to identify's handler.
-    pub fn _identify(&self) -> &IdentifyHandler {
+    pub fn identify(&self) -> &IdentifyHandler {
         &self.identify_handler
     }
-
-    /// Gives access to discovery's handler.
-    pub fn _discovery(&self) -> &DiscoveryHandler<TSpec> {
-        &self.discovery_handler
-    }
 }
 
 // TODO: this can all be created with macros

beacon_node/eth2-libp2p/src/behaviour/handler/mod.rs

@@ -22,8 +22,8 @@ mod delegate;
 pub struct BehaviourHandler<TSpec: EthSpec> {
     /// Handler combining all sub behaviour's handlers.
     delegate: DelegatingHandler<TSpec>,
-    /// Keep alive for this handler.
+    /// Flag indicating if the handler is shutting down.
-    keep_alive: KeepAlive,
+    shutting_down: bool,
 }
 
 impl<TSpec: EthSpec> BehaviourHandler<TSpec> {
@@ -35,7 +35,7 @@ impl<TSpec: EthSpec> BehaviourHandler<TSpec> {
     ) -> Self {
         BehaviourHandler {
             delegate: DelegatingHandler::new(gossipsub, rpc, identify, discovery),
-            keep_alive: KeepAlive::Yes,
+            shutting_down: false,
         }
     }
 }
@@ -43,8 +43,8 @@ impl<TSpec: EthSpec> BehaviourHandler<TSpec> {
 #[derive(Clone)]
 pub enum BehaviourHandlerIn<TSpec: EthSpec> {
     Delegate(DelegateIn<TSpec>),
-    // TODO: replace custom with incoming events
+    /// Start the shutdown process.
-    Custom,
+    Shutdown(Option<(RequestId, RPCRequest<TSpec>)>),
 }
 
 pub enum BehaviourHandlerOut<TSpec: EthSpec> {
@@ -84,8 +84,9 @@ impl<TSpec: EthSpec> ProtocolsHandler for BehaviourHandler<TSpec> {
         match event {
             BehaviourHandlerIn::Delegate(delegated_ev) => self.delegate.inject_event(delegated_ev),
             /* Events comming from the behaviour */
-            BehaviourHandlerIn::Custom => {
+            BehaviourHandlerIn::Shutdown(last_message) => {
-                // TODO: implement
+                self.shutting_down = true;
+                self.delegate.rpc_mut().shutdown(last_message);
             }
         }
     }
@@ -101,8 +102,13 @@ impl<TSpec: EthSpec> ProtocolsHandler for BehaviourHandler<TSpec> {
     }
 
     fn connection_keep_alive(&self) -> KeepAlive {
-        // TODO: refine this logic
+        if self.shutting_down {
-        self.keep_alive.min(self.delegate.connection_keep_alive())
+            let rpc_keep_alive = self.delegate.rpc().connection_keep_alive();
+            let identify_keep_alive = self.delegate.identify().connection_keep_alive();
+            rpc_keep_alive.max(identify_keep_alive)
+        } else {
+            KeepAlive::Yes
+        }
     }
 
     fn poll(
@@ -135,7 +141,5 @@ impl<TSpec: EthSpec> ProtocolsHandler for BehaviourHandler<TSpec> {
         }
 
         Poll::Pending
-
-        // TODO: speak to our behaviour here
     }
 }

beacon_node/eth2-libp2p/src/behaviour/mod.rs

@@ -15,7 +15,8 @@ use libp2p::{
     gossipsub::{Gossipsub, GossipsubEvent, MessageId},
     identify::{Identify, IdentifyEvent},
     swarm::{
-        NetworkBehaviour, NetworkBehaviourAction as NBAction, PollParameters, ProtocolsHandler,
+        NetworkBehaviour, NetworkBehaviourAction as NBAction, NotifyHandler, PollParameters,
+        ProtocolsHandler,
     },
     PeerId,
 };
@@ -51,7 +52,8 @@ pub struct Behaviour<TSpec: EthSpec> {
     peer_manager: PeerManager<TSpec>,
     /// The events generated by this behaviour to be consumed in the swarm poll.
     events: Vec<BehaviourEvent<TSpec>>,
-    // TODO: add events to send to the handler
+    /// Queue of peers to disconnect.
+    peers_to_dc: Vec<PeerId>,
     /// The current meta data of the node, so respond to pings and get metadata
     meta_data: MetaData<TSpec>,
     /// A cache of recently seen gossip messages. This is used to filter out any possible
@@ -285,6 +287,7 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             identify,
             peer_manager: PeerManager::new(network_globals.clone(), log),
             events: Vec::new(),
+            peers_to_dc: Vec::new(),
             seen_gossip_messages: LruCache::new(100_000),
             meta_data,
             network_globals,
@@ -396,36 +399,34 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
 
     /// Send a request to a peer over RPC.
     pub fn send_request(&mut self, peer_id: PeerId, request_id: RequestId, request: Request) {
-        self.send_rpc(peer_id, RPCSend::Request(request_id, request.into()))
+        self.eth2_rpc
+            .send_request(peer_id, request_id, request.into())
     }
 
     /// Send a successful response to a peer over RPC.
     pub fn send_successful_response(
         &mut self,
         peer_id: PeerId,
-        stream_id: SubstreamId,
+        id: PeerRequestId,
         response: Response<TSpec>,
     ) {
-        self.send_rpc(peer_id, RPCSend::Response(stream_id, response.into()))
+        self.eth2_rpc.send_response(peer_id, id, response.into())
     }
 
     /// Inform the peer that their request produced an error.
     pub fn _send_error_reponse(
         &mut self,
         peer_id: PeerId,
-        stream_id: SubstreamId,
+        id: PeerRequestId,
         error: RPCResponseErrorCode,
         reason: String,
     ) {
-        self.send_rpc(
+        self.eth2_rpc.send_response(
             peer_id,
-            RPCSend::Response(stream_id, RPCCodedResponse::from_error_code(error, reason)),
+            id,
+            RPCCodedResponse::from_error_code(error, reason),
         )
     }
-    /// Sends an RPC Request/Response via the RPC protocol.
-    fn send_rpc(&mut self, peer_id: PeerId, rpc_event: RPCSend<TSpec>) {
-        self.eth2_rpc.send_rpc(peer_id, rpc_event);
-    }
 
     /* Discovery / Peer management functions */
 
@@ -512,36 +513,32 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
             data: self.meta_data.seq_number,
         };
         debug!(self.log, "Sending Ping"; "request_id" => id, "peer_id" => peer_id.to_string());
-        let event = RPCSend::Request(id, RPCRequest::Ping(ping));
 
-        self.send_rpc(peer_id, event);
+        self.eth2_rpc
+            .send_request(peer_id, id, RPCRequest::Ping(ping));
     }
 
     /// Sends a Pong response to the peer.
-    fn pong(&mut self, id: SubstreamId, peer_id: PeerId) {
+    fn pong(&mut self, id: PeerRequestId, peer_id: PeerId) {
         let ping = crate::rpc::Ping {
             data: self.meta_data.seq_number,
         };
-        debug!(self.log, "Sending Pong"; "request_id" => id, "peer_id" => peer_id.to_string());
+        debug!(self.log, "Sending Pong"; "request_id" => id.1, "peer_id" => peer_id.to_string());
-        let event = RPCSend::Response(id, RPCCodedResponse::Success(RPCResponse::Pong(ping)));
+        let event = RPCCodedResponse::Success(RPCResponse::Pong(ping));
-
+        self.eth2_rpc.send_response(peer_id, id, event);
-        self.send_rpc(peer_id, event);
     }
 
     /// Sends a METADATA request to a peer.
     fn send_meta_data_request(&mut self, peer_id: PeerId) {
-        let metadata_request =
+        let event = RPCRequest::MetaData(PhantomData);
-            RPCSend::Request(RequestId::Behaviour, RPCRequest::MetaData(PhantomData));
+        self.eth2_rpc
-        self.send_rpc(peer_id, metadata_request);
+            .send_request(peer_id, RequestId::Behaviour, event);
     }
 
     /// Sends a METADATA response to a peer.
-    fn send_meta_data_response(&mut self, id: SubstreamId, peer_id: PeerId) {
+    fn send_meta_data_response(&mut self, id: PeerRequestId, peer_id: PeerId) {
-        let metadata_response = RPCSend::Response(
+        let event = RPCCodedResponse::Success(RPCResponse::MetaData(self.meta_data.clone()));
-            id,
+        self.eth2_rpc.send_response(peer_id, id, event);
-            RPCCodedResponse::Success(RPCResponse::MetaData(self.meta_data.clone())),
-        );
-        self.send_rpc(peer_id, metadata_response);
     }
 
     /// Returns a reference to the peer manager to allow the swarm to notify the manager of peer
@@ -635,7 +632,7 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
     }
 
     /// Convenience function to propagate a request.
-    fn propagate_request(&mut self, id: SubstreamId, peer_id: PeerId, request: Request) {
+    fn propagate_request(&mut self, id: PeerRequestId, peer_id: PeerId, request: Request) {
         self.events.push(BehaviourEvent::RequestReceived {
             peer_id,
             id,
@@ -645,6 +642,7 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
 
     fn on_rpc_event(&mut self, message: RPCMessage<TSpec>) {
         let peer_id = message.peer_id;
+        let handler_id = message.conn_id;
         // The METADATA and PING RPC responses are handled within the behaviour and not propagated
         match message.event {
             Err(handler_err) => {
@@ -654,6 +652,10 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
                         proto,
                         error,
                     } => {
+                        if matches!(error, RPCError::HandlerRejected) {
+                            // this peer's request got canceled
+                            // TODO: cancel processing for this request
+                        }
                         // Inform the peer manager of the error.
                         // An inbound error here means we sent an error to the peer, or the stream
                         // timed out.
@@ -670,37 +672,48 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
                     }
                 }
             }
-            Ok(RPCReceived::Request(id, request)) => match request {
+            Ok(RPCReceived::Request(id, request)) => {
+                let peer_request_id = (handler_id, id);
+                match request {
                     /* Behaviour managed protocols: Ping and Metadata */
                     RPCRequest::Ping(ping) => {
                         // inform the peer manager and send the response
                         self.peer_manager.ping_request(&peer_id, ping.data);
                         // send a ping response
-                    self.pong(id, peer_id);
+                        self.pong(peer_request_id, peer_id);
                     }
                     RPCRequest::MetaData(_) => {
                         // send the requested meta-data
-                    self.send_meta_data_response(id, peer_id);
+                        self.send_meta_data_response((handler_id, id), peer_id);
                         // TODO: inform the peer manager?
                     }
+                    RPCRequest::Goodbye(reason) => {
+                        // let the peer manager know this peer is in the process of disconnecting
+                        self.peer_manager._disconnecting_peer(&peer_id);
+                        // queue for disconnection without a goodbye message
+                        debug!(self.log, "Received a Goodbye, queueing for disconnection";
+                            "peer_id" => peer_id.to_string());
+                        self.peers_to_dc.push(peer_id.clone());
+                        // TODO: do not propagate
+                        self.propagate_request(peer_request_id, peer_id, Request::Goodbye(reason));
+                    }
                     /* Protocols propagated to the Network */
                     RPCRequest::Status(msg) => {
                         // inform the peer manager that we have received a status from a peer
                         self.peer_manager.peer_statusd(&peer_id);
                         // propagate the STATUS message upwards
-                    self.propagate_request(id, peer_id, Request::Status(msg))
+                        self.propagate_request(peer_request_id, peer_id, Request::Status(msg))
-                }
-                RPCRequest::BlocksByRange(req) => {
-                    self.propagate_request(id, peer_id, Request::BlocksByRange(req))
                     }
+                    RPCRequest::BlocksByRange(req) => self.propagate_request(
+                        peer_request_id,
+                        peer_id,
+                        Request::BlocksByRange(req),
+                    ),
                     RPCRequest::BlocksByRoot(req) => {
-                    self.propagate_request(id, peer_id, Request::BlocksByRoot(req))
+                        self.propagate_request(peer_request_id, peer_id, Request::BlocksByRoot(req))
+                    }
                 }
-                RPCRequest::Goodbye(reason) => {
-                    // TODO: do not propagate
-                    self.propagate_request(id, peer_id, Request::Goodbye(reason));
             }
-            },
             Ok(RPCReceived::Response(id, resp)) => {
                 match resp {
                     /* Behaviour managed protocols */
@@ -734,10 +747,19 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
     }
 
     /// Consumes the events list when polled.
-    fn custom_poll<TBehaviourIn>(
+    fn custom_poll(
         &mut self,
         cx: &mut Context,
-    ) -> Poll<NBAction<TBehaviourIn, BehaviourEvent<TSpec>>> {
+    ) -> Poll<NBAction<BehaviourHandlerIn<TSpec>, BehaviourEvent<TSpec>>> {
+        // handle pending disconnections to perform
+        if !self.peers_to_dc.is_empty() {
+            return Poll::Ready(NBAction::NotifyHandler {
+                peer_id: self.peers_to_dc.remove(0),
+                handler: NotifyHandler::All,
+                event: BehaviourHandlerIn::Shutdown(None),
+            });
+        }
+
         // check the peer manager for events
         loop {
             match self.peer_manager.poll_next_unpin(cx) {
@@ -756,11 +778,20 @@ impl<TSpec: EthSpec> Behaviour<TSpec> {
                     PeerManagerEvent::MetaData(peer_id) => {
                         self.send_meta_data_request(peer_id);
                     }
-                    PeerManagerEvent::_DisconnectPeer(_peer_id) => {
+                    PeerManagerEvent::DisconnectPeer(peer_id) => {
-                        //TODO: Implement
+                        debug!(self.log, "PeerManager requested to disconnect a peer";
-                    }
+                            "peer_id" => peer_id.to_string());
-                    PeerManagerEvent::_BanPeer(_peer_id) => {
+                        // queue for disabling
-                        //TODO: Implement
+                        self.peers_to_dc.push(peer_id.clone());
+                        // send one goodbye
+                        return Poll::Ready(NBAction::NotifyHandler {
+                            peer_id,
+                            handler: NotifyHandler::Any,
+                            event: BehaviourHandlerIn::Shutdown(Some((
+                                RequestId::Behaviour,
+                                RPCRequest::Goodbye(GoodbyeReason::Fault),
+                            ))),
+                        });
                     }
                 },
                 Poll::Pending => break,
@@ -872,6 +903,9 @@ impl<TSpec: EthSpec> std::convert::From<Response<TSpec>> for RPCCodedResponse<TS
     }
 }
 
+/// Identifier of requests sent by a peer.
+pub type PeerRequestId = (ConnectionId, SubstreamId);
+
 /// The types of events than can be obtained from polling the behaviour.
 #[derive(Debug)]
 pub enum BehaviourEvent<TSpec: EthSpec> {
@@ -888,7 +922,7 @@ pub enum BehaviourEvent<TSpec: EthSpec> {
         /// The peer that sent the request.
         peer_id: PeerId,
         /// Identifier of the request. All responses to this request must use this id.
-        id: SubstreamId,
+        id: PeerRequestId,
         /// Request the peer sent.
         request: Request,
     },

beacon_node/eth2-libp2p/src/lib.rs

@@ -15,7 +15,7 @@ mod service;
 pub mod types;
 
 pub use crate::types::{error, Enr, GossipTopic, NetworkGlobals, PubsubMessage};
-pub use behaviour::{BehaviourEvent, Request, Response};
+pub use behaviour::{BehaviourEvent, PeerRequestId, Request, Response};
 pub use config::Config as NetworkConfig;
 pub use discovery::enr_ext::{CombinedKeyExt, EnrExt};
 pub use libp2p::gossipsub::{MessageId, Topic, TopicHash};

beacon_node/eth2-libp2p/src/peer_manager/mod.rs

@@ -96,9 +96,7 @@ pub enum PeerManagerEvent {
     /// Request METADATA from a peer.
     MetaData(PeerId),
     /// The peer should be disconnected.
-    _DisconnectPeer(PeerId),
+    DisconnectPeer(PeerId),
-    /// The peer should be disconnected and banned.
-    _BanPeer(PeerId),
 }
 
 impl<TSpec: EthSpec> PeerManager<TSpec> {
@@ -234,6 +232,12 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
         self.connect_peer(peer_id, ConnectingType::Dialing)
     }
 
+    /// Updates the database informing that a peer is being disconnected.
+    pub fn _disconnecting_peer(&mut self, _peer_id: &PeerId) -> bool {
+        // TODO: implement
+        true
+    }
+
     /// Reports a peer for some action.
     ///
     /// If the peer doesn't exist, log a warning and insert defaults.
@@ -267,7 +271,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
                 // They closed early, this could mean poor connection
                 PeerAction::MidToleranceError
             }
-            RPCError::InternalError(_reason) => {
+            RPCError::InternalError(_) | RPCError::HandlerRejected => {
                 // Our fault. Do nothing
                 return;
             }
@@ -444,7 +448,8 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
         for id in ban_queue {
             pdb.ban(&id);
 
-            self.events.push(PeerManagerEvent::_BanPeer(id.clone()));
+            self.events
+                .push(PeerManagerEvent::DisconnectPeer(id.clone()));
         }
 
         for id in unban_queue {

beacon_node/eth2-libp2p/src/rpc/handler.rs

@@ -22,7 +22,7 @@ use std::{
     task::{Context, Poll},
     time::{Duration, Instant},
 };
-use tokio::time::{delay_queue, DelayQueue};
+use tokio::time::{delay_queue, delay_until, Delay, DelayQueue, Instant as TInstant};
 use types::EthSpec;
 
 //TODO: Implement check_timeout() on the substream types
@@ -33,6 +33,9 @@ pub const RESPONSE_TIMEOUT: u64 = 10;
 /// The number of times to retry an outbound upgrade in the case of IO errors.
 const IO_ERROR_RETRIES: u8 = 3;
 
+/// Maximum time given to the handler to perform shutdown operations.
+const SHUTDOWN_TIMEOUT_SECS: u8 = 15;
+
 /// Identifier of inbound and outbound substreams from the handler's perspective.
 #[derive(Debug, Clone, Copy, Hash, Eq, PartialEq)]
 pub struct SubstreamId(usize);
@@ -116,6 +119,9 @@ where
     /// Value to return from `connection_keep_alive`.
     keep_alive: KeepAlive,
 
+    /// State of the handler.
+    state: HandlerState,
+
     /// After the given duration has elapsed, an inactive connection will shutdown.
     inactive_timeout: Duration,
 
@@ -127,6 +133,18 @@ where
     log: slog::Logger,
 }
 
+enum HandlerState {
+    /// The handler is active. All messages are sent and received.
+    Active,
+    /// The handler is shutting_down.
+    ///
+    /// While in this state the handler rejects new requests but tries to finish existing ones.
+    ShuttingDown(Delay),
+    /// The handler is deactivated. A goodbye has been sent and no more messages are sent or
+    /// received.
+    Deactivated,
+}
+
 /// Contains the information the handler keeps on established outbound substreams.
 struct OutboundInfo<TSpec: EthSpec> {
     /// State of the substream.
@@ -278,6 +296,7 @@ where
             outbound_substreams_delay: DelayQueue::new(),
             current_inbound_substream_id: SubstreamId(0),
             current_outbound_substream_id: SubstreamId(0),
+            state: HandlerState::Active,
             max_dial_negotiated: 8,
             keep_alive: KeepAlive::Yes,
             inactive_timeout,
@@ -302,11 +321,171 @@ where
         &mut self.listen_protocol
     }
 
+    /// Initiates the handler's shutdown process, sending an optional last message to the peer.
+    pub fn shutdown(&mut self, final_msg: Option<(RequestId, RPCRequest<TSpec>)>) {
+        if matches!(self.state, HandlerState::Active) {
+            debug!(self.log, "Starting handler shutdown"; "unsent_queued_requests" => self.dial_queue.len());
+            // we now drive to completion communications already dialed/established
+            for (id, req) in self.dial_queue.pop() {
+                self.pending_errors.push(HandlerErr::Outbound {
+                    id,
+                    proto: req.protocol(),
+                    error: RPCError::HandlerRejected,
+                })
+            }
+
+            // Queue our final message, if any
+            if let Some((id, req)) = final_msg {
+                self.dial_queue.push((id, req));
+            }
+
+            self.state = HandlerState::ShuttingDown(delay_until(
+                TInstant::now() + Duration::from_secs(SHUTDOWN_TIMEOUT_SECS as u64),
+            ));
+        }
+        self.update_keep_alive();
+    }
+
     /// Opens an outbound substream with a request.
     fn send_request(&mut self, id: RequestId, req: RPCRequest<TSpec>) {
+        match self.state {
+            HandlerState::Active => {
                 self.dial_queue.push((id, req));
                 self.update_keep_alive();
             }
+            _ => {
+                self.pending_errors.push(HandlerErr::Outbound {
+                    id,
+                    proto: req.protocol(),
+                    error: RPCError::HandlerRejected,
+                });
+            }
+        }
+    }
+
+    /// Sends a response to a peer's request.
+    // NOTE: If the substream has closed due to inactivity, or the substream is in the
+    // wrong state a response will fail silently.
+    fn send_response(&mut self, inbound_id: SubstreamId, response: RPCCodedResponse<TSpec>) {
+        // Variables indicating if the response is an error response or a multi-part
+        // response
+        let res_is_error = response.is_error();
+        let res_is_multiple = response.multiple_responses();
+
+        // check if the stream matching the response still exists
+        let (substream_state, protocol) = match self.inbound_substreams.get_mut(&inbound_id) {
+            Some((substream_state, _, protocol)) => (substream_state, protocol),
+            None => {
+                warn!(self.log, "Stream has expired. Response not sent";
+                    "response" => response.to_string(), "id" => inbound_id);
+                return;
+            }
+        };
+
+        // If the response we are sending is an error, report back for handling
+        match response {
+            RPCCodedResponse::InvalidRequest(ref reason)
+            | RPCCodedResponse::ServerError(ref reason)
+            | RPCCodedResponse::Unknown(ref reason) => {
+                let code = &response
+                    .error_code()
+                    .expect("Error response should map to an error code");
+                let err = HandlerErr::Inbound {
+                    id: inbound_id,
+                    proto: *protocol,
+                    error: RPCError::ErrorResponse(*code, reason.clone()),
+                };
+                self.pending_errors.push(err);
+            }
+            _ => {} // not an error, continue.
+        }
+
+        if matches!(self.state, HandlerState::Deactivated) {
+            // we no longer send responses after the handler is deactivated
+            debug!(self.log, "Response not sent. Deactivated handler";
+                "response" => response.to_string(), "id" => inbound_id);
+            return;
+        }
+
+        match std::mem::replace(substream_state, InboundSubstreamState::Poisoned) {
+            InboundSubstreamState::ResponseIdle(substream) => {
+                // close the stream if there is no response
+                if let RPCCodedResponse::StreamTermination(_) = response {
+                    *substream_state = InboundSubstreamState::Closing(substream);
+                } else {
+                    // send the response
+                    // if it's a single rpc request or an error close the stream after.
+                    *substream_state = InboundSubstreamState::ResponsePendingSend {
+                        substream,
+                        message: response,
+                        closing: !res_is_multiple | res_is_error,
+                    }
+                }
+            }
+            InboundSubstreamState::ResponsePendingSend {
+                substream,
+                message,
+                closing,
+            } if res_is_multiple => {
+                // the stream is in use, add the request to a pending queue if active
+                self.queued_outbound_items
+                    .entry(inbound_id)
+                    .or_insert_with(Vec::new)
+                    .push(response);
+
+                // return the state
+                *substream_state = InboundSubstreamState::ResponsePendingSend {
+                    substream,
+                    message,
+                    closing,
+                };
+            }
+            InboundSubstreamState::ResponsePendingFlush { substream, closing }
+                if res_is_multiple =>
+            {
+                // the stream is in use, add the request to a pending queue
+                self.queued_outbound_items
+                    .entry(inbound_id)
+                    .or_insert_with(Vec::new)
+                    .push(response);
+
+                // return the state
+                *substream_state =
+                    InboundSubstreamState::ResponsePendingFlush { substream, closing };
+            }
+            InboundSubstreamState::Closing(substream) => {
+                *substream_state = InboundSubstreamState::Closing(substream);
+                debug!(self.log, "Response not sent. Stream is closing"; "response" => response.to_string());
+            }
+            InboundSubstreamState::ResponsePendingSend {
+                substream, message, ..
+            } => {
+                *substream_state = InboundSubstreamState::ResponsePendingSend {
+                    substream,
+                    message,
+                    closing: true,
+                };
+                error!(
+                    self.log,
+                    "Attempted sending multiple responses to a single response request"
+                );
+            }
+            InboundSubstreamState::ResponsePendingFlush { substream, .. } => {
+                *substream_state = InboundSubstreamState::ResponsePendingFlush {
+                    substream,
+                    closing: true,
+                };
+                error!(
+                    self.log,
+                    "Attempted sending multiple responses to a single response request"
+                );
+            }
+            InboundSubstreamState::Poisoned => {
+                crit!(self.log, "Poisoned inbound substream");
+                unreachable!("Coding error: Poisoned substream");
+            }
+        }
+    }
 
     /// Updates the `KeepAlive` returned by `connection_keep_alive`.
     ///
@@ -316,15 +495,25 @@ where
     fn update_keep_alive(&mut self) {
         // Check that we don't have outbound items pending for dialing, nor dialing, nor
         // established. Also check that there are no established inbound substreams.
+        // Errors and events need to be reported back, so check those too.
         let should_shutdown = self.dial_queue.is_empty()
-            && self.dial_negotiated == 0
             && self.outbound_substreams.is_empty()
-            && self.inbound_substreams.is_empty();
+            && self.inbound_substreams.is_empty()
+            && self.pending_errors.is_empty()
+            && self.events_out.is_empty()
+            && self.dial_negotiated == 0;
 
-        if should_shutdown {
+        match self.keep_alive {
-            self.keep_alive = KeepAlive::Until(Instant::now() + self.inactive_timeout)
+            KeepAlive::Yes if should_shutdown => {
-        } else {
+                self.keep_alive = KeepAlive::Until(Instant::now() + self.inactive_timeout);
-            self.keep_alive = KeepAlive::Yes
+            }
+            KeepAlive::Yes => {} // We continue being active
+            KeepAlive::Until(_) if should_shutdown => {} // Already deemed inactive
+            KeepAlive::Until(_) => {
+                // No longer idle
+                self.keep_alive = KeepAlive::Yes;
+            }
+            KeepAlive::No => {} // currently not used
         }
     }
 }
@@ -348,16 +537,16 @@ where
         &mut self,
         substream: <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output,
     ) {
-        let (req, substream) = substream;
+        // only accept new peer requests when active
-        // drop the stream
+        if !matches!(self.state, HandlerState::Active) {
-        if let RPCRequest::Goodbye(_) = req {
-            self.events_out
-                .push(RPCReceived::Request(self.current_inbound_substream_id, req));
-            self.current_inbound_substream_id.0 += 1;
             return;
         }
 
-        // New inbound request. Store the stream and tag the output.
+        let (req, substream) = substream;
+
+        // store requests that expect responses
+        if req.expected_responses() > 0 {
+            // Store the stream and tag the output.
             let delay_key = self.inbound_substreams_delay.insert(
                 self.current_inbound_substream_id,
                 Duration::from_secs(RESPONSE_TIMEOUT),
@@ -367,10 +556,13 @@ where
                 self.current_inbound_substream_id,
                 (awaiting_stream, Some(delay_key), req.protocol()),
             );
+        }
 
         self.events_out
             .push(RPCReceived::Request(self.current_inbound_substream_id, req));
         self.current_inbound_substream_id.0 += 1;
+
+        self.update_keep_alive();
     }
 
     fn inject_fully_negotiated_outbound(
@@ -379,9 +571,20 @@ where
         request_info: Self::OutboundOpenInfo,
     ) {
         self.dial_negotiated -= 1;
+        let (id, request) = request_info;
+        let proto = request.protocol();
+
+        // accept outbound connections only if the handler is not deactivated
+        if matches!(self.state, HandlerState::Deactivated) {
+            self.pending_errors.push(HandlerErr::Outbound {
+                id,
+                proto,
+                error: RPCError::HandlerRejected,
+            });
+            return;
+        }
 
         // add the stream to substreams if we expect a response, otherwise drop the stream.
-        let (id, request) = request_info;
         let expected_responses = request.expected_responses();
         if expected_responses > 0 {
             // new outbound request. Store the stream and tag the output.
@@ -389,7 +592,6 @@ where
                 self.current_outbound_substream_id,
                 Duration::from_secs(RESPONSE_TIMEOUT),
             );
-            let proto = request.protocol();
             let awaiting_stream = OutboundSubstreamState::RequestPendingResponse {
                 substream: out,
                 request,
@@ -422,128 +624,10 @@ where
         self.update_keep_alive();
     }
 
-    // NOTE: If the substream has closed due to inactivity, or the substream is in the
-    // wrong state a response will fail silently.
     fn inject_event(&mut self, rpc_event: Self::InEvent) {
         match rpc_event {
             RPCSend::Request(id, req) => self.send_request(id, req),
-            RPCSend::Response(inbound_id, response) => {
+            RPCSend::Response(inbound_id, response) => self.send_response(inbound_id, response),
-                // Variables indicating if the response is an error response or a multi-part
-                // response
-                let res_is_error = response.is_error();
-                let res_is_multiple = response.multiple_responses();
-
-                // check if the stream matching the response still exists
-                let (substream_state, protocol) = match self.inbound_substreams.get_mut(&inbound_id)
-                {
-                    Some((substream_state, _, protocol)) => (substream_state, protocol),
-                    None => {
-                        warn!(self.log, "Stream has expired. Response not sent";
-                            "response" => response.to_string(), "id" => inbound_id);
-                        return;
-                    }
-                };
-
-                // If the response we are sending is an error, report back for handling
-                match response {
-                    RPCCodedResponse::InvalidRequest(ref reason)
-                    | RPCCodedResponse::ServerError(ref reason)
-                    | RPCCodedResponse::Unknown(ref reason) => {
-                        let code = &response
-                            .error_code()
-                            .expect("Error response should map to an error code");
-                        let err = HandlerErr::Inbound {
-                            id: inbound_id,
-                            proto: *protocol,
-                            error: RPCError::ErrorResponse(*code, reason.clone()),
-                        };
-                        self.pending_errors.push(err);
-                    }
-                    _ => {} // not an error, continue.
-                }
-
-                match std::mem::replace(substream_state, InboundSubstreamState::Poisoned) {
-                    InboundSubstreamState::ResponseIdle(substream) => {
-                        // close the stream if there is no response
-                        match response {
-                            RPCCodedResponse::StreamTermination(_) => {
-                                *substream_state = InboundSubstreamState::Closing(substream);
-                            }
-                            _ => {
-                                // send the response
-                                // if it's a single rpc request or an error, close the stream after
-                                *substream_state = InboundSubstreamState::ResponsePendingSend {
-                                    substream,
-                                    message: response,
-                                    closing: !res_is_multiple | res_is_error, // close if an error or we are not expecting more responses
-                                };
-                            }
-                        }
-                    }
-                    InboundSubstreamState::ResponsePendingSend {
-                        substream,
-                        message,
-                        closing,
-                    } if res_is_multiple => {
-                        // the stream is in use, add the request to a pending queue
-                        self.queued_outbound_items
-                            .entry(inbound_id)
-                            .or_insert_with(Vec::new)
-                            .push(response);
-
-                        // return the state
-                        *substream_state = InboundSubstreamState::ResponsePendingSend {
-                            substream,
-                            message,
-                            closing,
-                        };
-                    }
-                    InboundSubstreamState::ResponsePendingFlush { substream, closing }
-                        if res_is_multiple =>
-                    {
-                        // the stream is in use, add the request to a pending queue
-                        self.queued_outbound_items
-                            .entry(inbound_id)
-                            .or_insert_with(Vec::new)
-                            .push(response);
-
-                        // return the state
-                        *substream_state =
-                            InboundSubstreamState::ResponsePendingFlush { substream, closing };
-                    }
-                    InboundSubstreamState::Closing(substream) => {
-                        *substream_state = InboundSubstreamState::Closing(substream);
-                        debug!(self.log, "Response not sent. Stream is closing"; "response" => format!("{}",response));
-                    }
-                    InboundSubstreamState::ResponsePendingSend {
-                        substream, message, ..
-                    } => {
-                        *substream_state = InboundSubstreamState::ResponsePendingSend {
-                            substream,
-                            message,
-                            closing: true,
-                        };
-                        error!(
-                            self.log,
-                            "Attempted sending multiple responses to a single response request"
-                        );
-                    }
-                    InboundSubstreamState::ResponsePendingFlush { substream, .. } => {
-                        *substream_state = InboundSubstreamState::ResponsePendingFlush {
-                            substream,
-                            closing: true,
-                        };
-                        error!(
-                            self.log,
-                            "Attempted sending multiple responses to a single response request"
-                        );
-                    }
-                    InboundSubstreamState::Poisoned => {
-                        crit!(self.log, "Poisoned inbound substream");
-                        unreachable!("Coding error: Poisoned substream");
-                    }
-                }
-            }
         }
     }
 
@@ -563,6 +647,10 @@ where
             }
         }
 
+        // This dialing is now considered failed
+        self.dial_negotiated -= 1;
+        self.update_keep_alive();
+
         self.outbound_io_error_retries = 0;
         // map the error
         let error = match error {
@@ -621,6 +709,21 @@ where
             self.events_out.shrink_to_fit();
         }
 
+        // Check if we are shutting down, and if the timer ran out
+        if let HandlerState::ShuttingDown(delay) = &self.state {
+            if delay.is_elapsed() {
+                self.state = HandlerState::Deactivated;
+                debug!(self.log, "Handler deactivated");
+                // Drain queued responses
+                for (inbound_id, queued_responses) in self.queued_outbound_items.drain() {
+                    for response in queued_responses {
+                        debug!(self.log, "Response not sent. Deactivated handler";
+                            "response" => response.to_string(), "id" => inbound_id);
+                    }
+                }
+            }
+        }
+
         // purge expired inbound substreams and send an error
         loop {
             match self.inbound_substreams_delay.poll_next_unpin(cx) {
@@ -686,6 +789,9 @@ where
             }
         }
 
+        // when deactivated, close all streams
+        let deactivated = matches!(self.state, HandlerState::Deactivated);
+
         // drive inbound streams that need to be processed
         for request_id in self.inbound_substreams.keys().copied().collect::<Vec<_>>() {
             // Drain all queued items until all messages have been processed for this stream
@@ -704,10 +810,25 @@ where
                                 message,
                                 closing,
                             } => {
+                                if deactivated {
+                                    if !closing {
+                                        // inform back to cancel this request's processing
+                                        self.pending_errors.push(HandlerErr::Inbound {
+                                            id: request_id,
+                                            proto: entry.get().2,
+                                            error: RPCError::HandlerRejected,
+                                        });
+                                    }
+                                    entry.get_mut().0 = InboundSubstreamState::Closing(substream);
+                                    drive_stream_further = true;
+                                } else {
                                     match Sink::poll_ready(Pin::new(&mut substream), cx) {
                                         Poll::Ready(Ok(())) => {
                                             // stream is ready to send data
-                                        match Sink::start_send(Pin::new(&mut substream), message) {
+                                            match Sink::start_send(
+                                                Pin::new(&mut substream),
+                                                message,
+                                            ) {
                                                 Ok(()) => {
                                                     // await flush
                                                     entry.get_mut().0 =
@@ -723,10 +844,11 @@ where
                                                     // keep connection with the peer and return the
                                                     // stream to awaiting response if this message
                                                     // wasn't closing the stream
-                                                // TODO: Duplicate code
                                                     if closing {
                                                         entry.get_mut().0 =
-                                                        InboundSubstreamState::Closing(substream);
+                                                            InboundSubstreamState::Closing(
+                                                                substream,
+                                                            );
                                                         drive_stream_further = true;
                                                     } else {
                                                         // check for queued chunks and update the stream
@@ -742,7 +864,11 @@ where
                                             }
                                         }
                                         Poll::Ready(Err(e)) => {
-                                        error!(self.log, "Outbound substream error while sending RPC message: {:?}", e);
+                                            error!(
+                                                self.log,
+                                                "Outbound substream error while sending RPC message: {:?}",
+                                                e
+                                            );
                                             entry.remove();
                                             self.update_keep_alive();
                                             return Poll::Ready(ProtocolsHandlerEvent::Close(e));
@@ -758,6 +884,7 @@ where
                                         }
                                     }
                                 }
+                            }
                             InboundSubstreamState::ResponsePendingFlush {
                                 mut substream,
                                 closing,
@@ -766,7 +893,15 @@ where
                                     Poll::Ready(Ok(())) => {
                                         // finished flushing
                                         // TODO: Duplicate code
-                                        if closing {
+                                        if closing | deactivated {
+                                            if !closing {
+                                                // inform back to cancel this request's processing
+                                                self.pending_errors.push(HandlerErr::Inbound {
+                                                    id: request_id,
+                                                    proto: entry.get().2,
+                                                    error: RPCError::HandlerRejected,
+                                                });
+                                            }
                                             entry.get_mut().0 =
                                                 InboundSubstreamState::Closing(substream);
                                             drive_stream_further = true;
@@ -805,11 +940,16 @@ where
                                 }
                             }
                             InboundSubstreamState::ResponseIdle(substream) => {
+                                if !deactivated {
                                     entry.get_mut().0 = apply_queued_responses(
                                         substream,
                                         &mut self.queued_outbound_items.get_mut(&request_id),
                                         &mut drive_stream_further,
                                     );
+                                } else {
+                                    entry.get_mut().0 = InboundSubstreamState::Closing(substream);
+                                    drive_stream_further = true;
+                                }
                             }
                             InboundSubstreamState::Closing(mut substream) => {
                                 match Sink::poll_close(Pin::new(&mut substream), cx) {
@@ -866,6 +1006,18 @@ where
             };
 
             match state {
+                OutboundSubstreamState::RequestPendingResponse {
+                    substream,
+                    request: _,
+                } if deactivated => {
+                    // the handler is deactivated. Close the stream
+                    entry.get_mut().state = OutboundSubstreamState::Closing(substream);
+                    self.pending_errors.push(HandlerErr::Outbound {
+                        id: entry.get().req_id,
+                        proto: entry.get().proto,
+                        error: RPCError::HandlerRejected,
+                    })
+                }
                 OutboundSubstreamState::RequestPendingResponse {
                     mut substream,
                     request,

beacon_node/eth2-libp2p/src/rpc/mod.rs

@@ -80,6 +80,8 @@ impl<T: EthSpec> std::fmt::Display for RPCSend<T> {
 pub struct RPCMessage<TSpec: EthSpec> {
     /// The peer that sent the message.
     pub peer_id: PeerId,
+    /// Handler managing this message.
+    pub conn_id: ConnectionId,
     /// The message that was sent.
     pub event: <RPCHandler<TSpec> as ProtocolsHandler>::OutEvent,
 }
@@ -102,14 +104,35 @@ impl<TSpec: EthSpec> RPC<TSpec> {
         }
     }
 
+    /// Sends an RPC response.
+    ///
+    /// The peer must be connected for this to succeed.
+    pub fn send_response(
+        &mut self,
+        peer_id: PeerId,
+        id: (ConnectionId, SubstreamId),
+        event: RPCCodedResponse<TSpec>,
+    ) {
+        self.events.push(NetworkBehaviourAction::NotifyHandler {
+            peer_id,
+            handler: NotifyHandler::One(id.0),
+            event: RPCSend::Response(id.1, event),
+        });
+    }
+
     /// Submits an RPC request.
     ///
     /// The peer must be connected for this to succeed.
-    pub fn send_rpc(&mut self, peer_id: PeerId, event: RPCSend<TSpec>) {
+    pub fn send_request(
+        &mut self,
+        peer_id: PeerId,
+        request_id: RequestId,
+        event: RPCRequest<TSpec>,
+    ) {
         self.events.push(NetworkBehaviourAction::NotifyHandler {
             peer_id,
             handler: NotifyHandler::Any,
-            event,
+            event: RPCSend::Request(request_id, event),
         });
     }
 }
@@ -126,7 +149,7 @@ where
             SubstreamProtocol::new(RPCProtocol {
                 phantom: PhantomData,
             }),
-            Duration::from_secs(5),
+            Duration::from_secs(30),
             &self.log,
         )
     }
@@ -169,13 +192,14 @@ where
     fn inject_event(
         &mut self,
         peer_id: PeerId,
-        _: ConnectionId,
+        conn_id: ConnectionId,
         event: <Self::ProtocolsHandler as ProtocolsHandler>::OutEvent,
     ) {
         // send the event to the user
         self.events
             .push(NetworkBehaviourAction::GenerateEvent(RPCMessage {
                 peer_id,
+                conn_id,
                 event,
             }));
     }

beacon_node/eth2-libp2p/src/rpc/protocol.rs

@@ -388,8 +388,10 @@ pub enum RPCError {
     InvalidData,
     /// An error occurred due to internal reasons. Ex: timer failure.
     InternalError(&'static str),
-    /// Negotiation with this peer timed out
+    /// Negotiation with this peer timed out.
     NegotiationTimeout,
+    /// Handler rejected this request.
+    HandlerRejected,
 }
 
 impl From<ssz::DecodeError> for RPCError {
@@ -427,6 +429,7 @@ impl std::fmt::Display for RPCError {
             RPCError::IncompleteStream => write!(f, "Stream ended unexpectedly"),
             RPCError::InternalError(ref err) => write!(f, "Internal error: {}", err),
             RPCError::NegotiationTimeout => write!(f, "Negotiation timeout"),
+            RPCError::HandlerRejected => write!(f, "Handler rejected the request"),
         }
     }
 }
@@ -444,6 +447,7 @@ impl std::error::Error for RPCError {
             RPCError::InternalError(_) => None,
             RPCError::ErrorResponse(_, _) => None,
             RPCError::NegotiationTimeout => None,
+            RPCError::HandlerRejected => None,
         }
     }
 }

beacon_node/eth2-libp2p/src/service.rs

@@ -1,7 +1,7 @@
-use crate::behaviour::{Behaviour, BehaviourEvent, Request, Response};
+use crate::behaviour::{Behaviour, BehaviourEvent, PeerRequestId, Request, Response};
 use crate::discovery::enr;
 use crate::multiaddr::Protocol;
-use crate::rpc::{RPCResponseErrorCode, RequestId, SubstreamId};
+use crate::rpc::{RPCResponseErrorCode, RequestId};
 use crate::types::{error, GossipKind};
 use crate::EnrExt;
 use crate::{NetworkConfig, NetworkGlobals};
@@ -239,23 +239,16 @@ impl<TSpec: EthSpec> Service<TSpec> {
     pub fn respond_with_error(
         &mut self,
         peer_id: PeerId,
-        stream_id: SubstreamId,
+        id: PeerRequestId,
         error: RPCResponseErrorCode,
         reason: String,
     ) {
-        self.swarm
+        self.swarm._send_error_reponse(peer_id, id, error, reason);
-            ._send_error_reponse(peer_id, stream_id, error, reason);
     }
 
     /// Sends a response to a peer's request.
-    pub fn send_response(
+    pub fn send_response(&mut self, peer_id: PeerId, id: PeerRequestId, response: Response<TSpec>) {
-        &mut self,
+        self.swarm.send_successful_response(peer_id, id, response);
-        peer_id: PeerId,
-        stream_id: SubstreamId,
-        response: Response<TSpec>,
-    ) {
-        self.swarm
-            .send_successful_response(peer_id, stream_id, response);
     }
 
     pub async fn next_event(&mut self) -> Libp2pEvent<TSpec> {

beacon_node/network/src/router/mod.rs

@@ -10,8 +10,8 @@ use crate::error;
 use crate::service::NetworkMessage;
 use beacon_chain::{BeaconChain, BeaconChainTypes, BlockError};
 use eth2_libp2p::{
-    rpc::{RPCError, RequestId, SubstreamId},
+    rpc::{RPCError, RequestId},
-    MessageId, NetworkGlobals, PeerId, PubsubMessage, Request, Response,
+    MessageId, NetworkGlobals, PeerId, PeerRequestId, PubsubMessage, Request, Response,
 };
 use futures::prelude::*;
 use processor::Processor;
@@ -46,7 +46,7 @@ pub enum RouterMessage<T: EthSpec> {
     /// An RPC request has been received.
     RPCRequestReceived {
         peer_id: PeerId,
-        stream_id: SubstreamId,
+        id: PeerRequestId,
         request: Request,
     },
     /// An RPC response has been received.
@@ -127,10 +127,10 @@ impl<T: BeaconChainTypes> Router<T> {
             }
             RouterMessage::RPCRequestReceived {
                 peer_id,
-                stream_id,
+                id,
                 request,
             } => {
-                self.handle_rpc_request(peer_id, stream_id, request);
+                self.handle_rpc_request(peer_id, id, request);
             }
             RouterMessage::RPCResponseReceived {
                 peer_id,
@@ -160,11 +160,11 @@ impl<T: BeaconChainTypes> Router<T> {
     /* RPC - Related functionality */
 
     /// A new RPC request has been received from the network.
-    fn handle_rpc_request(&mut self, peer_id: PeerId, stream_id: SubstreamId, request: Request) {
+    fn handle_rpc_request(&mut self, peer_id: PeerId, id: PeerRequestId, request: Request) {
         match request {
             Request::Status(status_message) => {
                 self.processor
-                    .on_status_request(peer_id, stream_id, status_message)
+                    .on_status_request(peer_id, id, status_message)
             }
             Request::Goodbye(goodbye_reason) => {
                 debug!(
@@ -177,10 +177,10 @@ impl<T: BeaconChainTypes> Router<T> {
             }
             Request::BlocksByRange(request) => self
                 .processor
-                .on_blocks_by_range_request(peer_id, stream_id, request),
+                .on_blocks_by_range_request(peer_id, id, request),
             Request::BlocksByRoot(request) => self
                 .processor
-                .on_blocks_by_root_request(peer_id, stream_id, request),
+                .on_blocks_by_root_request(peer_id, id, request),
         }
     }
 

beacon_node/network/src/router/processor.rs

@@ -8,7 +8,7 @@ use beacon_chain::{
     BeaconChain, BeaconChainTypes, BlockError, BlockProcessingOutcome, GossipVerifiedBlock,
 };
 use eth2_libp2p::rpc::*;
-use eth2_libp2p::{NetworkGlobals, PeerId, Request, Response};
+use eth2_libp2p::{NetworkGlobals, PeerId, PeerRequestId, Request, Response};
 use itertools::process_results;
 use slog::{debug, error, o, trace, warn};
 use ssz::Encode;
@@ -118,7 +118,7 @@ impl<T: BeaconChainTypes> Processor<T> {
     pub fn on_status_request(
         &mut self,
         peer_id: PeerId,
-        request_id: SubstreamId,
+        request_id: PeerRequestId,
         status: StatusMessage,
     ) {
         debug!(
@@ -283,7 +283,7 @@ impl<T: BeaconChainTypes> Processor<T> {
     pub fn on_blocks_by_root_request(
         &mut self,
         peer_id: PeerId,
-        request_id: SubstreamId,
+        request_id: PeerRequestId,
         request: BlocksByRootRequest,
     ) {
         let mut send_block_count = 0;
@@ -321,7 +321,7 @@ impl<T: BeaconChainTypes> Processor<T> {
     pub fn on_blocks_by_range_request(
         &mut self,
         peer_id: PeerId,
-        request_id: SubstreamId,
+        request_id: PeerRequestId,
         req: BlocksByRangeRequest,
     ) {
         debug!(
@@ -958,29 +958,24 @@ impl<T: EthSpec> HandlerNetworkContext<T> {
         })
     }
 
-    pub fn send_response(
+    pub fn send_response(&mut self, peer_id: PeerId, response: Response<T>, id: PeerRequestId) {
-        &mut self,
-        peer_id: PeerId,
-        response: Response<T>,
-        stream_id: SubstreamId,
-    ) {
         self.inform_network(NetworkMessage::SendResponse {
             peer_id,
-            stream_id,
+            id,
             response,
         })
     }
     pub fn _send_error_response(
         &mut self,
         peer_id: PeerId,
-        substream_id: SubstreamId,
+        id: PeerRequestId,
         error: RPCResponseErrorCode,
         reason: String,
     ) {
         self.inform_network(NetworkMessage::SendError {
             peer_id,
             error,
-            substream_id,
+            id,
             reason,
         })
     }

beacon_node/network/src/service.rs

@@ -8,8 +8,8 @@ use crate::{error, metrics};
 use beacon_chain::{BeaconChain, BeaconChainTypes};
 use eth2_libp2p::Service as LibP2PService;
 use eth2_libp2p::{
-    rpc::{RPCResponseErrorCode, RequestId, SubstreamId},
+    rpc::{RPCResponseErrorCode, RequestId},
-    Libp2pEvent, PubsubMessage, Request, Response,
+    Libp2pEvent, PeerRequestId, PubsubMessage, Request, Response,
 };
 use eth2_libp2p::{BehaviourEvent, Enr, MessageId, NetworkGlobals, PeerId};
 use futures::prelude::*;
@@ -164,11 +164,11 @@ fn spawn_service<T: BeaconChainTypes>(
                         NetworkMessage::SendRequest{ peer_id, request, request_id } => {
                             service.libp2p.send_request(peer_id, request_id, request);
                         }
-                        NetworkMessage::SendResponse{ peer_id, response, stream_id } => {
+                        NetworkMessage::SendResponse{ peer_id, response, id } => {
-                            service.libp2p.send_response(peer_id, stream_id, response);
+                            service.libp2p.send_response(peer_id, id, response);
                         }
-                        NetworkMessage::SendError{ peer_id, error, substream_id, reason } => {
+                        NetworkMessage::SendError{ peer_id, error, id, reason } => {
-                            service.libp2p.respond_with_error(peer_id, substream_id, error, reason);
+                            service.libp2p.respond_with_error(peer_id, id, error, reason);
                         }
                         NetworkMessage::Propagate {
                             propagation_source,
@@ -280,7 +280,7 @@ fn spawn_service<T: BeaconChainTypes>(
                                 };
                                 let _ = service
                                     .router_send
-                                    .send(RouterMessage::RPCRequestReceived{peer_id, stream_id:id, request})
+                                    .send(RouterMessage::RPCRequestReceived{peer_id, id, request})
                                     .map_err(|_| {
                                         debug!(service.log, "Failed to send RPC to router");
                                     });
@@ -288,7 +288,7 @@ fn spawn_service<T: BeaconChainTypes>(
                             BehaviourEvent::ResponseReceived{peer_id, id, response} => {
                                 let _ = service
                                     .router_send
-                                    .send(RouterMessage::RPCResponseReceived{ peer_id, request_id:id, response })
+                                    .send(RouterMessage::RPCResponseReceived{ peer_id, request_id: id, response })
                                     .map_err(|_| {
                                         debug!(service.log, "Failed to send RPC to router");
                                     });
@@ -297,7 +297,7 @@ fn spawn_service<T: BeaconChainTypes>(
                             BehaviourEvent::RPCFailed{id, peer_id, error} => {
                                 let _ = service
                                     .router_send
-                                    .send(RouterMessage::RPCFailed{ peer_id, request_id:id, error })
+                                    .send(RouterMessage::RPCFailed{ peer_id, request_id: id, error })
                                     .map_err(|_| {
                                         debug!(service.log, "Failed to send RPC to router");
                                     });
@@ -425,7 +425,7 @@ pub enum NetworkMessage<T: EthSpec> {
     SendResponse {
         peer_id: PeerId,
         response: Response<T>,
-        stream_id: SubstreamId,
+        id: PeerRequestId,
     },
     /// Respond to a peer's request with an error.
     SendError {
@@ -434,7 +434,7 @@ pub enum NetworkMessage<T: EthSpec> {
         peer_id: PeerId,
         error: RPCResponseErrorCode,
         reason: String,
-        substream_id: SubstreamId,
+        id: PeerRequestId,
     },
     /// Publish a list of messages to the gossipsub protocol.
     Publish { messages: Vec<PubsubMessage<T>> },