From 850a2d5985cc08e978b5b492bb0cb90a47359605 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Mon, 17 Aug 2020 02:13:28 +0000 Subject: [PATCH] Persist metadata and enr across restarts (#1513) ## Issue Addressed Resolves #1489 ## Proposed Changes - Change starting metadata seq num to 0 according to the [spec](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/phase0/p2p-interface.md#metadata). - Remove metadata field from `NetworkGlobals` - Persist metadata to disk on every update - Load metadata seq number from disk on restart - Persist enr to disk on update to ensure enr sequence number increments are persisted as well. ## Additional info Since we modified starting metadata seq num to 0 from 1, we might still see `Invalid Sequence number provided` like in #1489 from prysm nodes if they have our metadata cached. --- beacon_node/eth2_libp2p/src/behaviour/mod.rs | 81 +++++++++++++++++--- beacon_node/eth2_libp2p/src/discovery/enr.rs | 2 + beacon_node/eth2_libp2p/src/discovery/mod.rs | 6 ++ beacon_node/eth2_libp2p/src/types/globals.rs | 14 +--- 4 files changed, 79 insertions(+), 24 deletions(-) diff --git a/beacon_node/eth2_libp2p/src/behaviour/mod.rs b/beacon_node/eth2_libp2p/src/behaviour/mod.rs index 2cf8188e9..ebcaca7f4 100644 --- a/beacon_node/eth2_libp2p/src/behaviour/mod.rs +++ b/beacon_node/eth2_libp2p/src/behaviour/mod.rs @@ -1,6 +1,6 @@ use crate::peer_manager::{score::PeerAction, PeerManager, PeerManagerEvent}; use crate::rpc::*; -use crate::types::{GossipEncoding, GossipKind, GossipTopic}; +use crate::types::{EnrBitfield, GossipEncoding, GossipKind, GossipTopic}; use crate::Eth2Enr; use crate::{error, metrics, Enr, NetworkConfig, NetworkGlobals, PubsubMessage, TopicHash}; use futures::prelude::*; @@ -19,7 +19,11 @@ use libp2p::{ }, PeerId, }; -use slog::{crit, debug, o, trace}; +use slog::{crit, debug, o, trace, warn}; +use ssz::{Decode, Encode}; +use std::fs::File; +use std::io::{Read, Write}; +use std::path::PathBuf; use std::{ collections::VecDeque, marker::PhantomData, @@ -32,6 +36,7 @@ use types::{EnrForkId, EthSpec, SignedBeaconBlock, SubnetId}; mod handler; const MAX_IDENTIFY_ADDRESSES: usize = 10; +const METADATA_FILENAME: &str = "metadata"; /// Builds the network behaviour that manages the core protocols of eth2. /// This core behaviour is managed by `Behaviour` which adds peer management to all core @@ -61,6 +66,8 @@ pub struct Behaviour { enr_fork_id: EnrForkId, /// The waker for the current thread. waker: Option, + /// Directory where metadata is stored + network_dir: PathBuf, /// Logger for behaviour actions. log: slog::Logger, } @@ -86,15 +93,7 @@ impl Behaviour { .eth2() .expect("Local ENR must have a fork id"); - let attnets = network_globals - .local_enr() - .bitfield::() - .expect("Local ENR must have subnet bitfield"); - - let meta_data = MetaData { - seq_number: 1, - attnets, - }; + let meta_data = load_or_build_metadata(&net_conf.network_dir, &log); // TODO: Until other clients support no author, we will use a 0 peer_id as our author. let message_author = PeerId::from_bytes(vec![0, 1, 0]).expect("Valid peer id"); @@ -114,6 +113,7 @@ impl Behaviour { network_globals, enr_fork_id, waker: None, + network_dir: net_conf.network_dir.clone(), log: behaviour_log, }) } @@ -346,6 +346,8 @@ impl Behaviour { .local_enr() .bitfield::() .expect("Local discovery must have bitfield"); + // Save the updated metadata to disk + save_metadata_to_disk(&self.network_dir, self.meta_data.clone(), &self.log); } /// Sends a Ping request to the peer. @@ -1036,3 +1038,60 @@ pub enum BehaviourEvent { /// Inform the network to send a Status to this peer. StatusPeer(PeerId), } + +/// Load metadata from persisted file. Return default metadata if loading fails. +fn load_or_build_metadata(network_dir: &PathBuf, log: &slog::Logger) -> MetaData { + // Default metadata + let mut meta_data = MetaData { + seq_number: 0, + attnets: EnrBitfield::::default(), + }; + // Read metadata from persisted file if available + let metadata_path = network_dir.join(METADATA_FILENAME); + if let Ok(mut metadata_file) = File::open(metadata_path) { + let mut metadata_ssz = Vec::new(); + if metadata_file.read_to_end(&mut metadata_ssz).is_ok() { + match MetaData::::from_ssz_bytes(&metadata_ssz) { + Ok(persisted_metadata) => { + meta_data.seq_number = persisted_metadata.seq_number; + // Increment seq number if persisted attnet is not default + if persisted_metadata.attnets != meta_data.attnets { + meta_data.seq_number += 1; + } + debug!(log, "Loaded metadata from disk"); + } + Err(e) => { + debug!( + log, + "Metadata from file could not be decoded"; + "error" => format!("{:?}", e), + ); + } + } + } + }; + + debug!(log, "Metadata sequence number"; "seq_num" => meta_data.seq_number); + save_metadata_to_disk(network_dir, meta_data.clone(), &log); + meta_data +} + +/// Persist metadata to disk +fn save_metadata_to_disk(dir: &PathBuf, metadata: MetaData, log: &slog::Logger) { + let _ = std::fs::create_dir_all(&dir); + match File::create(dir.join(METADATA_FILENAME)) + .and_then(|mut f| f.write_all(&metadata.as_ssz_bytes())) + { + Ok(_) => { + debug!(log, "Metadata written to disk"); + } + Err(e) => { + warn!( + log, + "Could not write metadata to disk"; + "file" => format!("{:?}{:?}",dir, METADATA_FILENAME), + "error" => format!("{}", e) + ); + } + } +} diff --git a/beacon_node/eth2_libp2p/src/discovery/enr.rs b/beacon_node/eth2_libp2p/src/discovery/enr.rs index e60bb38ba..dc44cb3ab 100644 --- a/beacon_node/eth2_libp2p/src/discovery/enr.rs +++ b/beacon_node/eth2_libp2p/src/discovery/enr.rs @@ -82,6 +82,8 @@ pub fn build_or_load_enr( } // same node id, different configuration - update the sequence number + // Note: local_enr is generated with default(0) attnets value, + // so a non default value in persisted enr will also update sequence number. let new_seq_no = disk_enr.seq().checked_add(1).ok_or_else(|| "ENR sequence number on file is too large. Remove it to generate a new NodeId")?; local_enr.set_seq(new_seq_no, &enr_key).map_err(|e| { format!("Could not update ENR sequence number: {:?}", e) diff --git a/beacon_node/eth2_libp2p/src/discovery/mod.rs b/beacon_node/eth2_libp2p/src/discovery/mod.rs index 7982e9351..be2cd6943 100644 --- a/beacon_node/eth2_libp2p/src/discovery/mod.rs +++ b/beacon_node/eth2_libp2p/src/discovery/mod.rs @@ -384,6 +384,9 @@ impl Discovery { // replace the global version *self.network_globals.local_enr.write() = self.discv5.local_enr(); + + // persist modified enr to disk + enr::save_enr_to_disk(Path::new(&self.enr_dir), &self.local_enr(), &self.log); Ok(()) } @@ -416,6 +419,9 @@ impl Discovery { // replace the global version with discovery version *self.network_globals.local_enr.write() = self.discv5.local_enr(); + + // persist modified enr to disk + enr::save_enr_to_disk(Path::new(&self.enr_dir), &self.local_enr(), &self.log); } /* Internal Functions */ diff --git a/beacon_node/eth2_libp2p/src/types/globals.rs b/beacon_node/eth2_libp2p/src/types/globals.rs index 84e183b6b..a4dec670a 100644 --- a/beacon_node/eth2_libp2p/src/types/globals.rs +++ b/beacon_node/eth2_libp2p/src/types/globals.rs @@ -1,10 +1,9 @@ //! A collection of variables that are accessible outside of the network thread itself. use crate::peer_manager::PeerDB; -use crate::rpc::methods::MetaData; use crate::types::SyncState; use crate::Client; use crate::EnrExt; -use crate::{Enr, Eth2Enr, GossipTopic, Multiaddr, PeerId}; +use crate::{Enr, GossipTopic, Multiaddr, PeerId}; use parking_lot::RwLock; use std::collections::HashSet; use std::sync::atomic::{AtomicU16, Ordering}; @@ -13,8 +12,6 @@ use types::EthSpec; pub struct NetworkGlobals { /// The current local ENR. pub local_enr: RwLock, - /// The current node's meta-data. - pub meta_data: RwLock>, /// The local peer_id. pub peer_id: RwLock, /// Listening multiaddrs. @@ -33,17 +30,8 @@ pub struct NetworkGlobals { impl NetworkGlobals { pub fn new(enr: Enr, tcp_port: u16, udp_port: u16, log: &slog::Logger) -> Self { - // set up the local meta data of the node - let meta_data = RwLock::new(MetaData { - seq_number: 0, - attnets: enr - .bitfield::() - .expect("Local ENR must have a bitfield specified"), - }); - NetworkGlobals { local_enr: RwLock::new(enr.clone()), - meta_data, peer_id: RwLock::new(enr.peer_id()), listen_multiaddrs: RwLock::new(Vec::new()), listen_port_tcp: AtomicU16::new(tcp_port),