From 5977c00edbfbad1ffa8a4d5b3aa6d19c829111d4 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 6 Jul 2020 20:34:40 +1000 Subject: [PATCH] Update libp2p and gossipsub (#1324) * Update libp2p and gossipsub * Remove gossipsub lru cache --- Cargo.lock | 84 ++++++++++---------- beacon_node/eth2_libp2p/Cargo.toml | 2 +- beacon_node/eth2_libp2p/src/behaviour/mod.rs | 51 +++++------- beacon_node/eth2_libp2p/src/config.rs | 2 +- 4 files changed, 63 insertions(+), 76 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 371465177..7695d39b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -691,9 +691,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0fee792e164f78f5fe0c296cc2eb3688a2ca2b70cdff33040922d298203f0c4" +checksum = "c74d84029116787153e02106bf53e66828452a4b325cc8652b788b5967c0a0b6" dependencies = [ "num-integer", "num-traits", @@ -2119,9 +2119,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9586eedd4ce6b3c498bc3b4dd92fc9f11166aa908a914071953768066c67909" +checksum = "3deed196b6e7f9e44a2ae8d94225d80302d81208b1bb673fd21fe634645c85a9" dependencies = [ "libc", ] @@ -2397,9 +2397,9 @@ dependencies = [ [[package]] name = "instant" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69da7ce1490173c2bf4d26bc8be429aaeeaf4cce6c4b970b7949651fa17655fe" +checksum = "5b141fdc7836c525d4d594027d318c84161ca17aaf8113ab1f81ab93ae897485" [[package]] name = "int_to_bytes" @@ -2598,12 +2598,12 @@ checksum = "c7d73b3f436185384286bd8098d17ec07c9a7d2388a6599f824d8502b529702a" [[package]] name = "libp2p" version = "0.20.1" -source = "git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2#a6232506278b9e686248f8d04b79400861b143c2" +source = "git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86#95e27446ca4371e41fc0035b187f60daa19b4b86" dependencies = [ "bytes 0.5.5", "futures 0.3.5", "lazy_static", - "libp2p-core 0.19.2 (git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2)", + "libp2p-core 0.19.2 (git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86)", "libp2p-core-derive", "libp2p-dns", "libp2p-gossipsub", @@ -2616,7 +2616,7 @@ dependencies = [ "libp2p-websocket", "libp2p-yamux", "multihash", - "parity-multiaddr 0.9.1 (git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2)", + "parity-multiaddr 0.9.1 (git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86)", "parking_lot 0.10.2", "pin-project", "smallvec 1.4.0", @@ -2626,7 +2626,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.19.2" -source = "git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2#a6232506278b9e686248f8d04b79400861b143c2" +source = "git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86#95e27446ca4371e41fc0035b187f60daa19b4b86" dependencies = [ "asn1_der", "bs58", @@ -2639,8 +2639,8 @@ dependencies = [ "libsecp256k1", "log 0.4.8", "multihash", - "multistream-select 0.8.2 (git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2)", - "parity-multiaddr 0.9.1 (git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2)", + "multistream-select 0.8.2 (git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86)", + "parity-multiaddr 0.9.1 (git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86)", "parking_lot 0.10.2", "pin-project", "prost", @@ -2693,7 +2693,7 @@ dependencies = [ [[package]] name = "libp2p-core-derive" version = "0.19.1" -source = "git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2#a6232506278b9e686248f8d04b79400861b143c2" +source = "git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86#95e27446ca4371e41fc0035b187f60daa19b4b86" dependencies = [ "quote", "syn", @@ -2702,17 +2702,17 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.19.0" -source = "git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2#a6232506278b9e686248f8d04b79400861b143c2" +source = "git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86#95e27446ca4371e41fc0035b187f60daa19b4b86" dependencies = [ "futures 0.3.5", - "libp2p-core 0.19.2 (git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2)", + "libp2p-core 0.19.2 (git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86)", "log 0.4.8", ] [[package]] name = "libp2p-gossipsub" version = "0.19.3" -source = "git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2#a6232506278b9e686248f8d04b79400861b143c2" +source = "git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86#95e27446ca4371e41fc0035b187f60daa19b4b86" dependencies = [ "base64 0.11.0", "byteorder", @@ -2720,7 +2720,7 @@ dependencies = [ "fnv", "futures 0.3.5", "futures_codec", - "libp2p-core 0.19.2 (git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2)", + "libp2p-core 0.19.2 (git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86)", "libp2p-swarm", "log 0.4.8", "lru 0.4.3", @@ -2736,10 +2736,10 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.19.2" -source = "git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2#a6232506278b9e686248f8d04b79400861b143c2" +source = "git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86#95e27446ca4371e41fc0035b187f60daa19b4b86" dependencies = [ "futures 0.3.5", - "libp2p-core 0.19.2 (git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2)", + "libp2p-core 0.19.2 (git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86)", "libp2p-swarm", "log 0.4.8", "prost", @@ -2751,13 +2751,13 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.19.2" -source = "git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2#a6232506278b9e686248f8d04b79400861b143c2" +source = "git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86#95e27446ca4371e41fc0035b187f60daa19b4b86" dependencies = [ "bytes 0.5.5", "fnv", "futures 0.3.5", "futures_codec", - "libp2p-core 0.19.2 (git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2)", + "libp2p-core 0.19.2 (git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86)", "log 0.4.8", "parking_lot 0.10.2", "unsigned-varint 0.4.0", @@ -2766,12 +2766,12 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.19.1" -source = "git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2#a6232506278b9e686248f8d04b79400861b143c2" +source = "git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86#95e27446ca4371e41fc0035b187f60daa19b4b86" dependencies = [ "curve25519-dalek", "futures 0.3.5", "lazy_static", - "libp2p-core 0.19.2 (git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2)", + "libp2p-core 0.19.2 (git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86)", "log 0.4.8", "prost", "prost-build", @@ -2786,7 +2786,7 @@ dependencies = [ [[package]] name = "libp2p-secio" version = "0.19.2" -source = "git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2#a6232506278b9e686248f8d04b79400861b143c2" +source = "git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86#95e27446ca4371e41fc0035b187f60daa19b4b86" dependencies = [ "aes-ctr 0.3.0", "ctr 0.3.2", @@ -2794,7 +2794,7 @@ dependencies = [ "hmac 0.7.1", "js-sys", "lazy_static", - "libp2p-core 0.19.2 (git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2)", + "libp2p-core 0.19.2 (git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86)", "log 0.4.8", "parity-send-wrapper", "pin-project", @@ -2815,10 +2815,10 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.19.1" -source = "git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2#a6232506278b9e686248f8d04b79400861b143c2" +source = "git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86#95e27446ca4371e41fc0035b187f60daa19b4b86" dependencies = [ "futures 0.3.5", - "libp2p-core 0.19.2 (git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2)", + "libp2p-core 0.19.2 (git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86)", "log 0.4.8", "rand 0.7.3", "smallvec 1.4.0", @@ -2829,13 +2829,13 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.19.2" -source = "git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2#a6232506278b9e686248f8d04b79400861b143c2" +source = "git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86#95e27446ca4371e41fc0035b187f60daa19b4b86" dependencies = [ "futures 0.3.5", "futures-timer", "get_if_addrs", "ipnet", - "libp2p-core 0.19.2 (git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2)", + "libp2p-core 0.19.2 (git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86)", "log 0.4.8", "socket2", "tokio 0.2.21", @@ -2844,12 +2844,12 @@ dependencies = [ [[package]] name = "libp2p-websocket" version = "0.20.0" -source = "git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2#a6232506278b9e686248f8d04b79400861b143c2" +source = "git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86#95e27446ca4371e41fc0035b187f60daa19b4b86" dependencies = [ "async-tls", "either", "futures 0.3.5", - "libp2p-core 0.19.2 (git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2)", + "libp2p-core 0.19.2 (git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86)", "log 0.4.8", "quicksink", "rustls", @@ -2863,10 +2863,10 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.19.1" -source = "git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2#a6232506278b9e686248f8d04b79400861b143c2" +source = "git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86#95e27446ca4371e41fc0035b187f60daa19b4b86" dependencies = [ "futures 0.3.5", - "libp2p-core 0.19.2 (git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2)", + "libp2p-core 0.19.2 (git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86)", "parking_lot 0.10.2", "thiserror", "yamux", @@ -2962,9 +2962,9 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de302ce1fe7482db13738fbaf2e21cfb06a986b89c0bf38d88abf16681aada4e" +checksum = "28247cc5a5be2f05fbcd76dd0cf2c7d3b5400cb978a28042abcd4fa0b3f8261c" dependencies = [ "scopeguard", ] @@ -3244,7 +3244,7 @@ checksum = "d8883adfde9756c1d30b0f519c9b8c502a94b41ac62f696453c37c7fc0a958ce" [[package]] name = "multistream-select" version = "0.8.2" -source = "git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2#a6232506278b9e686248f8d04b79400861b143c2" +source = "git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86#95e27446ca4371e41fc0035b187f60daa19b4b86" dependencies = [ "bytes 0.5.5", "futures 0.3.5", @@ -3532,7 +3532,7 @@ dependencies = [ [[package]] name = "parity-multiaddr" version = "0.9.1" -source = "git+https://github.com/sigp/rust-libp2p?rev=a6232506278b9e686248f8d04b79400861b143c2#a6232506278b9e686248f8d04b79400861b143c2" +source = "git+https://github.com/sigp/rust-libp2p?rev=95e27446ca4371e41fc0035b187f60daa19b4b86#95e27446ca4371e41fc0035b187f60daa19b4b86" dependencies = [ "arrayref", "bs58", @@ -3610,7 +3610,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4893845fa2ca272e647da5d0e46660a314ead9c2fdd9a883aabc32e481a8733" dependencies = [ "instant", - "lock_api 0.4.0", + "lock_api 0.4.1", "parking_lot_core 0.8.0", ] @@ -3905,9 +3905,9 @@ dependencies = [ [[package]] name = "protobuf" -version = "2.15.1" +version = "2.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4951a8253c06334be9fe320bbcf73f14949fde62a0c8128d697eec1ff0fa8cd" +checksum = "d883f78645c21b7281d21305181aa1f4dd9e9363e7cf2566c93121552cff003e" [[package]] name = "psutil" @@ -4469,9 +4469,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "scrypt" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83ecb7ed8e2266bd539feb932c52b9e7ce01d14384166ac50fecb173f762f1aa" +checksum = "10e7e75e27e8cd47e4be027d4b9fdc0b696116f981c22de21ca7bad63a9cb33a" dependencies = [ "hmac 0.8.1", "pbkdf2 0.4.0", diff --git a/beacon_node/eth2_libp2p/Cargo.toml b/beacon_node/eth2_libp2p/Cargo.toml index 4ec33538c..f38b6d2c2 100644 --- a/beacon_node/eth2_libp2p/Cargo.toml +++ b/beacon_node/eth2_libp2p/Cargo.toml @@ -39,7 +39,7 @@ environment = { path = "../../lighthouse/environment" } [dependencies.libp2p] #version = "0.19.1" git = "https://github.com/sigp/rust-libp2p" -rev = "a6232506278b9e686248f8d04b79400861b143c2" +rev = "95e27446ca4371e41fc0035b187f60daa19b4b86" default-features = false features = ["websocket", "identify", "mplex", "yamux", "noise", "gossipsub", "dns", "secio", "tcp-tokio"] diff --git a/beacon_node/eth2_libp2p/src/behaviour/mod.rs b/beacon_node/eth2_libp2p/src/behaviour/mod.rs index cd54c7f6d..ca5ec1499 100644 --- a/beacon_node/eth2_libp2p/src/behaviour/mod.rs +++ b/beacon_node/eth2_libp2p/src/behaviour/mod.rs @@ -11,7 +11,7 @@ use libp2p::{ identity::Keypair, Multiaddr, }, - gossipsub::{Gossipsub, GossipsubEvent, MessageId}, + gossipsub::{Gossipsub, GossipsubEvent, MessageId, Signing}, identify::{Identify, IdentifyEvent}, swarm::{ NetworkBehaviour, NetworkBehaviourAction as NBAction, NotifyHandler, PollParameters, @@ -19,7 +19,6 @@ use libp2p::{ }, PeerId, }; -use lru::LruCache; use slog::{crit, debug, o}; use std::{ marker::PhantomData, @@ -53,10 +52,6 @@ pub struct Behaviour { peers_to_dc: Vec, /// The current meta data of the node, so respond to pings and get metadata meta_data: MetaData, - /// A cache of recently seen gossip messages. This is used to filter out any possible - /// duplicates that may still be seen over gossipsub. - // TODO: Remove this - seen_gossip_messages: LruCache, /// A collections of variables accessible outside the network service. network_globals: Arc>, /// Keeps track of the current EnrForkId for upgrading gossipsub topics. @@ -238,7 +233,6 @@ impl Behaviour { network_globals: Arc>, log: &slog::Logger, ) -> error::Result { - let local_peer_id = local_key.public().into_peer_id(); let behaviour_log = log.new(o!()); let identify = Identify::new( @@ -264,12 +258,14 @@ impl Behaviour { Ok(Behaviour { eth2_rpc: RPC::new(log.clone()), - gossipsub: Gossipsub::new(local_peer_id, net_conf.gs_config.clone()), + gossipsub: Gossipsub::new( + Signing::Disabled(PeerId::random()), + net_conf.gs_config.clone(), + ), identify, peer_manager: PeerManager::new(local_key, net_conf, network_globals.clone(), log)?, events: Vec::new(), peers_to_dc: Vec::new(), - seen_gossip_messages: LruCache::new(100_000), meta_data, network_globals, enr_fork_id, @@ -361,7 +357,9 @@ impl Behaviour { for topic in message.topics(GossipEncoding::default(), self.enr_fork_id.fork_digest) { match message.encode(GossipEncoding::default()) { Ok(message_data) => { - self.gossipsub.publish(&topic.into(), message_data); + if let Err(e) = self.gossipsub.publish(&topic.into(), message_data) { + slog::warn!(self.log, "Could not publish message"; "error" => format!("{:?}", e)); + } } Err(e) => crit!(self.log, "Could not publish message"; "error" => e), } @@ -572,29 +570,18 @@ impl Behaviour { GossipsubEvent::Message(propagation_source, id, gs_msg) => { // Note: We are keeping track here of the peer that sent us the message, not the // peer that originally published the message. - if self.seen_gossip_messages.put(id.clone(), ()).is_none() { - match PubsubMessage::decode(&gs_msg.topics, &gs_msg.data) { - Err(e) => { - debug!(self.log, "Could not decode gossipsub message"; "error" => format!("{}", e)) - } - Ok(msg) => { - // if this message isn't a duplicate, notify the network - self.events.push(BehaviourEvent::PubsubMessage { - id, - source: propagation_source, - topics: gs_msg.topics, - message: msg, - }); - } + match PubsubMessage::decode(&gs_msg.topics, &gs_msg.data) { + Err(e) => { + debug!(self.log, "Could not decode gossipsub message"; "error" => format!("{}", e)) } - } else { - match PubsubMessage::::decode(&gs_msg.topics, &gs_msg.data) { - Err(e) => { - debug!(self.log, "Could not decode gossipsub message"; "error" => format!("{}", e)) - } - Ok(msg) => { - debug!(self.log, "A duplicate gossipsub message was received"; "message_source" => format!("{}", gs_msg.source), "propagated_peer" => format!("{}",propagation_source), "message" => format!("{}", msg)); - } + Ok(msg) => { + // if this message isn't a duplicate, notify the network + self.events.push(BehaviourEvent::PubsubMessage { + id, + source: propagation_source, + topics: gs_msg.topics, + message: msg, + }); } } } diff --git a/beacon_node/eth2_libp2p/src/config.rs b/beacon_node/eth2_libp2p/src/config.rs index a9af3eea6..204ff405a 100644 --- a/beacon_node/eth2_libp2p/src/config.rs +++ b/beacon_node/eth2_libp2p/src/config.rs @@ -94,8 +94,8 @@ impl Default for Config { let gs_config = GossipsubConfigBuilder::new() .max_transmit_size(GOSSIP_MAX_SIZE) .heartbeat_interval(Duration::from_secs(1)) + .history_length(385) // A heartbeat is 1 second. We want to keep an epoch worth of history, .manual_propagation() // require validation before propagation - .no_source_id() .message_id_fn(gossip_message_id) .build();