From ba0f3daf9dff20c6b047c472ce44cb2dcf00fa9f Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 29 Jul 2020 03:40:22 +0000 Subject: [PATCH] Gossipsub update (#1400) ## Issue Addressed N/A ## Proposed Changes This provides a number of corrections and improvements to gossipsub. Specifically - Enables options for greater privacy around the message author - Provides greater flexibility on message validation - Prevents unvalidated messages from being gossiped - Shifts the duplicate cache to a time-based cache inside gossipsub - Updates the message-id to handle bytes - Bug fixes related to mesh maintenance and topic subscription. This should improve our attestation inclusion rate. --- Cargo.lock | 192 ++++++++++--------- beacon_node/eth2_libp2p/Cargo.toml | 2 +- beacon_node/eth2_libp2p/src/behaviour/mod.rs | 58 +++--- beacon_node/eth2_libp2p/src/config.rs | 12 +- beacon_node/network/src/router/mod.rs | 4 +- beacon_node/network/src/service.rs | 10 +- 6 files changed, 139 insertions(+), 139 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8b93e4f92..2d38665be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,7 +25,7 @@ dependencies = [ "slog", "slog-async", "slog-term", - "tokio 0.2.21", + "tokio 0.2.22", "types", "validator_dir", "web3", @@ -203,9 +203,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.31" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85bb70cc08ec97ca5450e6eba421deeea5f172c0fc61f78b5357b2a8e8be195f" +checksum = "6b602bfe940d21c130f3895acd65221e8a61270debe89d628b9cb4e3ccb8569b" [[package]] name = "arbitrary" @@ -271,6 +271,12 @@ dependencies = [ "webpki-roots 0.19.0", ] +[[package]] +name = "atomic" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64f46ca51dca4837f1520754d1c8c36636356b81553d928dc9c177025369a06e" + [[package]] name = "atomic-option" version = "0.1.2" @@ -366,7 +372,7 @@ dependencies = [ "lazy_static", "lighthouse_metrics", "log 0.4.11", - "lru 0.5.3", + "lru", "merkle_proof", "operation_pool", "parking_lot 0.11.0", @@ -386,7 +392,7 @@ dependencies = [ "state_processing", "store", "tempfile", - "tokio 0.2.21", + "tokio 0.2.22", "tree_hash", "types", "websocket_server", @@ -419,7 +425,7 @@ dependencies = [ "slog-async", "slog-term", "store", - "tokio 0.2.21", + "tokio 0.2.22", "types", "version", ] @@ -574,7 +580,7 @@ dependencies = [ "slog-stdlog", "slog-term", "sloggers", - "tokio 0.2.21", + "tokio 0.2.22", ] [[package]] @@ -783,7 +789,7 @@ dependencies = [ "store", "time 0.2.16", "timer", - "tokio 0.2.21", + "tokio 0.2.22", "toml", "tree_hash", "types", @@ -963,12 +969,12 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cced8691919c02aac3cb0a1bc2e9b73d89e832bf9a06fc579d4e71b68a2da061" +checksum = "09ee0cc8804d5393478d743b035099520087a5186f3b93fa58cec08fa62407b6" dependencies = [ + "cfg-if", "crossbeam-utils", - "maybe-uninit", ] [[package]] @@ -1259,7 +1265,7 @@ dependencies = [ "rlp", "sha2 0.8.2", "smallvec 1.4.1", - "tokio 0.2.21", + "tokio 0.2.22", "uint", "zeroize", ] @@ -1339,9 +1345,9 @@ dependencies = [ [[package]] name = "enr" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1de1cc94d04bd0d2d7014300dd790f02cb0e9d6c8499cb72c273ddeca66d1e17" +checksum = "1c78d64a14865c080072c05ffb2c11aab15963d5e763ca4dbc02865dc1b615ee" dependencies = [ "base64 0.12.3", "bs58", @@ -1388,7 +1394,7 @@ dependencies = [ "slog-json", "slog-term", "sloggers", - "tokio 0.2.21", + "tokio 0.2.22", "types", ] @@ -1424,7 +1430,7 @@ dependencies = [ "slog", "sloggers", "state_processing", - "tokio 0.2.21", + "tokio 0.2.22", "toml", "tree_hash", "types", @@ -1438,7 +1444,7 @@ dependencies = [ "deposit_contract", "futures 0.3.5", "serde_json", - "tokio 0.2.21", + "tokio 0.2.22", "types", "web3", ] @@ -1533,7 +1539,7 @@ dependencies = [ "lazy_static", "libp2p", "lighthouse_metrics", - "lru 0.5.3", + "lru", "parking_lot 0.11.0", "rand 0.7.3", "serde", @@ -1547,7 +1553,7 @@ dependencies = [ "snap", "tempdir", "tiny-keccak 2.0.2", - "tokio 0.2.21", + "tokio 0.2.22", "tokio-io-timeout", "tokio-util", "types", @@ -1980,7 +1986,7 @@ dependencies = [ "serde_derive", "slog", "state_processing", - "tokio 0.2.21", + "tokio 0.2.22", "tree_hash", "types", ] @@ -2093,7 +2099,7 @@ dependencies = [ "http 0.2.1", "indexmap", "slab 0.4.2", - "tokio 0.2.21", + "tokio 0.2.22", "tokio-util", "tracing", ] @@ -2128,7 +2134,7 @@ name = "hashset_delay" version = "0.2.0" dependencies = [ "futures 0.3.5", - "tokio 0.2.21", + "tokio 0.2.22", ] [[package]] @@ -2161,6 +2167,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" +[[package]] +name = "hex_fmt" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b07f60793ff0a4d9cef0f18e63b5357e06209987153a64648c972c1e5aff336f" + [[package]] name = "hkdf" version = "0.8.0" @@ -2328,7 +2340,7 @@ dependencies = [ "pin-project", "socket2", "time 0.1.43", - "tokio 0.2.21", + "tokio 0.2.22", "tower-service", "tracing", "want 0.3.0", @@ -2356,7 +2368,7 @@ dependencies = [ "bytes 0.5.6", "hyper 0.13.7", "native-tls", - "tokio 0.2.21", + "tokio 0.2.22", "tokio-tls 0.3.1", ] @@ -2561,7 +2573,7 @@ dependencies = [ "serde_yaml", "simple_logger", "state_processing", - "tokio 0.2.21", + "tokio 0.2.22", "tree_hash", "types", "validator_dir", @@ -2622,12 +2634,13 @@ checksum = "c7d73b3f436185384286bd8098d17ec07c9a7d2388a6599f824d8502b529702a" [[package]] name = "libp2p" version = "0.22.0" -source = "git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94#147bb43fa56c1b84253606eabedb0794eeed8b94" +source = "git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22#8211054ddd6576d3f1cc317ecb207f1b47308c22" dependencies = [ + "atomic", "bytes 0.5.6", "futures 0.3.5", "lazy_static", - "libp2p-core 0.20.1 (git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94)", + "libp2p-core 0.20.1 (git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22)", "libp2p-core-derive", "libp2p-dns", "libp2p-gossipsub", @@ -2640,7 +2653,7 @@ dependencies = [ "libp2p-websocket", "libp2p-yamux", "multihash", - "parity-multiaddr 0.9.1 (git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94)", + "parity-multiaddr 0.9.1 (git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22)", "parking_lot 0.10.2", "pin-project", "smallvec 1.4.1", @@ -2650,7 +2663,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.20.1" -source = "git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94#147bb43fa56c1b84253606eabedb0794eeed8b94" +source = "git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22#8211054ddd6576d3f1cc317ecb207f1b47308c22" dependencies = [ "asn1_der", "bs58", @@ -2663,8 +2676,8 @@ dependencies = [ "libsecp256k1", "log 0.4.11", "multihash", - "multistream-select 0.8.2 (git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94)", - "parity-multiaddr 0.9.1 (git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94)", + "multistream-select 0.8.2 (git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22)", + "parity-multiaddr 0.9.1 (git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22)", "parking_lot 0.10.2", "pin-project", "prost", @@ -2717,7 +2730,7 @@ dependencies = [ [[package]] name = "libp2p-core-derive" version = "0.20.1" -source = "git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94#147bb43fa56c1b84253606eabedb0794eeed8b94" +source = "git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22#8211054ddd6576d3f1cc317ecb207f1b47308c22" dependencies = [ "quote", "syn", @@ -2726,17 +2739,17 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.20.0" -source = "git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94#147bb43fa56c1b84253606eabedb0794eeed8b94" +source = "git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22#8211054ddd6576d3f1cc317ecb207f1b47308c22" dependencies = [ "futures 0.3.5", - "libp2p-core 0.20.1 (git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94)", + "libp2p-core 0.20.1 (git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22)", "log 0.4.11", ] [[package]] name = "libp2p-gossipsub" version = "0.20.0" -source = "git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94#147bb43fa56c1b84253606eabedb0794eeed8b94" +source = "git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22#8211054ddd6576d3f1cc317ecb207f1b47308c22" dependencies = [ "base64 0.11.0", "byteorder", @@ -2744,10 +2757,11 @@ dependencies = [ "fnv", "futures 0.3.5", "futures_codec", - "libp2p-core 0.20.1 (git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94)", + "hex_fmt", + "libp2p-core 0.20.1 (git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22)", "libp2p-swarm", "log 0.4.11", - "lru 0.4.3", + "lru_time_cache", "prost", "prost-build", "rand 0.7.3", @@ -2760,10 +2774,10 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.20.0" -source = "git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94#147bb43fa56c1b84253606eabedb0794eeed8b94" +source = "git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22#8211054ddd6576d3f1cc317ecb207f1b47308c22" dependencies = [ "futures 0.3.5", - "libp2p-core 0.20.1 (git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94)", + "libp2p-core 0.20.1 (git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22)", "libp2p-swarm", "log 0.4.11", "prost", @@ -2775,13 +2789,13 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.20.0" -source = "git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94#147bb43fa56c1b84253606eabedb0794eeed8b94" +source = "git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22#8211054ddd6576d3f1cc317ecb207f1b47308c22" dependencies = [ "bytes 0.5.6", "fnv", "futures 0.3.5", "futures_codec", - "libp2p-core 0.20.1 (git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94)", + "libp2p-core 0.20.1 (git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22)", "log 0.4.11", "parking_lot 0.10.2", "unsigned-varint 0.4.0", @@ -2790,13 +2804,13 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.21.0" -source = "git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94#147bb43fa56c1b84253606eabedb0794eeed8b94" +source = "git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22#8211054ddd6576d3f1cc317ecb207f1b47308c22" dependencies = [ "bytes 0.5.6", "curve25519-dalek", "futures 0.3.5", "lazy_static", - "libp2p-core 0.20.1 (git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94)", + "libp2p-core 0.20.1 (git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22)", "log 0.4.11", "prost", "prost-build", @@ -2811,7 +2825,7 @@ dependencies = [ [[package]] name = "libp2p-secio" version = "0.20.0" -source = "git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94#147bb43fa56c1b84253606eabedb0794eeed8b94" +source = "git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22#8211054ddd6576d3f1cc317ecb207f1b47308c22" dependencies = [ "aes-ctr 0.3.0", "ctr 0.3.2", @@ -2819,7 +2833,7 @@ dependencies = [ "hmac 0.7.1", "js-sys", "lazy_static", - "libp2p-core 0.20.1 (git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94)", + "libp2p-core 0.20.1 (git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22)", "log 0.4.11", "parity-send-wrapper", "pin-project", @@ -2840,10 +2854,10 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.20.1" -source = "git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94#147bb43fa56c1b84253606eabedb0794eeed8b94" +source = "git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22#8211054ddd6576d3f1cc317ecb207f1b47308c22" dependencies = [ "futures 0.3.5", - "libp2p-core 0.20.1 (git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94)", + "libp2p-core 0.20.1 (git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22)", "log 0.4.11", "rand 0.7.3", "smallvec 1.4.1", @@ -2854,27 +2868,27 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.20.0" -source = "git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94#147bb43fa56c1b84253606eabedb0794eeed8b94" +source = "git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22#8211054ddd6576d3f1cc317ecb207f1b47308c22" dependencies = [ "futures 0.3.5", "futures-timer", "get_if_addrs", "ipnet", - "libp2p-core 0.20.1 (git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94)", + "libp2p-core 0.20.1 (git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22)", "log 0.4.11", "socket2", - "tokio 0.2.21", + "tokio 0.2.22", ] [[package]] name = "libp2p-websocket" version = "0.21.1" -source = "git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94#147bb43fa56c1b84253606eabedb0794eeed8b94" +source = "git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22#8211054ddd6576d3f1cc317ecb207f1b47308c22" dependencies = [ "async-tls", "either", "futures 0.3.5", - "libp2p-core 0.20.1 (git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94)", + "libp2p-core 0.20.1 (git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22)", "log 0.4.11", "quicksink", "rustls", @@ -2888,10 +2902,10 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.20.0" -source = "git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94#147bb43fa56c1b84253606eabedb0794eeed8b94" +source = "git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22#8211054ddd6576d3f1cc317ecb207f1b47308c22" dependencies = [ "futures 0.3.5", - "libp2p-core 0.20.1 (git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94)", + "libp2p-core 0.20.1 (git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22)", "parking_lot 0.10.2", "thiserror", "yamux", @@ -2956,7 +2970,7 @@ dependencies = [ "slog-term", "sloggers", "tempfile", - "tokio 0.2.21", + "tokio 0.2.22", "types", "validator_client", "validator_dir", @@ -3022,15 +3036,6 @@ dependencies = [ "slog-term", ] -[[package]] -name = "lru" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0609345ddee5badacf857d4f547e0e5a2e987db77085c24cd887f73573a04237" -dependencies = [ - "hashbrown 0.6.3", -] - [[package]] name = "lru" version = "0.5.3" @@ -3249,7 +3254,7 @@ checksum = "d8883adfde9756c1d30b0f519c9b8c502a94b41ac62f696453c37c7fc0a958ce" [[package]] name = "multistream-select" version = "0.8.2" -source = "git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94#147bb43fa56c1b84253606eabedb0794eeed8b94" +source = "git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22#8211054ddd6576d3f1cc317ecb207f1b47308c22" dependencies = [ "bytes 0.5.6", "futures 0.3.5", @@ -3333,7 +3338,7 @@ dependencies = [ "state_processing", "store", "tempfile", - "tokio 0.2.21", + "tokio 0.2.22", "tree_hash", "types", ] @@ -3537,7 +3542,7 @@ dependencies = [ [[package]] name = "parity-multiaddr" version = "0.9.1" -source = "git+https://github.com/sigp/rust-libp2p?rev=147bb43fa56c1b84253606eabedb0794eeed8b94#147bb43fa56c1b84253606eabedb0794eeed8b94" +source = "git+https://github.com/sigp/rust-libp2p?rev=8211054ddd6576d3f1cc317ecb207f1b47308c22#8211054ddd6576d3f1cc317ecb207f1b47308c22" dependencies = [ "arrayref", "bs58", @@ -3706,18 +3711,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "0.4.22" +version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12e3a6cdbfe94a5e4572812a0201f8c0ed98c1c452c7b8563ce2276988ef9c17" +checksum = "ca4433fff2ae79342e497d9f8ee990d174071408f28f726d6d83af93e58e48aa" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "0.4.22" +version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a0ffd45cf79d88737d7cc85bfd5d2894bee1139b356e616fe85dc389c61aaf7" +checksum = "2c0e815c3ee9a031fdf5af21c10aa17c573c9c6a566328d99e3936c34e36461f" dependencies = [ "proc-macro2", "quote", @@ -3800,9 +3805,9 @@ dependencies = [ [[package]] name = "proc-macro-hack" -version = "0.5.16" +version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4" +checksum = "99c605b9a0adc77b7211c6b1f722dcb613d68d66859a44f3d485a6da332b0598" [[package]] name = "proc-macro-nested" @@ -4207,9 +4212,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b82c9238b305f26f53443e3a4bc8528d64b8d0bee408ec949eb7bf5635ec680" +checksum = "12427a5577082c24419c9c417db35cfeb65962efc7675bb6b0d5f1f9d315bfe6" dependencies = [ "base64 0.12.3", "bytes 0.5.6", @@ -4220,6 +4225,7 @@ dependencies = [ "http-body 0.3.1", "hyper 0.13.7", "hyper-tls 0.4.3", + "ipnet", "js-sys", "lazy_static", "log 0.4.11", @@ -4231,7 +4237,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "tokio 0.2.21", + "tokio 0.2.22", "tokio-tls 0.3.1", "url 2.1.1", "wasm-bindgen", @@ -4276,7 +4282,7 @@ dependencies = [ "slot_clock", "state_processing", "store", - "tokio 0.2.21", + "tokio 0.2.22", "tree_hash", "types", "uhttp_sse", @@ -4595,9 +4601,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3433e879a558dde8b5e8feb2a04899cf34fdde1fafb894687e52105fc1162ac3" +checksum = "164eacbdb13512ec2745fb09d51fd5b22b0d65ed294a1dcf7285a360c80a675c" dependencies = [ "itoa", "ryu", @@ -4735,7 +4741,7 @@ dependencies = [ "node_test_rig", "parking_lot 0.11.0", "rayon", - "tokio 0.2.21", + "tokio 0.2.22", "types", "validator_client", ] @@ -5077,7 +5083,7 @@ dependencies = [ "lazy_static", "leveldb", "lighthouse_metrics", - "lru 0.5.3", + "lru", "parking_lot 0.11.0", "rayon", "serde", @@ -5147,9 +5153,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.35" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb7f4c519df8c117855e19dd8cc851e89eb746fe7a73f0157e0d95fdec5369b0" +checksum = "4cdb98bcb1f9d81d07b536179c269ea15999b5d14ea958196413869445bb5250" dependencies = [ "proc-macro2", "quote", @@ -5336,7 +5342,7 @@ dependencies = [ "parking_lot 0.11.0", "slog", "slot_clock", - "tokio 0.2.21", + "tokio 0.2.22", "types", ] @@ -5416,9 +5422,9 @@ dependencies = [ [[package]] name = "tokio" -version = "0.2.21" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d099fa27b9702bed751524694adbe393e18b36b204da91eb1cbbbbb4a5ee2d58" +checksum = "5d34ca54d84bf2b5b4d7d31e901a8464f7b60ac145a284fba25ceb801f2ddccd" dependencies = [ "bytes 0.5.6", "fnv", @@ -5528,7 +5534,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9390a43272c8a6ac912ed1d1e2b6abeafd5047e05530a2fa304deee041a06215" dependencies = [ "bytes 0.5.6", - "tokio 0.2.21", + "tokio 0.2.22", ] [[package]] @@ -5642,7 +5648,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343" dependencies = [ "native-tls", - "tokio 0.2.21", + "tokio 0.2.22", ] [[package]] @@ -5707,7 +5713,7 @@ dependencies = [ "futures-sink", "log 0.4.11", "pin-project-lite", - "tokio 0.2.21", + "tokio 0.2.22", ] [[package]] @@ -5727,9 +5733,9 @@ checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" [[package]] name = "tracing" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e2a2de6b0d5cbb13fc21193a2296888eaab62b6044479aafb3c54c01c29fcd" +checksum = "dbdf4ccd1652592b01286a5dbe1e2a77d78afaa34beadd9872a5f7396f92aaa9" dependencies = [ "cfg-if", "log 0.4.11", @@ -6047,7 +6053,7 @@ dependencies = [ "slog-term", "slot_clock", "tempdir", - "tokio 0.2.21", + "tokio 0.2.22", "tree_hash", "types", "validator_dir", @@ -6355,7 +6361,7 @@ dependencies = [ "serde_derive", "serde_json", "slog", - "tokio 0.2.21", + "tokio 0.2.22", "types", "ws", ] diff --git a/beacon_node/eth2_libp2p/Cargo.toml b/beacon_node/eth2_libp2p/Cargo.toml index 5e10675ab..3aa156674 100644 --- a/beacon_node/eth2_libp2p/Cargo.toml +++ b/beacon_node/eth2_libp2p/Cargo.toml @@ -41,7 +41,7 @@ rand = "0.7.3" [dependencies.libp2p] #version = "0.19.1" git = "https://github.com/sigp/rust-libp2p" -rev = "147bb43fa56c1b84253606eabedb0794eeed8b94" +rev = "8e9e35994e63716c6eb0a05b9702133d113b3822" default-features = false features = ["websocket", "identify", "mplex", "yamux", "noise", "gossipsub", "dns", "secio", "tcp-tokio"] diff --git a/beacon_node/eth2_libp2p/src/behaviour/mod.rs b/beacon_node/eth2_libp2p/src/behaviour/mod.rs index 857a9eded..4a5c4cddb 100644 --- a/beacon_node/eth2_libp2p/src/behaviour/mod.rs +++ b/beacon_node/eth2_libp2p/src/behaviour/mod.rs @@ -11,7 +11,7 @@ use libp2p::{ identity::Keypair, Multiaddr, }, - gossipsub::{Gossipsub, GossipsubEvent, MessageId}, + gossipsub::{Gossipsub, GossipsubEvent, MessageAuthenticity, MessageId}, identify::{Identify, IdentifyEvent}, swarm::{ NetworkBehaviour, NetworkBehaviourAction as NBAction, NotifyHandler, PollParameters, @@ -19,7 +19,6 @@ use libp2p::{ }, PeerId, }; -use lru::LruCache; use slog::{crit, debug, o, trace}; use std::{ collections::VecDeque, @@ -56,10 +55,6 @@ pub struct Behaviour { peers_to_dc: VecDeque, /// The current meta data of the node, so respond to pings and get metadata meta_data: MetaData, - /// A cache of recently seen gossip messages. This is used to filter out any possible - /// duplicates that may still be seen over gossipsub. - // TODO: Remove this - seen_gossip_messages: LruCache, /// A collections of variables accessible outside the network service. network_globals: Arc>, /// Keeps track of the current EnrForkId for upgrading gossipsub topics. @@ -80,7 +75,6 @@ impl Behaviour { network_globals: Arc>, log: &slog::Logger, ) -> error::Result { - let local_peer_id = local_key.public().into_peer_id(); let behaviour_log = log.new(o!()); let identify = Identify::new( @@ -104,15 +98,20 @@ impl Behaviour { attnets, }; + // TODO: Until other clients support no author, we will use a 0 peer_id as our author. + let message_author = PeerId::from_bytes(vec![0, 1, 0]).expect("Valid peer id"); + Ok(Behaviour { eth2_rpc: RPC::new(log.clone()), - gossipsub: Gossipsub::new(local_peer_id, net_conf.gs_config.clone()), + gossipsub: Gossipsub::new( + MessageAuthenticity::Author(message_author), + net_conf.gs_config.clone(), + ), identify, peer_manager: PeerManager::new(local_key, net_conf, network_globals.clone(), log)?, events: VecDeque::new(), handler_events: VecDeque::new(), peers_to_dc: VecDeque::new(), - seen_gossip_messages: LruCache::new(100_000), meta_data, network_globals, enr_fork_id, @@ -215,7 +214,9 @@ impl Behaviour { for topic in message.topics(GossipEncoding::default(), self.enr_fork_id.fork_digest) { match message.encode(GossipEncoding::default()) { Ok(message_data) => { - self.gossipsub.publish(&topic.into(), message_data); + if let Err(e) = self.gossipsub.publish(&topic.into(), message_data) { + slog::warn!(self.log, "Could not publish message"; "error" => format!("{:?}", e)); + } } Err(e) => crit!(self.log, "Could not publish message"; "error" => e), } @@ -225,9 +226,9 @@ impl Behaviour { /// Forwards a message that is waiting in gossipsub's mcache. Messages are only propagated /// once validated by the beacon chain. - pub fn propagate_message(&mut self, propagation_source: &PeerId, message_id: MessageId) { + pub fn validate_message(&mut self, propagation_source: &PeerId, message_id: MessageId) { self.gossipsub - .propagate_message(&message_id, propagation_source); + .validate_message(&message_id, propagation_source); } /* Eth2 RPC behaviour functions */ @@ -394,29 +395,16 @@ impl Behaviour { GossipsubEvent::Message(propagation_source, id, gs_msg) => { // Note: We are keeping track here of the peer that sent us the message, not the // peer that originally published the message. - if self.seen_gossip_messages.put(id.clone(), ()).is_none() { - match PubsubMessage::decode(&gs_msg.topics, &gs_msg.data) { - Err(e) => { - debug!(self.log, "Could not decode gossipsub message"; "error" => e) - } - Ok(msg) => { - // if this message isn't a duplicate, notify the network - self.add_event(BehaviourEvent::PubsubMessage { - id, - source: propagation_source, - topics: gs_msg.topics, - message: msg, - }); - } - } - } else { - match PubsubMessage::::decode(&gs_msg.topics, &gs_msg.data) { - Err(e) => { - debug!(self.log, "Could not decode gossipsub message"; "error" => e) - } - Ok(msg) => { - debug!(self.log, "A duplicate gossipsub message was received"; "message_source" => format!("{}", gs_msg.source), "propagated_peer" => format!("{}",propagation_source), "message" => format!("{}", msg)); - } + match PubsubMessage::decode(&gs_msg.topics, &gs_msg.data) { + Err(e) => debug!(self.log, "Could not decode gossipsub message"; "error" => e), + Ok(msg) => { + // Notify the network + self.add_event(BehaviourEvent::PubsubMessage { + id, + source: propagation_source, + topics: gs_msg.topics, + message: msg, + }); } } } diff --git a/beacon_node/eth2_libp2p/src/config.rs b/beacon_node/eth2_libp2p/src/config.rs index 6b75253e0..4cc66d332 100644 --- a/beacon_node/eth2_libp2p/src/config.rs +++ b/beacon_node/eth2_libp2p/src/config.rs @@ -1,7 +1,9 @@ use crate::types::GossipKind; use crate::Enr; use discv5::{Discv5Config, Discv5ConfigBuilder}; -use libp2p::gossipsub::{GossipsubConfig, GossipsubConfigBuilder, GossipsubMessage, MessageId}; +use libp2p::gossipsub::{ + GossipsubConfig, GossipsubConfigBuilder, GossipsubMessage, MessageId, ValidationMode, +}; use libp2p::Multiaddr; use serde_derive::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -82,7 +84,7 @@ impl Default for Config { // The function used to generate a gossipsub message id // We use base64(SHA256(data)) for content addressing let gossip_message_id = |message: &GossipsubMessage| { - MessageId(base64::encode_config( + MessageId::from(base64::encode_config( &Sha256::digest(&message.data), base64::URL_SAFE_NO_PAD, )) @@ -94,8 +96,10 @@ impl Default for Config { let gs_config = GossipsubConfigBuilder::new() .max_transmit_size(GOSSIP_MAX_SIZE) .heartbeat_interval(Duration::from_secs(1)) - .manual_propagation() // require validation before propagation - .no_source_id() + .validate_messages() // require validation before propagation + .validation_mode(ValidationMode::Permissive) + // Prevent duplicates by caching messages from an epoch + 1 slot amount of time (33*12) + .duplicate_cache_time(Duration::from_secs(396)) .message_id_fn(gossip_message_id) .build(); diff --git a/beacon_node/network/src/router/mod.rs b/beacon_node/network/src/router/mod.rs index e06bd1a9d..e3cbc9ffa 100644 --- a/beacon_node/network/src/router/mod.rs +++ b/beacon_node/network/src/router/mod.rs @@ -297,10 +297,10 @@ impl Router { } } - /// Informs the network service that the message should be forwarded to other peers. + /// Informs the network service that the message should be forwarded to other peers (is valid). fn propagate_message(&mut self, message_id: MessageId, propagation_source: PeerId) { self.network_send - .send(NetworkMessage::Propagate { + .send(NetworkMessage::Validate { propagation_source, message_id, }) diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index d026704af..7409a89de 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -54,9 +54,11 @@ pub enum NetworkMessage { }, /// Publish a list of messages to the gossipsub protocol. Publish { messages: Vec> }, - /// Propagate a received gossipsub message. - Propagate { + /// Validates a received gossipsub message. This will propagate the message on the network. + Validate { + /// The peer that sent us the message. We don't send back to this peer. propagation_source: PeerId, + /// The id of the message we are validating and propagating. message_id: MessageId, }, /// Reports a peer to the peer manager for performing an action. @@ -213,7 +215,7 @@ fn spawn_service( NetworkMessage::SendError{ peer_id, error, id, reason } => { service.libp2p.respond_with_error(peer_id, id, error, reason); } - NetworkMessage::Propagate { + NetworkMessage::Validate { propagation_source, message_id, } => { @@ -224,7 +226,7 @@ fn spawn_service( service .libp2p .swarm - .propagate_message(&propagation_source, message_id); + .validate_message(&propagation_source, message_id); } NetworkMessage::Publish { messages } => { let mut topic_kinds = Vec::new();