diff --git a/Cargo.lock b/Cargo.lock index eb164bef1..37a60ce0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -178,6 +178,12 @@ dependencies = [ "syn", ] +[[package]] +name = "assert_approx_eq" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c07dab4369547dbe5114677b33fbbf724971019f3818172d59a97a61c774ffd" + [[package]] name = "assert_matches" version = "1.3.0" @@ -196,6 +202,12 @@ dependencies = [ "webpki-roots 0.19.0", ] +[[package]] +name = "atomic-option" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0db678acb667b525ac40a324fc5f7d3390e29239b31c7327bb8157f5b4fff593" + [[package]] name = "atty" version = "0.2.14" @@ -266,6 +278,7 @@ version = "0.1.2" dependencies = [ "bitvec", "bls", + "bus", "environment", "eth1", "eth2_config", @@ -279,7 +292,7 @@ dependencies = [ "lazy_static", "lighthouse_metrics", "log 0.4.8", - "lru", + "lru 0.5.1", "merkle_proof", "operation_pool", "parking_lot 0.10.2", @@ -468,9 +481,21 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5356f1d23ee24a1f785a56d1d1a5f0fd5b0f6a0c0fb2412ce11da71649ab78f6" +checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" + +[[package]] +name = "bus" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1e66e1779f5b1440f1a58220ba3b3ded4427175f0a9fb8d7066521f8b4e8f2b" +dependencies = [ + "atomic-option", + "crossbeam-channel", + "num_cpus", + "parking_lot_core 0.7.2", +] [[package]] name = "byte-slice-cast" @@ -598,9 +623,9 @@ dependencies = [ [[package]] name = "clear_on_drop" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97276801e127ffb46b66ce23f35cc96bd454fa311294bced4bbace7baa8b1d17" +checksum = "c9cc5db465b294c3fa986d5bbb0f3017cd850bff6dd6c52f9ccff8b4d21b7b08" dependencies = [ "cc", ] @@ -610,6 +635,7 @@ name = "client" version = "0.1.2" dependencies = [ "beacon_chain", + "bus", "dirs", "environment", "error-chain", @@ -653,6 +679,15 @@ dependencies = [ "bitflags 1.2.1", ] +[[package]] +name = "cmake" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e56268c17a6248366d66d4a47a3381369d068cce8409bb1716ed77ea32163bb" +dependencies = [ + "cc", +] + [[package]] name = "colored" version = "1.9.3" @@ -827,9 +862,9 @@ dependencies = [ [[package]] name = "crossbeam-queue" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c695eeca1e7173472a32221542ae469b3e9aac3a4fc81f7696bcad82029493db" +checksum = "ab6bffe714b6bb07e42f201352c34f51fefd355ace793f9e638ebd52d23f98d2" dependencies = [ "cfg-if", "crossbeam-utils", @@ -906,14 +941,14 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26778518a7f6cffa1d25a44b602b62b979bd88adb9e99ffec546998cf3404839" +checksum = "5d85653f070353a16313d0046f173f70d1aadd5b42600a14de626f0dfb3473a5" dependencies = [ "byteorder", "digest", "rand_core 0.5.1", - "subtle 2.2.2", + "subtle 2.2.3", "zeroize", ] @@ -1161,7 +1196,10 @@ dependencies = [ "env_logger", "eth2_config", "eth2_testnet_config", + "exit-future", "futures 0.3.5", + "lazy_static", + "lighthouse_metrics", "logging", "parking_lot 0.10.2", "slog", @@ -1231,10 +1269,12 @@ dependencies = [ "base64 0.12.1", "dirs", "discv5", + "environment", "error-chain", "eth2_ssz", "eth2_ssz_derive", "eth2_ssz_types", + "exit-future", "fnv", "futures 0.3.5", "hashset_delay", @@ -1243,7 +1283,7 @@ dependencies = [ "libp2p", "libp2p-tcp", "lighthouse_metrics", - "lru", + "lru 0.5.1", "parking_lot 0.10.2", "serde", "serde_derive", @@ -1367,7 +1407,7 @@ dependencies = [ name = "eth2_testnet_config" version = "0.2.0" dependencies = [ - "eth2-libp2p", + "enr", "eth2_ssz", "reqwest", "serde", @@ -2007,9 +2047,9 @@ dependencies = [ [[package]] name = "hyper" -version = "0.13.5" +version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96816e1d921eca64d208a85aab4f7798455a8e34229ee5a88c935bdee1b78b14" +checksum = "a6e7655b9594024ad0ee439f3b5a7299369dc2a3f459b47c696f9ff676f9aa1f" dependencies = [ "bytes 0.5.4", "futures-channel", @@ -2021,8 +2061,8 @@ dependencies = [ "httparse", "itoa", "log 0.4.8", - "net2", "pin-project", + "socket2", "time 0.1.43", "tokio 0.2.21", "tower-service", @@ -2049,7 +2089,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3adcd308402b9553630734e9c36b77a7e48b3821251ca2493e8cd596763aafaa" dependencies = [ "bytes 0.5.4", - "hyper 0.13.5", + "hyper 0.13.6", "native-tls", "tokio 0.2.21", "tokio-tls 0.3.1", @@ -2106,9 +2146,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.3.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076f042c5b7b98f31d205f1249267e12a6518c1481e9dae9764af19b707d2292" +checksum = "c398b2b113b55809ceb9ee3e753fcbac793f1956663f3c36549c1346015c2afe" dependencies = [ "autocfg 1.0.0", ] @@ -2169,18 +2209,18 @@ checksum = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e" [[package]] name = "js-sys" -version = "0.3.39" +version = "0.3.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa5a448de267e7358beaf4a5d849518fe9a0c13fce7afd44b06e68550e5562a7" +checksum = "ce10c23ad2ea25ceca0093bd3192229da4c5b3c0f2de499c1ecac0d98d452177" dependencies = [ "wasm-bindgen", ] [[package]] name = "jsonrpc-core" -version = "14.1.0" +version = "14.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25525f6002338fb4debb5167a89a0b47f727a5a48418417545ad3429758b7fec" +checksum = "a0747307121ffb9703afd93afbd0fb4f854c38fb873f2c8b90e0e902f27c7b62" dependencies = [ "futures 0.1.29", "log 0.4.8", @@ -2259,9 +2299,9 @@ dependencies = [ [[package]] name = "leveldb" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8438a36a31c982ac399c4477d7e3c62cc7a6bf91bb6f42837b7e1033359fcbad" +checksum = "a5a2fa830c44ac4762564389a7efe22688a469c8d7b71dd11da2e35c33ae96c2" dependencies = [ "db-key", "leveldb-sys", @@ -2270,24 +2310,26 @@ dependencies = [ [[package]] name = "leveldb-sys" -version = "2.0.5" +version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71f46429bb70612c3e939aaeed27ffd31a24a773d21728a1a426e4089d6778d2" +checksum = "2f4abc211bb716f076618bca48aaae128f6feb326195608f40f41a9cdbedc502" dependencies = [ + "cmake", "libc", + "num_cpus", ] [[package]] name = "libc" -version = "0.2.70" +version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3baa92041a6fec78c687fa0cc2b3fae8884f743d672cf551bed1d6dac6988d0f" +checksum = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49" [[package]] name = "libflate" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1fbe6b967a94346446d37ace319ae85be7eca261bb8149325811ac435d35d64" +checksum = "784f4ec5908a9d7f4e53658906386667e8b02e9389a47cfebf45d324ba9e8d25" dependencies = [ "adler32", "crc32fast", @@ -2405,7 +2447,7 @@ dependencies = [ "libp2p-core", "libp2p-swarm", "log 0.4.8", - "lru", + "lru 0.4.3", "prost", "prost-build", "rand 0.7.3", @@ -2575,7 +2617,7 @@ dependencies = [ "hmac-drbg", "rand 0.7.3", "sha2", - "subtle 2.2.2", + "subtle 2.2.3", "typenum", ] @@ -2679,9 +2721,18 @@ dependencies = [ [[package]] name = "lru" -version = "0.4.5" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e488db3a9e108382265a30764f43cfc87517322e5d04ae0603b32a33461dca3" +checksum = "0609345ddee5badacf857d4f547e0e5a2e987db77085c24cd887f73573a04237" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "lru" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28e0c685219cd60e49a2796bba7e4fe6523e10daca4fd721e84e7f905093d60c" dependencies = [ "hashbrown", ] @@ -2933,10 +2984,13 @@ dependencies = [ name = "network" version = "0.1.2" dependencies = [ + "assert_approx_eq", "beacon_chain", + "environment", "error-chain", "eth2-libp2p", "eth2_ssz", + "exit-future", "fnv", "futures 0.3.5", "genesis", @@ -3276,18 +3330,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "0.4.17" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edc93aeee735e60ecb40cf740eb319ff23eab1c5748abfdb5c180e4ce49f7791" +checksum = "ba3a1acf4a3e70849f8a673497ef984f043f95d2d8252dcdf74d54e6a1e47e8a" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "0.4.17" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e58db2081ba5b4c93bd6be09c40fd36cb9193a8336c384f3b40012e531aa7e40" +checksum = "194e88048b71a3e02eb4ee36a6995fed9b8236c11a7bb9f7247a9d9835b3f265" dependencies = [ "proc-macro2", "quote", @@ -3296,9 +3350,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.1.5" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7505eeebd78492e0f6108f7171c4948dbb120ee8119d9d77d0afa5469bef67f" +checksum = "282adbf10f2698a7a77f8e983a74b2d18176c19a7fd32a45446139ae7b02b715" [[package]] name = "pin-utils" @@ -3320,9 +3374,9 @@ checksum = "feb3b2b1033b8a60b4da6ee470325f887758c95d5320f52f9ce0df055a55940e" [[package]] name = "plotters" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9b1d9ca091d370ea3a78d5619145d1b59426ab0c9eedbad2514a4cee08bf389" +checksum = "0d1685fbe7beba33de0330629da9d955ac75bd54f33d7b79f9a895590124f6bb" dependencies = [ "js-sys", "num-traits", @@ -3363,9 +3417,9 @@ checksum = "8e946095f9d3ed29ec38de908c22f95d9ac008e424c7bcae54c75a79c527c694" [[package]] name = "proc-macro2" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1502d12e458c49a4c9cbff560d0fe0060c252bc29799ed94ca2ed4bb665a0101" +checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" dependencies = [ "unicode-xid", ] @@ -3715,9 +3769,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.3.7" +version = "1.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6020f034922e3194c711b82a627453881bc4682166cabb07134a10c26ba7692" +checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" dependencies = [ "aho-corasick", "memchr", @@ -3736,9 +3790,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.17" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe5bd57d1d7414c6b5ed48563a2c855d995ff777729dcd91c369ec7fea395ae" +checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" [[package]] name = "remote_beacon_node" @@ -3769,18 +3823,18 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.10.4" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b81e49ddec5109a9dcfc5f2a317ff53377c915e9ae9d4f2fb50914b85614e2" +checksum = "3b82c9238b305f26f53443e3a4bc8528d64b8d0bee408ec949eb7bf5635ec680" dependencies = [ - "base64 0.11.0", + "base64 0.12.1", "bytes 0.5.4", "encoding_rs", "futures-core", "futures-util", "http 0.2.1", "http-body 0.3.1", - "hyper 0.13.5", + "hyper 0.13.6", "hyper-tls 0.4.1", "js-sys", "lazy_static", @@ -3793,7 +3847,6 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "time 0.1.43", "tokio 0.2.21", "tokio-tls 0.3.1", "url 2.1.1", @@ -3810,6 +3863,8 @@ dependencies = [ "assert_matches", "beacon_chain", "bls", + "bus", + "environment", "eth2-libp2p", "eth2_config", "eth2_ssz", @@ -3817,7 +3872,7 @@ dependencies = [ "futures 0.3.5", "hex 0.4.2", "http 0.2.1", - "hyper 0.13.5", + "hyper 0.13.6", "lazy_static", "lighthouse_metrics", "network", @@ -3839,6 +3894,7 @@ dependencies = [ "tokio 0.2.21", "tree_hash", "types", + "uhttp_sse", "url 2.1.1", "version", ] @@ -3990,9 +4046,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed3d612bc64430efeb3f7ee6ef26d590dce0c43249217bddc62112540c7941e1" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" [[package]] name = "safe_arith" @@ -4124,18 +4180,18 @@ checksum = "a0eddf2e8f50ced781f288c19f18621fa72a3779e3cb58dbf23b07469b0abeb4" [[package]] name = "serde" -version = "1.0.110" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99e7b308464d16b56eba9964e4972a3eee817760ab60d88c3f86e1fecb08204c" +checksum = "c9124df5b40cbd380080b2cc6ab894c040a3070d995f5c9dc77e18c34a8ae37d" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.110" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818fbf6bfa9a42d3bfcaca148547aa00c7b915bec71d1757aa2d44ca68771984" +checksum = "3f2c3ac8e6ca1e9c80b8be1023940162bf81ae3cffbb1809474152f2ce1eb250" dependencies = [ "proc-macro2", "quote", @@ -4369,9 +4425,9 @@ dependencies = [ [[package]] name = "slog-term" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "124501187c410b6a46fe8a47a48435ae462fae4e02d03c558d358f40b17308cb" +checksum = "bab1d807cf71129b05ce36914e1dbb6fbfbdecaf686301cb457f4fa967f9f5b6" dependencies = [ "atty", "chrono", @@ -4465,7 +4521,7 @@ dependencies = [ "ring", "rustc_version", "sha2", - "subtle 2.2.2", + "subtle 2.2.3", "x25519-dalek", ] @@ -4509,9 +4565,12 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "standback" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e4b8c631c998468961a9ea159f064c5c8499b95b5e4a34b77849d45949d540" +checksum = "b0437cfb83762844799a60e1e3b489d5ceb6a650fbacb86437badc1b6d87b246" +dependencies = [ + "version_check 0.9.2", +] [[package]] name = "state_processing" @@ -4618,7 +4677,7 @@ dependencies = [ "lazy_static", "leveldb", "lighthouse_metrics", - "lru", + "lru 0.5.1", "parking_lot 0.10.2", "rayon", "serde", @@ -4663,9 +4722,9 @@ checksum = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee" [[package]] name = "subtle" -version = "2.2.2" +version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c65d530b10ccaeac294f349038a597e435b18fb456aadd0840a623f83b9e941" +checksum = "502d53007c02d7605a05df1c1a73ee436952781653da5d0bf57ad608f66932c1" [[package]] name = "swap_or_not_shuffle" @@ -4680,9 +4739,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.25" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f14a640819f79b72a710c0be059dce779f9339ae046c8bef12c361d56702146f" +checksum = "93a56fabc59dce20fe48b6c832cc249c713e7ed88fa28b0ee0a3bfcaae5fe4e2" dependencies = [ "proc-macro2", "quote", @@ -4855,6 +4914,7 @@ name = "timer" version = "0.1.2" dependencies = [ "beacon_chain", + "environment", "futures 0.3.5", "parking_lot 0.10.2", "slog", @@ -4899,9 +4959,9 @@ dependencies = [ [[package]] name = "tinytemplate" -version = "1.0.4" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45e4bc5ac99433e0dcb8b9f309dd271a165ae37dde129b9e0ce1bfdd8bfe4891" +checksum = "6d3dc76004a03cec1c5932bca4cdc2e39aaa798e3f82363dd94f9adf6098c12f" dependencies = [ "serde", "serde_json", @@ -5358,6 +5418,12 @@ dependencies = [ "tree_hash_derive", ] +[[package]] +name = "uhttp_sse" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6ff93345ba2206230b1bb1aa3ece1a63dd9443b7531024575d16a0680a59444" + [[package]] name = "uint" version = "0.8.3" @@ -5548,9 +5614,9 @@ dependencies = [ [[package]] name = "vcpkg" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fc439f2794e98976c88a2a2dafce96b930fe8010b0a256b3c2199a773933168" +checksum = "55d1e41d56121e07f1e223db0a4def204e45c85425f6a16d462fd07c8d10d74c" [[package]] name = "vec_map" @@ -5623,9 +5689,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "wasm-bindgen" -version = "0.2.62" +version = "0.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c7d40d09cdbf0f4895ae58cf57d92e1e57a9dd8ed2e8390514b54a47cc5551" +checksum = "4c2dc4aa152834bc334f506c1a06b866416a8b6697d5c9f75b9a689c8486def0" dependencies = [ "cfg-if", "serde", @@ -5635,9 +5701,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.62" +version = "0.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3972e137ebf830900db522d6c8fd74d1900dcfc733462e9a12e942b00b4ac94" +checksum = "ded84f06e0ed21499f6184df0e0cb3494727b0c5da89534e0fcc55c51d812101" dependencies = [ "bumpalo", "lazy_static", @@ -5650,9 +5716,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a369c5e1dfb7569e14d62af4da642a3cbc2f9a3652fe586e26ac22222aa4b04" +checksum = "64487204d863f109eb77e8462189d111f27cb5712cc9fdb3461297a76963a2f6" dependencies = [ "cfg-if", "js-sys", @@ -5662,9 +5728,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.62" +version = "0.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd85aa2c579e8892442954685f0d801f9129de24fa2136b2c6a539c76b65776" +checksum = "838e423688dac18d73e31edce74ddfac468e37b1506ad163ffaf0a46f703ffe3" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5672,9 +5738,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.62" +version = "0.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eb197bd3a47553334907ffd2f16507b4f4f01bbec3ac921a7719e0decdfe72a" +checksum = "3156052d8ec77142051a533cdd686cba889537b213f948cd1d20869926e68e92" dependencies = [ "proc-macro2", "quote", @@ -5685,15 +5751,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.62" +version = "0.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a91c2916119c17a8e316507afaaa2dd94b47646048014bbdf6bef098c1bb58ad" +checksum = "c9ba19973a58daf4db6f352eda73dc0e289493cd29fb2632eb172085b6521acd" [[package]] name = "wasm-bindgen-test" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd8e9dad8040e378f0696b017570c6bc929aac373180e06b3d67ac5059c52da3" +checksum = "0f0dfda4d3b3f8acbc3c291b09208081c203af457fb14a229783b06e2f128aa7" dependencies = [ "console_error_panic_hook", "js-sys", @@ -5705,9 +5771,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-test-macro" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c358c8d2507c1bae25efa069e62ea907aa28700b25c8c33dafb0b15ba4603627" +checksum = "2c2e18093f11c19ca4e188c177fecc7c372304c311189f12c2f9bea5b7324ac7" dependencies = [ "proc-macro2", "quote", @@ -5731,9 +5797,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.39" +version = "0.3.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bc359e5dd3b46cb9687a051d50a2fdd228e4ba7cf6fcf861a5365c3d671a642" +checksum = "7b72fe77fd39e4bd3eaa4412fd299a0be6b3dfe9d2597e2f1c20beb968f41d17" dependencies = [ "js-sys", "wasm-bindgen", @@ -5742,7 +5808,7 @@ dependencies = [ [[package]] name = "web3" version = "0.11.0" -source = "git+https://github.com/tomusdrw/rust-web3#a3e5a5315f0a6bf907183322844f7d6650fa8da7" +source = "git+https://github.com/tomusdrw/rust-web3#69d5746f124033dee922d7d36acef9321c1df0b0" dependencies = [ "arrayvec 0.5.1", "base64 0.12.1", @@ -5825,6 +5891,7 @@ dependencies = [ name = "websocket_server" version = "0.1.2" dependencies = [ + "environment", "futures 0.3.5", "serde", "serde_derive", @@ -5889,9 +5956,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "winreg" -version = "0.6.2" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" dependencies = [ "winapi 0.3.8", ] @@ -5937,9 +6004,9 @@ dependencies = [ [[package]] name = "yaml-rust" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65923dd1784f44da1d2c3dbbc5e822045628c590ba72123e1c73d3c230c4434d" +checksum = "39f0c922f1a334134dc2f7a8b67dc5d25f0735263feec974345ff706bcf20b0d" dependencies = [ "linked-hash-map", ] diff --git a/Dockerfile b/Dockerfile index 39d0eb2b6..8856d1493 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ FROM rust:1.43.1 AS builder +RUN apt-get update && apt-get install -y cmake COPY . lighthouse RUN cd lighthouse && make RUN cd lighthouse && cargo install --path lcli --locked diff --git a/account_manager/Cargo.toml b/account_manager/Cargo.toml index 005ff2878..f53bf52a5 100644 --- a/account_manager/Cargo.toml +++ b/account_manager/Cargo.toml @@ -31,5 +31,5 @@ eth2_wallet = { path = "../crypto/eth2_wallet" } eth2_wallet_manager = { path = "../common/eth2_wallet_manager" } rand = "0.7.2" validator_dir = { path = "../common/validator_dir", features = ["unencrypted_keys"] } -tokio = {version = "0.2.20", features = ["full"]} +tokio = { version = "0.2.21", features = ["full"] } eth2_keystore = { path = "../crypto/eth2_keystore" } diff --git a/account_manager/src/validator/deposit.rs b/account_manager/src/validator/deposit.rs index aa54039f8..b052fd1da 100644 --- a/account_manager/src/validator/deposit.rs +++ b/account_manager/src/validator/deposit.rs @@ -8,8 +8,9 @@ use slog::{info, Logger}; use std::path::PathBuf; use tokio::time::{delay_until, Duration, Instant}; use types::EthSpec; -use validator_dir::Manager as ValidatorManager; +use validator_dir::{Eth1DepositData, Manager as ValidatorManager, ValidatorDir}; use web3::{ + transports::Http, transports::Ipc, types::{Address, SyncInfo, SyncState, TransactionRequest, U256}, Transport, Web3, @@ -18,6 +19,7 @@ use web3::{ pub const CMD: &str = "deposit"; pub const VALIDATOR_FLAG: &str = "validator"; pub const ETH1_IPC_FLAG: &str = "eth1-ipc"; +pub const ETH1_HTTP_FLAG: &str = "eth1-http"; pub const FROM_ADDRESS_FLAG: &str = "from-address"; const GWEI: u64 = 1_000_000_000; @@ -64,7 +66,15 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .value_name("ETH1_IPC_PATH") .help("Path to an Eth1 JSON-RPC IPC endpoint") .takes_value(true) - .required(true), + .required(false), + ) + .arg( + Arg::with_name(ETH1_HTTP_FLAG) + .long(ETH1_HTTP_FLAG) + .value_name("ETH1_HTTP_URL") + .help("URL to an Eth1 JSON-RPC endpoint") + .takes_value(true) + .required(false), ) .arg( Arg::with_name(FROM_ADDRESS_FLAG) @@ -79,11 +89,65 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { ) } +fn send_deposit_transactions( + mut env: Environment, + log: Logger, + eth1_deposit_datas: Vec<(ValidatorDir, Eth1DepositData)>, + from_address: Address, + deposit_contract: Address, + transport: T2, +) -> Result<(), String> +where + T1: EthSpec, + T2: Transport + std::marker::Send, + ::Out: std::marker::Send, +{ + let web3 = Web3::new(transport); + + let deposits_fut = async { + poll_until_synced(web3.clone(), log.clone()).await?; + + for (mut validator_dir, eth1_deposit_data) in eth1_deposit_datas { + let tx_hash = web3 + .eth() + .send_transaction(TransactionRequest { + from: from_address, + to: Some(deposit_contract), + gas: Some(DEPOSIT_GAS.into()), + gas_price: None, + value: Some(from_gwei(eth1_deposit_data.deposit_data.amount)), + data: Some(eth1_deposit_data.rlp.into()), + nonce: None, + condition: None, + }) + .compat() + .await + .map_err(|e| format!("Failed to send transaction: {:?}", e))?; + + info!( + log, + "Submitted deposit"; + "tx_hash" => format!("{:?}", tx_hash), + ); + + validator_dir + .save_eth1_deposit_tx_hash(&format!("{:?}", tx_hash)) + .map_err(|e| format!("Failed to save tx hash {:?} to disk: {:?}", tx_hash, e))?; + } + + Ok::<(), String>(()) + }; + + env.runtime().block_on(deposits_fut)?; + + Ok(()) +} + pub fn cli_run( matches: &ArgMatches<'_>, mut env: Environment, ) -> Result<(), String> { - let log = env.core_context().log; + let log = env.core_context().log().clone(); let data_dir = clap_utils::parse_path_with_default_in_home_dir( matches, @@ -91,7 +155,8 @@ pub fn cli_run( PathBuf::new().join(".lighthouse").join("validators"), )?; let validator: String = clap_utils::parse_required(matches, VALIDATOR_FLAG)?; - let eth1_ipc_path: PathBuf = clap_utils::parse_required(matches, ETH1_IPC_FLAG)?; + let eth1_ipc_path: Option = clap_utils::parse_optional(matches, ETH1_IPC_FLAG)?; + let eth1_http_url: Option = clap_utils::parse_optional(matches, ETH1_HTTP_FLAG)?; let from_address: Address = clap_utils::parse_required(matches, FROM_ADDRESS_FLAG)?; let manager = ValidatorManager::open(&data_dir) @@ -167,41 +232,40 @@ pub fn cli_run( return Err("Refusing to deposit to the zero address. Check testnet configuration.".into()); } - let (_event_loop_handle, transport) = - Ipc::new(eth1_ipc_path).map_err(|e| format!("Unable to connect to eth1 IPC: {:?}", e))?; - let web3 = Web3::new(transport); - - let deposits_fut = async { - poll_until_synced(web3.clone(), log.clone()).await?; - - for (mut validator_dir, eth1_deposit_data) in eth1_deposit_datas { - let tx_hash = web3 - .eth() - .send_transaction(TransactionRequest { - from: from_address, - to: Some(deposit_contract), - gas: Some(DEPOSIT_GAS.into()), - gas_price: None, - value: Some(from_gwei(eth1_deposit_data.deposit_data.amount)), - data: Some(eth1_deposit_data.rlp.into()), - nonce: None, - condition: None, - }) - .compat() - .await - .map_err(|e| format!("Failed to send transaction: {:?}", e))?; - - validator_dir - .save_eth1_deposit_tx_hash(&format!("{:?}", tx_hash)) - .map_err(|e| format!("Failed to save tx hash {:?} to disk: {:?}", tx_hash, e))?; + match (eth1_ipc_path, eth1_http_url) { + (Some(_), Some(_)) => Err(format!( + "error: Cannot supply both --{} and --{}", + ETH1_IPC_FLAG, ETH1_HTTP_FLAG + )), + (None, None) => Err(format!( + "error: Must supply one of --{} or --{}", + ETH1_IPC_FLAG, ETH1_HTTP_FLAG + )), + (Some(ipc_path), None) => { + let (_event_loop_handle, ipc_transport) = Ipc::new(ipc_path) + .map_err(|e| format!("Unable to connect to eth1 IPC: {:?}", e))?; + send_deposit_transactions( + env, + log, + eth1_deposit_datas, + from_address, + deposit_contract, + ipc_transport, + ) } - - Ok::<(), String>(()) - }; - - env.runtime().block_on(deposits_fut)?; - - Ok(()) + (None, Some(http_url)) => { + let (_event_loop_handle, http_transport) = Http::new(http_url.as_str()) + .map_err(|e| format!("Unable to connect to eth1 http RPC: {:?}", e))?; + send_deposit_transactions( + env, + log, + eth1_deposit_datas, + from_address, + deposit_contract, + http_transport, + ) + } + } } /// Converts gwei to wei. diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 6ffa70df8..b58b1ce51 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -27,7 +27,7 @@ slog = { version = "2.5.2", features = ["max_level_trace", "release_max_level_tr slog-term = "2.5.0" slog-async = "2.5.0" ctrlc = { version = "3.1.4", features = ["termination"] } -tokio = {version = "0.2.20", features = ["time"] } +tokio = { version = "0.2.21", features = ["time"] } exit-future = "0.2.0" env_logger = "0.7.1" dirs = "2.0.2" diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index 796ca4b6d..262cc485c 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -33,7 +33,7 @@ eth2_ssz_derive = "0.1.0" state_processing = { path = "../../consensus/state_processing" } tree_hash = "0.1.0" types = { path = "../../consensus/types" } -tokio = "0.2.20" +tokio = "0.2.21" eth1 = { path = "../eth1" } websocket_server = { path = "../websocket_server" } futures = "0.3.5" @@ -41,12 +41,14 @@ genesis = { path = "../genesis" } integer-sqrt = "0.1.3" rand = "0.7.3" proto_array_fork_choice = { path = "../../consensus/proto_array_fork_choice" } -lru = "0.4.3" +lru = "0.5.1" tempfile = "3.1.0" bitvec = "0.17.4" bls = { path = "../../crypto/bls" } safe_arith = { path = "../../consensus/safe_arith" } +environment = { path = "../../lighthouse/environment" } +bus = "2.2.3" [dev-dependencies] lazy_static = "1.4.0" -environment = { path = "../../lighthouse/environment" } + diff --git a/beacon_node/beacon_chain/src/eth1_chain.rs b/beacon_node/beacon_chain/src/eth1_chain.rs index 53a3104f1..213344413 100644 --- a/beacon_node/beacon_chain/src/eth1_chain.rs +++ b/beacon_node/beacon_chain/src/eth1_chain.rs @@ -1,4 +1,5 @@ use crate::metrics; +use environment::TaskExecutor; use eth1::{Config as Eth1Config, Eth1Block, Service as HttpService}; use eth2_hashing::hash; use slog::{debug, error, trace, Logger}; @@ -285,10 +286,8 @@ impl> CachingEth1Backend { } /// Starts the routine which connects to the external eth1 node and updates the caches. - pub fn start(&self, exit: tokio::sync::oneshot::Receiver<()>) { - // don't need to spawn as a task is being spawned in auto_update - // TODO: check if this is correct - HttpService::auto_update(self.core.clone(), exit); + pub fn start(&self, handle: TaskExecutor) { + HttpService::auto_update(self.core.clone(), handle); } /// Instantiates `self` from an existing service. diff --git a/beacon_node/beacon_chain/src/events.rs b/beacon_node/beacon_chain/src/events.rs index d293bcb50..441d63be1 100644 --- a/beacon_node/beacon_chain/src/events.rs +++ b/beacon_node/beacon_chain/src/events.rs @@ -1,6 +1,10 @@ +use bus::Bus; +use parking_lot::Mutex; use serde_derive::{Deserialize, Serialize}; +use slog::{error, Logger}; use std::marker::PhantomData; -use types::{Attestation, Epoch, EthSpec, Hash256, SignedBeaconBlock}; +use std::sync::Arc; +use types::{Attestation, Epoch, EthSpec, Hash256, SignedBeaconBlock, SignedBeaconBlockHash}; pub use websocket_server::WebSocketSender; pub trait EventHandler: Sized + Send + Sync { @@ -18,6 +22,80 @@ impl EventHandler for WebSocketSender { } } +pub struct ServerSentEvents { + // Bus<> is itself Sync + Send. We use Mutex<> here only because of the surrounding code does + // not enforce mutability statically (i.e. relies on interior mutability). + head_changed_queue: Arc>>, + log: Logger, + _phantom: PhantomData, +} + +impl ServerSentEvents { + pub fn new(log: Logger) -> (Self, Arc>>) { + let bus = Bus::new(T::slots_per_epoch() as usize); + let mutex = Mutex::new(bus); + let arc = Arc::new(mutex); + let this = Self { + head_changed_queue: arc.clone(), + log: log, + _phantom: PhantomData, + }; + (this, arc) + } +} + +impl EventHandler for ServerSentEvents { + fn register(&self, kind: EventKind) -> Result<(), String> { + match kind { + EventKind::BeaconHeadChanged { + current_head_beacon_block_root, + .. + } => { + let mut guard = self.head_changed_queue.lock(); + if let Err(_) = guard.try_broadcast(current_head_beacon_block_root.into()) { + error!( + self.log, + "Head change streaming queue full"; + "dropped_change" => format!("{}", current_head_beacon_block_root), + ); + } + Ok(()) + } + _ => Ok(()), + } + } +} + +// An event handler that pushes events to both the websockets handler and the SSE handler. +// Named after the unix `tee` command. Meant as a temporary solution before ditching WebSockets +// completely once SSE functions well enough. +pub struct TeeEventHandler { + websockets_handler: WebSocketSender, + sse_handler: ServerSentEvents, +} + +impl TeeEventHandler { + pub fn new( + log: Logger, + websockets_handler: WebSocketSender, + ) -> Result<(Self, Arc>>), String> { + let (sse_handler, bus) = ServerSentEvents::new(log); + let result = Self { + websockets_handler: websockets_handler, + sse_handler: sse_handler, + }; + Ok((result, bus)) + } +} + +impl EventHandler for TeeEventHandler { + fn register(&self, kind: EventKind) -> Result<(), String> { + self.websockets_handler.register(kind.clone())?; + self.sse_handler.register(kind)?; + Ok(()) + } +} + impl EventHandler for NullEventHandler { fn register(&self, _kind: EventKind) -> Result<(), String> { Ok(()) @@ -30,7 +108,7 @@ impl Default for NullEventHandler { } } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] #[serde( bound = "T: EthSpec", rename_all = "snake_case", diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index 144d8f0c4..1b9ef0ab8 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -28,7 +28,7 @@ error-chain = "0.12.2" serde_yaml = "0.8.11" slog = { version = "2.5.2", features = ["max_level_trace"] } slog-async = "2.5.0" -tokio = "0.2.20" +tokio = "0.2.21" dirs = "2.0.2" futures = "0.3.5" reqwest = "0.10.4" @@ -40,3 +40,4 @@ eth2_ssz = "0.1.2" lazy_static = "1.4.0" lighthouse_metrics = { path = "../../common/lighthouse_metrics" } time = "0.2.16" +bus = "2.2.3" diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index 717471ec6..7c29e0f76 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -1,6 +1,7 @@ use crate::config::{ClientGenesis, Config as ClientConfig}; use crate::notifier::spawn_notifier; use crate::Client; +use beacon_chain::events::TeeEventHandler; use beacon_chain::{ builder::{BeaconChainBuilder, Witness}, eth1_chain::{CachingEth1Backend, Eth1Chain}, @@ -9,20 +10,26 @@ use beacon_chain::{ store::{HotColdDB, MemoryStore, Store, StoreConfig}, BeaconChain, BeaconChainTypes, Eth1ChainBackend, EventHandler, }; +use bus::Bus; use environment::RuntimeContext; use eth1::{Config as Eth1Config, Service as Eth1Service}; use eth2_config::Eth2Config; use eth2_libp2p::NetworkGlobals; use genesis::{interop_genesis_state, Eth1GenesisService}; use network::{NetworkConfig, NetworkMessage, NetworkService}; +use parking_lot::Mutex; use slog::info; use ssz::Decode; use std::net::SocketAddr; use std::path::Path; use std::sync::Arc; use std::time::Duration; +use timer::spawn_timer; use tokio::sync::mpsc::UnboundedSender; -use types::{test_utils::generate_deterministic_keypairs, BeaconState, ChainSpec, EthSpec}; +use types::{ + test_utils::generate_deterministic_keypairs, BeaconState, ChainSpec, EthSpec, + SignedBeaconBlockHash, +}; use websocket_server::{Config as WebSocketConfig, WebSocketSender}; /// Interval between polling the eth1 node for genesis information. @@ -50,7 +57,6 @@ pub struct ClientBuilder { beacon_chain_builder: Option>, beacon_chain: Option>>, eth1_service: Option, - exit_channels: Vec>, event_handler: Option, network_globals: Option>>, network_send: Option>>, @@ -84,7 +90,6 @@ where beacon_chain_builder: None, beacon_chain: None, eth1_service: None, - exit_channels: vec![], event_handler: None, network_globals: None, network_send: None, @@ -132,7 +137,7 @@ where .ok_or_else(|| "beacon_chain_start_method requires a chain spec".to_string())?; let builder = BeaconChainBuilder::new(eth_spec_instance) - .logger(context.log.clone()) + .logger(context.log().clone()) .store(store) .store_migrator(store_migrator) .data_dir(data_dir) @@ -150,7 +155,7 @@ where // Alternatively, if there's a beacon chain in the database then always resume // using it. let client_genesis = if client_genesis == ClientGenesis::FromStore && !chain_exists { - info!(context.log, "Defaulting to deposit contract genesis"); + info!(context.log(), "Defaulting to deposit contract genesis"); ClientGenesis::DepositContract } else if chain_exists { @@ -172,7 +177,7 @@ where genesis_state_bytes, } => { info!( - context.log, + context.log(), "Starting from known genesis state"; ); @@ -183,14 +188,14 @@ where } ClientGenesis::DepositContract => { info!( - context.log, + context.log(), "Waiting for eth2 genesis from eth1"; "eth1_endpoint" => &config.eth1.endpoint, "contract_deploy_block" => config.eth1.deposit_contract_deploy_block, "deposit_contract" => &config.eth1.deposit_contract_address ); - let genesis_service = Eth1GenesisService::new(config.eth1, context.log.clone()); + let genesis_service = Eth1GenesisService::new(config.eth1, context.log().clone()); let genesis_state = genesis_service .wait_for_genesis_state( @@ -223,19 +228,18 @@ where .ok_or_else(|| "network requires a runtime_context")? .clone(); - let (network_globals, network_send, network_exit) = - NetworkService::start(beacon_chain, config, &context.runtime_handle, context.log) + let (network_globals, network_send) = + NetworkService::start(beacon_chain, config, context.executor) .map_err(|e| format!("Failed to start network: {:?}", e))?; self.network_globals = Some(network_globals); self.network_send = Some(network_send); - self.exit_channels.push(network_exit); Ok(self) } /// Immediately starts the timer service. - fn timer(mut self) -> Result { + fn timer(self) -> Result { let context = self .runtime_context .as_ref() @@ -251,13 +255,9 @@ where .ok_or_else(|| "node timer requires a chain spec".to_string())? .milliseconds_per_slot; - let timer_exit = context - .runtime_handle - .enter(|| timer::spawn(beacon_chain, milliseconds_per_slot)) + spawn_timer(context.executor, beacon_chain, milliseconds_per_slot) .map_err(|e| format!("Unable to start node timer: {}", e))?; - self.exit_channels.push(timer_exit); - Ok(self) } @@ -266,6 +266,7 @@ where mut self, client_config: &ClientConfig, eth2_config: &Eth2Config, + events: Arc>>, ) -> Result { let beacon_chain = self .beacon_chain @@ -290,32 +291,29 @@ where network_chan: network_send, }; - let log = context.log.clone(); - let (exit_channel, listening_addr) = context.runtime_handle.enter(|| { - rest_api::start_server( - &client_config.rest_api, - beacon_chain, - network_info, - client_config - .create_db_path() - .map_err(|_| "unable to read data dir")?, - client_config - .create_freezer_db_path() - .map_err(|_| "unable to read freezer DB dir")?, - eth2_config.clone(), - log, - ) - .map_err(|e| format!("Failed to start HTTP API: {:?}", e)) - })?; + let listening_addr = rest_api::start_server( + context.executor, + &client_config.rest_api, + beacon_chain, + network_info, + client_config + .create_db_path() + .map_err(|_| "unable to read data dir")?, + client_config + .create_freezer_db_path() + .map_err(|_| "unable to read freezer DB dir")?, + eth2_config.clone(), + events, + ) + .map_err(|e| format!("Failed to start HTTP API: {:?}", e))?; - self.exit_channels.push(exit_channel); self.http_listen_addr = Some(listening_addr); Ok(self) } /// Immediately starts the service that periodically logs information each slot. - pub fn notifier(mut self) -> Result { + pub fn notifier(self) -> Result { let context = self .runtime_context .as_ref() @@ -335,19 +333,13 @@ where .ok_or_else(|| "slot_notifier requires a chain spec".to_string())? .milliseconds_per_slot; - let exit_channel = context - .runtime_handle - .enter(|| { - spawn_notifier( - beacon_chain, - network_globals, - milliseconds_per_slot, - context.log.clone(), - ) - }) - .map_err(|e| format!("Unable to start slot notifier: {}", e))?; - - self.exit_channels.push(exit_channel); + spawn_notifier( + context.executor, + beacon_chain, + network_globals, + milliseconds_per_slot, + ) + .map_err(|e| format!("Unable to start slot notifier: {}", e))?; Ok(self) } @@ -365,7 +357,6 @@ where network_globals: self.network_globals, http_listen_addr: self.http_listen_addr, websocket_listen_addr: self.websocket_listen_addr, - _exit_channels: self.exit_channels, } } } @@ -436,22 +427,14 @@ where .ok_or_else(|| "websocket_event_handler requires a runtime_context")? .service_context("ws".into()); - let (sender, exit_channel, listening_addr): ( - WebSocketSender, - Option<_>, - Option<_>, - ) = if config.enabled { - let (sender, exit, listening_addr) = context - .runtime_handle - .enter(|| websocket_server::start_server(&config, &context.log))?; - (sender, Some(exit), Some(listening_addr)) + let (sender, listening_addr): (WebSocketSender, Option<_>) = if config.enabled { + let (sender, listening_addr) = + websocket_server::start_server(context.executor, &config)?; + (sender, Some(listening_addr)) } else { - (WebSocketSender::dummy(), None, None) + (WebSocketSender::dummy(), None) }; - if let Some(channel) = exit_channel { - self.exit_channels.push(channel); - } self.event_handler = Some(sender); self.websocket_listen_addr = listening_addr; @@ -459,6 +442,51 @@ where } } +impl + ClientBuilder< + Witness< + TStore, + TStoreMigrator, + TSlotClock, + TEth1Backend, + TEthSpec, + TeeEventHandler, + >, + > +where + TStore: Store + 'static, + TStoreMigrator: Migrate, + TSlotClock: SlotClock + 'static, + TEth1Backend: Eth1ChainBackend + 'static, + TEthSpec: EthSpec + 'static, +{ + /// Specifies that the `BeaconChain` should publish events using the WebSocket server. + pub fn tee_event_handler( + mut self, + config: WebSocketConfig, + ) -> Result<(Self, Arc>>), String> { + let context = self + .runtime_context + .as_ref() + .ok_or_else(|| "websocket_event_handler requires a runtime_context")? + .service_context("ws".into()); + + let log = context.log().clone(); + let (sender, listening_addr): (WebSocketSender, Option<_>) = if config.enabled { + let (sender, listening_addr) = + websocket_server::start_server(context.executor, &config)?; + (sender, Some(listening_addr)) + } else { + (WebSocketSender::dummy(), None) + }; + + self.websocket_listen_addr = listening_addr; + let (tee_event_handler, bus) = TeeEventHandler::new(log, sender)?; + self.event_handler = Some(tee_event_handler); + Ok((self, bus)) + } +} + impl ClientBuilder< Witness< @@ -494,7 +522,7 @@ where .clone() .ok_or_else(|| "disk_store requires a chain spec".to_string())?; - let store = HotColdDB::open(hot_path, cold_path, config, spec, context.log) + let store = HotColdDB::open(hot_path, cold_path, config, spec, context.log().clone()) .map_err(|e| format!("Unable to open database: {:?}", e))?; self.store = Some(Arc::new(store)); Ok(self) @@ -555,7 +583,7 @@ where let store = self.store.clone().ok_or_else(|| { "background_migrator requires the store to be initialized".to_string() })?; - self.store_migrator = Some(BackgroundMigrator::new(store, context.log.clone())); + self.store_migrator = Some(BackgroundMigrator::new(store, context.log().clone())); Ok(self) } } @@ -617,25 +645,23 @@ where &persisted, config.clone(), store.clone(), - &context.log, + &context.log().clone(), ) .map(|chain| chain.into_backend()) }) .unwrap_or_else(|| { - Ok(CachingEth1Backend::new(config, context.log.clone(), store)) + Ok(CachingEth1Backend::new( + config, + context.log().clone(), + store, + )) })? }; self.eth1_service = None; - let exit = { - let (tx, rx) = tokio::sync::oneshot::channel(); - self.exit_channels.push(tx); - rx - }; - // Starts the service that connects to an eth1 node and periodically updates caches. - context.runtime_handle.enter(|| backend.start(exit)); + backend.start(context.executor); self.beacon_chain_builder = Some(beacon_chain_builder.eth1_backend(Some(backend))); diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index 7f665b9cb..da670ff13 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -25,8 +25,6 @@ pub struct Client { network_globals: Option>>, http_listen_addr: Option, websocket_listen_addr: Option, - /// Exit channels will complete/error when dropped, causing each service to exit gracefully. - _exit_channels: Vec>, } impl Client { diff --git a/beacon_node/client/src/notifier.rs b/beacon_node/client/src/notifier.rs index 9375de0a1..644e13c94 100644 --- a/beacon_node/client/src/notifier.rs +++ b/beacon_node/client/src/notifier.rs @@ -23,11 +23,11 @@ const SPEEDO_OBSERVATIONS: usize = 4; /// Spawns a notifier service which periodically logs information about the node. pub fn spawn_notifier( + executor: environment::TaskExecutor, beacon_chain: Arc>, network: Arc>, milliseconds_per_slot: u64, - log: slog::Logger, -) -> Result, String> { +) -> Result<(), String> { let slot_duration = Duration::from_millis(milliseconds_per_slot); let duration_to_next_slot = beacon_chain .slot_clock @@ -41,6 +41,7 @@ pub fn spawn_notifier( let interval_duration = slot_duration; let speedo = Mutex::new(Speedo::default()); + let log = executor.log().clone(); let mut interval = tokio::time::interval_at(start_instant, interval_duration); let interval_future = async move { @@ -163,12 +164,10 @@ pub fn spawn_notifier( Ok::<(), ()>(()) }; - let (exit_signal, exit) = tokio::sync::oneshot::channel(); - // run the notifier on the current executor - tokio::spawn(futures::future::select(Box::pin(interval_future), exit)); + executor.spawn(interval_future.unwrap_or_else(|_| ()), "notifier"); - Ok(exit_signal) + Ok(()) } /// Returns the peer count, returning something helpful if it's `usize::max_value` (effectively a diff --git a/beacon_node/eth1/Cargo.toml b/beacon_node/eth1/Cargo.toml index 695bdb1cb..ba211a4e8 100644 --- a/beacon_node/eth1/Cargo.toml +++ b/beacon_node/eth1/Cargo.toml @@ -6,7 +6,6 @@ edition = "2018" [dev-dependencies] eth1_test_rig = { path = "../../testing/eth1_test_rig" } -environment = { path = "../../lighthouse/environment" } toml = "0.5.6" web3 = "0.11.0" sloggers = "1.0.0" @@ -25,8 +24,9 @@ tree_hash = "0.1.0" eth2_hashing = "0.1.0" parking_lot = "0.10.2" slog = "2.5.2" -tokio = { version = "0.2.20", features = ["full"] } +tokio = { version = "0.2.21", features = ["full"] } state_processing = { path = "../../consensus/state_processing" } libflate = "1.0.0" lighthouse_metrics = { path = "../../common/lighthouse_metrics"} lazy_static = "1.4.0" +environment = { path = "../../lighthouse/environment" } diff --git a/beacon_node/eth1/src/service.rs b/beacon_node/eth1/src/service.rs index 8f8e7db13..8c299251d 100644 --- a/beacon_node/eth1/src/service.rs +++ b/beacon_node/eth1/src/service.rs @@ -290,7 +290,7 @@ impl Service { /// - Err(_) if there is an error. /// /// Emits logs for debugging and errors. - pub fn auto_update(service: Self, exit: tokio::sync::oneshot::Receiver<()>) { + pub fn auto_update(service: Self, handle: environment::TaskExecutor) { let update_interval = Duration::from_millis(service.config().auto_update_interval_millis); let mut interval = interval_at(Instant::now(), update_interval); @@ -303,9 +303,7 @@ impl Service { } }; - let future = futures::future::select(Box::pin(update_future), exit); - - tokio::task::spawn(future); + handle.spawn(update_future, "eth1"); } async fn do_update(service: Self, update_interval: Duration) -> Result<(), ()> { diff --git a/beacon_node/eth2-libp2p/Cargo.toml b/beacon_node/eth2-libp2p/Cargo.toml index e3a3164d0..8b64e6ad9 100644 --- a/beacon_node/eth2-libp2p/Cargo.toml +++ b/beacon_node/eth2-libp2p/Cargo.toml @@ -24,7 +24,7 @@ unsigned-varint = { git = "https://github.com/sigp/unsigned-varint", branch = "l lazy_static = "1.4.0" lighthouse_metrics = { path = "../../common/lighthouse_metrics" } smallvec = "1.4.0" -lru = "0.4.5" +lru = "0.5.1" parking_lot = "0.10.2" sha2 = "0.8.2" base64 = "0.12.1" @@ -35,6 +35,7 @@ tokio-util = { version = "0.3.1", features = ["codec", "compat"] } # Patched for quick updates discv5 = { git = "https://github.com/sigp/discv5", rev = "7b3bd40591b62b8c002ffdb85de008aa9f82e2e5" } tiny-keccak = "2.0.2" +environment = { path = "../../lighthouse/environment" } libp2p-tcp = { version = "0.19.1", default-features = false, features = ["tokio"] } [dependencies.libp2p] @@ -49,3 +50,4 @@ slog-stdlog = "4.0.0" slog-term = "2.5.0" slog-async = "2.5.0" tempdir = "0.3.7" +exit-future = "0.2.0" diff --git a/beacon_node/eth2-libp2p/src/behaviour/mod.rs b/beacon_node/eth2-libp2p/src/behaviour/mod.rs index 3316364ca..aadc9cbc2 100644 --- a/beacon_node/eth2-libp2p/src/behaviour/mod.rs +++ b/beacon_node/eth2-libp2p/src/behaviour/mod.rs @@ -25,8 +25,9 @@ use std::{ marker::PhantomData, sync::Arc, task::{Context, Poll}, + time::Instant, }; -use types::{EnrForkId, EthSpec, SubnetId}; +use types::{EnrForkId, EthSpec, SignedBeaconBlock, SubnetId}; mod handler; @@ -393,8 +394,36 @@ impl Behaviour { /* Eth2 RPC behaviour functions */ + /// Send a request to a peer over RPC. + pub fn send_request(&mut self, peer_id: PeerId, request_id: RequestId, request: Request) { + self.send_rpc(peer_id, RPCSend::Request(request_id, request.into())) + } + + /// Send a successful response to a peer over RPC. + pub fn send_successful_response( + &mut self, + peer_id: PeerId, + stream_id: SubstreamId, + response: Response, + ) { + self.send_rpc(peer_id, RPCSend::Response(stream_id, response.into())) + } + + /// Inform the peer that their request produced an error. + pub fn _send_error_reponse( + &mut self, + peer_id: PeerId, + stream_id: SubstreamId, + error: RPCResponseErrorCode, + reason: String, + ) { + self.send_rpc( + peer_id, + RPCSend::Response(stream_id, RPCCodedResponse::from_error_code(error, reason)), + ) + } /// Sends an RPC Request/Response via the RPC protocol. - pub fn send_rpc(&mut self, peer_id: PeerId, rpc_event: RPCEvent) { + fn send_rpc(&mut self, peer_id: PeerId, rpc_event: RPCSend) { self.eth2_rpc.send_rpc(peer_id, rpc_event); } @@ -431,9 +460,10 @@ impl Behaviour { self.update_metadata(); } - /// A request to search for peers connected to a long-lived subnet. - pub fn peers_request(&mut self, subnet_id: SubnetId) { - self.discovery.peers_request(subnet_id); + /// Attempts to discover new peers for a given subnet. The `min_ttl` gives the time at which we + /// would like to retain the peers for. + pub fn discover_subnet_peers(&mut self, subnet_id: SubnetId, min_ttl: Option) { + self.discovery.discover_subnet_peers(subnet_id, min_ttl) } /// Updates the local ENR's "eth2" field with the latest EnrForkId. @@ -476,32 +506,38 @@ impl Behaviour { .expect("Local discovery must have bitfield"); } - /// Sends a PING/PONG request/response to a peer. - fn send_ping(&mut self, id: RequestId, peer_id: PeerId, is_request: bool) { - let ping = crate::rpc::methods::Ping { + /// Sends a Ping request to the peer. + fn ping(&mut self, id: RequestId, peer_id: PeerId) { + let ping = crate::rpc::Ping { data: self.meta_data.seq_number, }; + debug!(self.log, "Sending Ping"; "request_id" => id, "peer_id" => peer_id.to_string()); + let event = RPCSend::Request(id, RPCRequest::Ping(ping)); - let event = if is_request { - debug!(self.log, "Sending Ping"; "request_id" => id, "peer_id" => peer_id.to_string()); - RPCEvent::Request(id, RPCRequest::Ping(ping)) - } else { - debug!(self.log, "Sending Pong"; "request_id" => id, "peer_id" => peer_id.to_string()); - RPCEvent::Response(id, RPCCodedResponse::Success(RPCResponse::Pong(ping))) + self.send_rpc(peer_id, event); + } + + /// Sends a Pong response to the peer. + fn pong(&mut self, id: SubstreamId, peer_id: PeerId) { + let ping = crate::rpc::Ping { + data: self.meta_data.seq_number, }; + debug!(self.log, "Sending Pong"; "request_id" => id, "peer_id" => peer_id.to_string()); + let event = RPCSend::Response(id, RPCCodedResponse::Success(RPCResponse::Pong(ping))); + self.send_rpc(peer_id, event); } /// Sends a METADATA request to a peer. fn send_meta_data_request(&mut self, peer_id: PeerId) { let metadata_request = - RPCEvent::Request(RequestId::from(0usize), RPCRequest::MetaData(PhantomData)); + RPCSend::Request(RequestId::Behaviour, RPCRequest::MetaData(PhantomData)); self.send_rpc(peer_id, metadata_request); } /// Sends a METADATA response to a peer. - fn send_meta_data_response(&mut self, id: RequestId, peer_id: PeerId) { - let metadata_response = RPCEvent::Response( + fn send_meta_data_response(&mut self, id: SubstreamId, peer_id: PeerId) { + let metadata_response = RPCSend::Response( id, RPCCodedResponse::Success(RPCResponse::MetaData(self.meta_data.clone())), ); @@ -587,45 +623,112 @@ impl Behaviour { } } + /// Queues the response to be sent upwards as long at it was requested outside the Behaviour. + fn propagate_response(&mut self, id: RequestId, peer_id: PeerId, response: Response) { + if !matches!(id, RequestId::Behaviour) { + self.events.push(BehaviourEvent::ResponseReceived { + peer_id, + id, + response, + }); + } + } + + /// Convenience function to propagate a request. + fn propagate_request(&mut self, id: SubstreamId, peer_id: PeerId, request: Request) { + self.events.push(BehaviourEvent::RequestReceived { + peer_id, + id, + request, + }); + } + fn on_rpc_event(&mut self, message: RPCMessage) { let peer_id = message.peer_id; - // The METADATA and PING RPC responses are handled within the behaviour and not - // propagated - // TODO: Improve the RPC types to better handle this logic discrepancy + // The METADATA and PING RPC responses are handled within the behaviour and not propagated match message.event { - RPCEvent::Request(id, RPCRequest::Ping(ping)) => { - // inform the peer manager and send the response - self.peer_manager.ping_request(&peer_id, ping.data); - // send a ping response - self.send_ping(id, peer_id, false); + Err(handler_err) => { + match handler_err { + HandlerErr::Inbound { + id: _, + proto, + error, + } => { + // Inform the peer manager of the error. + // An inbound error here means we sent an error to the peer, or the stream + // timed out. + self.peer_manager.handle_rpc_error(&peer_id, proto, &error); + } + HandlerErr::Outbound { id, proto, error } => { + // Inform the peer manager that a request we sent to the peer failed + self.peer_manager.handle_rpc_error(&peer_id, proto, &error); + // inform failures of requests comming outside the behaviour + if !matches!(id, RequestId::Behaviour) { + self.events + .push(BehaviourEvent::RPCFailed { peer_id, id, error }); + } + } + } } - RPCEvent::Request(id, RPCRequest::MetaData(_)) => { - // send the requested meta-data - self.send_meta_data_response(id, peer_id); + Ok(RPCReceived::Request(id, request)) => match request { + /* Behaviour managed protocols: Ping and Metadata */ + RPCRequest::Ping(ping) => { + // inform the peer manager and send the response + self.peer_manager.ping_request(&peer_id, ping.data); + // send a ping response + self.pong(id, peer_id); + } + RPCRequest::MetaData(_) => { + // send the requested meta-data + self.send_meta_data_response(id, peer_id); + // TODO: inform the peer manager? + } + /* Protocols propagated to the Network */ + RPCRequest::Status(msg) => { + // inform the peer manager that we have received a status from a peer + self.peer_manager.peer_statusd(&peer_id); + // propagate the STATUS message upwards + self.propagate_request(id, peer_id, Request::Status(msg)) + } + RPCRequest::BlocksByRange(req) => { + self.propagate_request(id, peer_id, Request::BlocksByRange(req)) + } + RPCRequest::BlocksByRoot(req) => { + self.propagate_request(id, peer_id, Request::BlocksByRoot(req)) + } + RPCRequest::Goodbye(reason) => { + // TODO: do not propagate + self.propagate_request(id, peer_id, Request::Goodbye(reason)); + } + }, + Ok(RPCReceived::Response(id, resp)) => { + match resp { + /* Behaviour managed protocols */ + RPCResponse::Pong(ping) => self.peer_manager.pong_response(&peer_id, ping.data), + RPCResponse::MetaData(meta_data) => { + self.peer_manager.meta_data_response(&peer_id, meta_data) + } + /* Network propagated protocols */ + RPCResponse::Status(msg) => { + // inform the peer manager that we have received a status from a peer + self.peer_manager.peer_statusd(&peer_id); + // propagate the STATUS message upwards + self.propagate_response(id, peer_id, Response::Status(msg)); + } + RPCResponse::BlocksByRange(resp) => { + self.propagate_response(id, peer_id, Response::BlocksByRange(Some(resp))) + } + RPCResponse::BlocksByRoot(resp) => { + self.propagate_response(id, peer_id, Response::BlocksByRoot(Some(resp))) + } + } } - RPCEvent::Response(_, RPCCodedResponse::Success(RPCResponse::Pong(ping))) => { - self.peer_manager.pong_response(&peer_id, ping.data); - } - RPCEvent::Response(_, RPCCodedResponse::Success(RPCResponse::MetaData(meta_data))) => { - self.peer_manager.meta_data_response(&peer_id, meta_data); - } - RPCEvent::Request(_, RPCRequest::Status(_)) - | RPCEvent::Response(_, RPCCodedResponse::Success(RPCResponse::Status(_))) => { - // inform the peer manager that we have received a status from a peer - self.peer_manager.peer_statusd(&peer_id); - // propagate the STATUS message upwards - self.events - .push(BehaviourEvent::RPC(peer_id, message.event)); - } - RPCEvent::Error(_, protocol, ref err) => { - self.peer_manager.handle_rpc_error(&peer_id, protocol, err); - self.events - .push(BehaviourEvent::RPC(peer_id, message.event)); - } - _ => { - // propagate all other RPC messages upwards - self.events - .push(BehaviourEvent::RPC(peer_id, message.event)) + Ok(RPCReceived::EndOfStream(id, termination)) => { + let response = match termination { + ResponseTermination::BlocksByRange => Response::BlocksByRange(None), + ResponseTermination::BlocksByRoot => Response::BlocksByRoot(None), + }; + self.propagate_response(id, peer_id, response); } } } @@ -648,7 +751,7 @@ impl Behaviour { } PeerManagerEvent::Ping(peer_id) => { // send a ping request to this peer - self.send_ping(RequestId::from(0usize), peer_id, true); + self.ping(RequestId::Behaviour, peer_id); } PeerManagerEvent::MetaData(peer_id) => { self.send_meta_data_request(peer_id); @@ -707,11 +810,96 @@ impl Behaviour { } } +/* Public API types */ + +/// The type of RPC requests the Behaviour informs it has received and allows for sending. +/// +// NOTE: This is an application-level wrapper over the lower network leve requests that can be +// sent. The main difference is the absense of the Ping and Metadata protocols, which don't +// leave the Behaviour. For all protocols managed by RPC see `RPCRequest`. +#[derive(Debug, Clone, PartialEq)] +pub enum Request { + /// A Status message. + Status(StatusMessage), + /// A Goobye message. + Goodbye(GoodbyeReason), + /// A blocks by range request. + BlocksByRange(BlocksByRangeRequest), + /// A request blocks root request. + BlocksByRoot(BlocksByRootRequest), +} + +impl std::convert::From for RPCRequest { + fn from(req: Request) -> RPCRequest { + match req { + Request::BlocksByRoot(r) => RPCRequest::BlocksByRoot(r), + Request::BlocksByRange(r) => RPCRequest::BlocksByRange(r), + Request::Goodbye(r) => RPCRequest::Goodbye(r), + Request::Status(s) => RPCRequest::Status(s), + } + } +} + +/// The type of RPC responses the Behaviour informs it has received, and allows for sending. +/// +// NOTE: This is an application-level wrapper over the lower network level responses that can be +// sent. The main difference is the absense of Pong and Metadata, which don't leave the +// Behaviour. For all protocol reponses managed by RPC see `RPCResponse` and +// `RPCCodedResponse`. +#[derive(Debug, Clone, PartialEq)] +pub enum Response { + /// A Status message. + Status(StatusMessage), + /// A response to a get BLOCKS_BY_RANGE request. A None response signals the end of the batch. + BlocksByRange(Option>>), + /// A response to a get BLOCKS_BY_ROOT request. + BlocksByRoot(Option>>), +} + +impl std::convert::From> for RPCCodedResponse { + fn from(resp: Response) -> RPCCodedResponse { + match resp { + Response::BlocksByRoot(r) => match r { + Some(b) => RPCCodedResponse::Success(RPCResponse::BlocksByRoot(b)), + None => RPCCodedResponse::StreamTermination(ResponseTermination::BlocksByRoot), + }, + Response::BlocksByRange(r) => match r { + Some(b) => RPCCodedResponse::Success(RPCResponse::BlocksByRange(b)), + None => RPCCodedResponse::StreamTermination(ResponseTermination::BlocksByRange), + }, + Response::Status(s) => RPCCodedResponse::Success(RPCResponse::Status(s)), + } + } +} + /// The types of events than can be obtained from polling the behaviour. #[derive(Debug)] pub enum BehaviourEvent { - /// A received RPC event and the peer that it was received from. - RPC(PeerId, RPCEvent), + /// An RPC Request that was sent failed. + RPCFailed { + /// The id of the failed request. + id: RequestId, + /// The peer to which this request was sent. + peer_id: PeerId, + /// The error that occurred. + error: RPCError, + }, + RequestReceived { + /// The peer that sent the request. + peer_id: PeerId, + /// Identifier of the request. All responses to this request must use this id. + id: SubstreamId, + /// Request the peer sent. + request: Request, + }, + ResponseReceived { + /// Peer that sent the response. + peer_id: PeerId, + /// Id of the request to which the peer is responding. + id: RequestId, + /// Response the peer sent. + response: Response, + }, PubsubMessage { /// The gossipsub message id. Used when propagating blocks after validation. id: MessageId, diff --git a/beacon_node/eth2-libp2p/src/discovery/mod.rs b/beacon_node/eth2-libp2p/src/discovery/mod.rs index 7349fe5ba..136b00a5d 100644 --- a/beacon_node/eth2-libp2p/src/discovery/mod.rs +++ b/beacon_node/eth2-libp2p/src/discovery/mod.rs @@ -8,7 +8,7 @@ pub use enr_ext::{CombinedKeyExt, EnrExt}; use crate::metrics; use crate::{error, Enr, NetworkConfig, NetworkGlobals}; -use discv5::{enr::NodeId, Discv5, Discv5Event}; +use discv5::{enr::NodeId, Discv5, Discv5Event, QueryId}; use enr::{Eth2Enr, BITFIELD_ENR_KEY, ETH2_ENR_KEY}; use futures::prelude::*; use libp2p::core::{connection::ConnectionId, Multiaddr, PeerId}; @@ -18,20 +18,24 @@ use libp2p::swarm::{ NetworkBehaviourAction, PollParameters, ProtocolsHandler, }; use lru::LruCache; -use slog::{crit, debug, info, warn}; +use slog::{crit, debug, info, trace, warn}; use ssz::{Decode, Encode}; use ssz_types::BitVector; use std::{ - collections::{HashSet, VecDeque}, + collections::{HashMap, HashSet, VecDeque}, net::SocketAddr, path::Path, sync::Arc, task::{Context, Poll}, - time::Duration, + time::{Duration, Instant}, }; -use tokio::time::{delay_until, Delay, Instant}; +use tokio::time::{delay_until, Delay}; use types::{EnrForkId, EthSpec, SubnetId}; +mod subnet_predicate; + +use subnet_predicate::subnet_predicate; + /// Maximum seconds before searching for extra peers. const MAX_TIME_BETWEEN_PEER_SEARCHES: u64 = 120; /// Initial delay between peer searches. @@ -41,7 +45,18 @@ const MINIMUM_PEERS_BEFORE_DELAY_INCREASE: usize = 5; /// Local ENR storage filename. pub const ENR_FILENAME: &str = "enr.dat"; /// Number of peers we'd like to have connected to a given long-lived subnet. -const TARGET_SUBNET_PEERS: u64 = 3; +const TARGET_SUBNET_PEERS: usize = 3; +/// Number of times to attempt a discovery request +const MAX_DISCOVERY_RETRY: u64 = 3; + +/// A struct representing the information associated with a single discovery request, +/// which can be retried with multiple queries +#[derive(Clone, Debug)] +pub struct Request { + pub query_id: Option, + pub min_ttl: Option, + pub retries: u64, +} /// Lighthouse discovery behaviour. This provides peer management and discovery using the Discv5 /// libp2p protocol. @@ -79,6 +94,9 @@ pub struct Discovery { /// A collection of network constants that can be read from other threads. network_globals: Arc>, + /// A mapping of SubnetId that we are currently searching for to all information associated with each request. + subnet_queries: HashMap, + /// Logger for the discovery behaviour. log: slog::Logger, } @@ -139,11 +157,12 @@ impl Discovery { cached_enrs: LruCache::new(50), banned_peers: HashSet::new(), max_peers: config.max_peers, - peer_discovery_delay: delay_until(Instant::now()), + peer_discovery_delay: delay_until(tokio::time::Instant::now()), past_discovery_delay: INITIAL_SEARCH_DELAY, tcp_port: config.libp2p_port, discovery, network_globals, + subnet_queries: HashMap::new(), log, enr_dir, }) @@ -280,57 +299,93 @@ impl Discovery { } /// A request to find peers on a given subnet. - // TODO: This logic should be improved with added sophistication in peer management - // This currently checks for currently connected peers and if we don't have - // PEERS_WANTED_BEFORE_DISCOVERY connected to a given subnet we search for more. - pub fn peers_request(&mut self, subnet_id: SubnetId) { + pub fn discover_subnet_peers(&mut self, subnet_id: SubnetId, min_ttl: Option) { + // TODO: Extend this to an event once discovery becomes a thread managed by the peer + // manager + if let Some(min_ttl) = min_ttl { + self.network_globals + .peers + .write() + .extend_peers_on_subnet(subnet_id, min_ttl); + } + + // If there is already a discovery request in process for this subnet, ignore this request, + // but update the min_ttl. + if let Some(request) = self.subnet_queries.get_mut(&subnet_id) { + // update the min_ttl if required + if let Some(min_ttl) = min_ttl { + if request.min_ttl < Some(min_ttl) { + request.min_ttl = Some(min_ttl); + } + } + return; + } + + // Insert a request and start a query for the subnet + self.subnet_queries.insert( + subnet_id.clone(), + Request { + query_id: None, + min_ttl, + retries: 0, + }, + ); + self.run_subnet_query(subnet_id); + } + + /// Runs a discovery request for a given subnet_id if one already exists. + fn run_subnet_query(&mut self, subnet_id: SubnetId) { + let mut request = match self.subnet_queries.remove(&subnet_id) { + Some(v) => v, + None => return, // request doesn't exist + }; + + // increment the retry count + request.retries += 1; + let peers_on_subnet = self .network_globals .peers .read() .peers_on_subnet(subnet_id) - .count() as u64; + .count(); - if peers_on_subnet < TARGET_SUBNET_PEERS { - let target_peers = TARGET_SUBNET_PEERS - peers_on_subnet; - debug!(self.log, "Searching for peers for subnet"; - "subnet_id" => *subnet_id, - "connected_peers_on_subnet" => peers_on_subnet, - "target_subnet_peers" => TARGET_SUBNET_PEERS, - "peers_to_find" => target_peers - ); - - let log_clone = self.log.clone(); - - let subnet_predicate = move |enr: &Enr| { - if let Some(bitfield_bytes) = enr.get(BITFIELD_ENR_KEY) { - let bitfield = match BitVector::::from_ssz_bytes( - bitfield_bytes, - ) { - Ok(v) => v, - Err(e) => { - warn!(log_clone, "Could not decode ENR bitfield for peer"; "peer_id" => format!("{}", enr.peer_id()), "error" => format!("{:?}", e)); - return false; - } - }; - - return bitfield.get(*subnet_id as usize).unwrap_or_else(|_| { - debug!(log_clone, "Peer found but not on desired subnet"; "peer_id" => format!("{}", enr.peer_id())); - false - }); - } - false - }; - - // start the query - self.start_query(subnet_predicate, target_peers as usize); - } else { - debug!(self.log, "Discovery ignored"; + if peers_on_subnet > TARGET_SUBNET_PEERS { + trace!(self.log, "Discovery ignored"; "reason" => "Already connected to desired peers", "connected_peers_on_subnet" => peers_on_subnet, "target_subnet_peers" => TARGET_SUBNET_PEERS, ); + return; } + + // remove the entry and complete the query if greater than the maximum search count + if request.retries >= MAX_DISCOVERY_RETRY { + debug!( + self.log, + "Subnet peer discovery did not find sufficient peers. Reached max retry limit" + ); + return; + } + + let target_peers = TARGET_SUBNET_PEERS - peers_on_subnet; + debug!(self.log, "Searching for peers for subnet"; + "subnet_id" => *subnet_id, + "connected_peers_on_subnet" => peers_on_subnet, + "target_subnet_peers" => TARGET_SUBNET_PEERS, + "peers_to_find" => target_peers, + "attempt" => request.retries, + ); + + // start the query, and update the queries map if necessary + let subnet_predicate = subnet_predicate::(subnet_id, &self.log); + if let Some(query_id) = self.start_query(subnet_predicate, target_peers) { + request.query_id = Some(query_id); + } else { + // ENR is not present remove the query + return; + } + self.subnet_queries.insert(subnet_id, request); } /* Internal Functions */ @@ -348,7 +403,7 @@ impl Discovery { /// This can optionally search for peers for a given predicate. Regardless of the predicate /// given, this will only search for peers on the same enr_fork_id as specified in the local /// ENR. - fn start_query(&mut self, enr_predicate: F, num_nodes: usize) + fn start_query(&mut self, enr_predicate: F, num_nodes: usize) -> Option where F: Fn(&Enr) -> bool + Send + 'static + Clone, { @@ -359,18 +414,54 @@ impl Discovery { Ok(v) => v, Err(e) => { crit!(self.log, "Local ENR has no fork id"; "error" => e); - return; + return None; } }; // predicate for finding nodes with a matching fork - let eth2_fork_predicate = move |enr: &Enr| { - enr.eth2().map(|enr| enr.fork_digest) == Ok(enr_fork_id.fork_digest.clone()) - }; + let eth2_fork_predicate = move |enr: &Enr| enr.eth2() == Ok(enr_fork_id.clone()); let predicate = move |enr: &Enr| eth2_fork_predicate(enr) && enr_predicate(enr); // general predicate - self.discovery - .find_enr_predicate(random_node, predicate, num_nodes); + Some( + self.discovery + .find_enr_predicate(random_node, predicate, num_nodes), + ) + } + + /// Peers that are found during discovery are optionally dialed. + // TODO: Shift to peer manager. As its own service, discovery should spit out discovered nodes + // and the peer manager should decide about who to connect to. + fn dial_discovered_peers(&mut self, peers: Vec, min_ttl: Option) { + for enr in peers { + // cache known peers + let peer_id = enr.peer_id(); + self.cached_enrs.put(enr.peer_id(), enr); + + // if we need more peers, attempt a connection + if self.network_globals.connected_or_dialing_peers() < self.max_peers + && !self + .network_globals + .peers + .read() + .is_connected_or_dialing(&peer_id) + && !self.banned_peers.contains(&peer_id) + { + debug!(self.log, "Connecting to discovered peer"; "peer_id"=> peer_id.to_string()); + // TODO: Update output + // This should be updated with the peer dialing. In fact created once the peer is + // dialed + if let Some(min_ttl) = min_ttl { + self.network_globals + .peers + .write() + .update_min_ttl(&peer_id, min_ttl); + } + self.events.push_back(NetworkBehaviourAction::DialPeer { + peer_id, + condition: DialPeerCondition::Disconnected, + }); + } + } } } @@ -440,7 +531,8 @@ impl NetworkBehaviour for Discovery { } // Set to maximum, and update to earlier, once we get our results back. self.peer_discovery_delay.reset( - Instant::now() + Duration::from_secs(MAX_TIME_BETWEEN_PEER_SEARCHES), + tokio::time::Instant::now() + + Duration::from_secs(MAX_TIME_BETWEEN_PEER_SEARCHES), ); } Poll::Pending => break, @@ -477,7 +569,11 @@ impl NetworkBehaviour for Discovery { address, }); } - Discv5Event::FindNodeResult { closer_peers, .. } => { + Discv5Event::FindNodeResult { + closer_peers, + query_id, + .. + } => { debug!(self.log, "Discovery query completed"; "peers_found" => closer_peers.len()); // update the time to the next query if self.past_discovery_delay < MAX_TIME_BETWEEN_PEER_SEARCHES @@ -486,40 +582,30 @@ impl NetworkBehaviour for Discovery { { self.past_discovery_delay *= 2; } - let delay = std::cmp::min( + let delay = std::cmp::max( self.past_discovery_delay, MAX_TIME_BETWEEN_PEER_SEARCHES, ); self.peer_discovery_delay - .reset(Instant::now() + Duration::from_secs(delay)); + .reset(tokio::time::Instant::now() + Duration::from_secs(delay)); - for enr in closer_peers { - // cache known peers - let peer_id = enr.peer_id(); - self.cached_enrs.put(enr.peer_id(), enr); - - // if we need more peers, attempt a connection - if self.network_globals.connected_or_dialing_peers() - < self.max_peers - && !self - .network_globals - .peers - .read() - .is_connected_or_dialing(&peer_id) - && !self.banned_peers.contains(&peer_id) - { - // TODO: Debugging only - // NOTE: The peer manager will get updated by the global swarm. - let connection_status = self - .network_globals - .peers - .read() - .connection_status(&peer_id); - debug!(self.log, "Connecting to discovered peer"; "peer_id"=> peer_id.to_string(), "status" => format!("{:?}", connection_status)); - self.events.push_back(NetworkBehaviourAction::DialPeer { - peer_id, - condition: DialPeerCondition::Disconnected, - }); + // if this is a subnet query, run it to completion + if let Some((subnet_id, min_ttl)) = self + .subnet_queries + .iter() + .find(|(_, request)| request.query_id == Some(query_id)) + .map(|(subnet_id, request)| { + (subnet_id.clone(), request.min_ttl.clone()) + }) + { + debug!(self.log, "Peer subnet discovery request completed"; "peers_found" => closer_peers.len(), "subnet_id" => *subnet_id); + self.dial_discovered_peers(closer_peers, min_ttl); + self.run_subnet_query(subnet_id); + } else { + if closer_peers.is_empty() { + debug!(self.log, "Peer Discovery request yielded no results."); + } else { + self.dial_discovered_peers(closer_peers, None); } } } diff --git a/beacon_node/eth2-libp2p/src/discovery/subnet_predicate.rs b/beacon_node/eth2-libp2p/src/discovery/subnet_predicate.rs new file mode 100644 index 000000000..89451d7f6 --- /dev/null +++ b/beacon_node/eth2-libp2p/src/discovery/subnet_predicate.rs @@ -0,0 +1,33 @@ +///! The subnet predicate used for searching for a particular subnet. +use super::*; + +/// Returns the predicate for a given subnet. +pub fn subnet_predicate( + subnet_id: SubnetId, + log: &slog::Logger, +) -> impl Fn(&Enr) -> bool + Send + 'static + Clone +where + TSpec: EthSpec, +{ + let log_clone = log.clone(); + + move |enr: &Enr| { + if let Some(bitfield_bytes) = enr.get(BITFIELD_ENR_KEY) { + let bitfield = match BitVector::::from_ssz_bytes( + bitfield_bytes, + ) { + Ok(v) => v, + Err(e) => { + warn!(log_clone, "Could not decode ENR bitfield for peer"; "peer_id" => format!("{}", enr.peer_id()), "error" => format!("{:?}", e)); + return false; + } + }; + + return bitfield.get(*subnet_id as usize).unwrap_or_else(|_| { + debug!(log_clone, "Peer found but not on desired subnet"; "peer_id" => format!("{}", enr.peer_id())); + false + }); + } + false + } +} diff --git a/beacon_node/eth2-libp2p/src/lib.rs b/beacon_node/eth2-libp2p/src/lib.rs index 2c028eac6..7e1f2cd2d 100644 --- a/beacon_node/eth2-libp2p/src/lib.rs +++ b/beacon_node/eth2-libp2p/src/lib.rs @@ -15,12 +15,11 @@ mod service; pub mod types; pub use crate::types::{error, Enr, GossipTopic, NetworkGlobals, PubsubMessage}; -pub use behaviour::BehaviourEvent; +pub use behaviour::{BehaviourEvent, Request, Response}; pub use config::Config as NetworkConfig; pub use discovery::enr_ext::{CombinedKeyExt, EnrExt}; pub use libp2p::gossipsub::{MessageId, Topic, TopicHash}; pub use libp2p::{core::ConnectedPoint, PeerId, Swarm}; pub use libp2p::{multiaddr, Multiaddr}; pub use peer_manager::{client::Client, PeerDB, PeerInfo, PeerSyncStatus, SyncInfo}; -pub use rpc::RPCEvent; pub use service::{Libp2pEvent, Service, NETWORK_KEY_FILENAME}; diff --git a/beacon_node/eth2-libp2p/src/peer_manager/mod.rs b/beacon_node/eth2-libp2p/src/peer_manager/mod.rs index 2f5d0e9d3..375ab71dc 100644 --- a/beacon_node/eth2-libp2p/src/peer_manager/mod.rs +++ b/beacon_node/eth2-libp2p/src/peer_manager/mod.rs @@ -279,7 +279,7 @@ impl PeerManager { // this could their fault or ours, so we tolerate this PeerAction::HighToleranceError } - RPCError::ErrorResponse(code) => match code { + RPCError::ErrorResponse(code, _) => match code { RPCResponseErrorCode::Unknown => PeerAction::HighToleranceError, RPCResponseErrorCode::ServerError => PeerAction::MidToleranceError, RPCResponseErrorCode::InvalidRequest => PeerAction::LowToleranceError, diff --git a/beacon_node/eth2-libp2p/src/peer_manager/peer_info.rs b/beacon_node/eth2-libp2p/src/peer_manager/peer_info.rs index 4c97b2c08..825162662 100644 --- a/beacon_node/eth2-libp2p/src/peer_manager/peer_info.rs +++ b/beacon_node/eth2-libp2p/src/peer_manager/peer_info.rs @@ -31,6 +31,10 @@ pub struct PeerInfo { /// The ENR subnet bitfield of the peer. This may be determined after it's initial /// connection. pub meta_data: Option>, + /// The time we would like to retain this peer. After this time, the peer is no longer + /// necessary. + #[serde(skip)] + pub min_ttl: Option, } impl Default for PeerInfo { @@ -43,6 +47,7 @@ impl Default for PeerInfo { listening_addresses: vec![], sync_status: PeerSyncStatus::Unknown, meta_data: None, + min_ttl: None, } } } diff --git a/beacon_node/eth2-libp2p/src/peer_manager/peerdb.rs b/beacon_node/eth2-libp2p/src/peer_manager/peerdb.rs index 69b7b1215..b5aa2cb3d 100644 --- a/beacon_node/eth2-libp2p/src/peer_manager/peerdb.rs +++ b/beacon_node/eth2-libp2p/src/peer_manager/peerdb.rs @@ -2,7 +2,7 @@ use super::peer_info::{PeerConnectionStatus, PeerInfo}; use super::peer_sync_status::PeerSyncStatus; use crate::rpc::methods::MetaData; use crate::PeerId; -use slog::{crit, debug, warn}; +use slog::{crit, debug, trace, warn}; use std::collections::{hash_map::Entry, HashMap}; use std::time::Instant; use types::{EthSpec, SubnetId}; @@ -233,7 +233,42 @@ impl PeerDB { info.connection_status = PeerConnectionStatus::Dialing { since: Instant::now(), }; - debug!(self.log, "Peer dialing in db"; "peer_id" => peer_id.to_string(), "n_dc" => self.n_dc); + } + + /// Update min ttl of a peer. + pub fn update_min_ttl(&mut self, peer_id: &PeerId, min_ttl: Instant) { + let info = self.peers.entry(peer_id.clone()).or_default(); + + // only update if the ttl is longer + if info.min_ttl.is_none() || Some(min_ttl) > info.min_ttl { + info.min_ttl = Some(min_ttl); + + let min_ttl_secs = min_ttl + .checked_duration_since(Instant::now()) + .map(|duration| duration.as_secs()) + .unwrap_or_else(|| 0); + debug!(self.log, "Updating the time a peer is required for"; "peer_id" => peer_id.to_string(), "future_min_ttl_secs" => min_ttl_secs); + } + } + + /// Extends the ttl of all peers on the given subnet that have a shorter + /// min_ttl than what's given. + pub fn extend_peers_on_subnet(&mut self, subnet_id: SubnetId, min_ttl: Instant) { + let log = &self.log; + self.peers.iter_mut() + .filter(move |(_, info)| { + info.connection_status.is_connected() && info.on_subnet(subnet_id) + }) + .for_each(|(peer_id,info)| { + if info.min_ttl.is_none() || Some(min_ttl) > info.min_ttl { + info.min_ttl = Some(min_ttl); + } + let min_ttl_secs = min_ttl + .checked_duration_since(Instant::now()) + .map(|duration| duration.as_secs()) + .unwrap_or_else(|| 0); + trace!(log, "Updating minimum duration a peer is required for"; "peer_id" => peer_id.to_string(), "min_ttl" => min_ttl_secs); + }); } /// Sets a peer as connected with an ingoing connection. @@ -244,7 +279,6 @@ impl PeerDB { self.n_dc = self.n_dc.saturating_sub(1); } info.connection_status.connect_ingoing(); - debug!(self.log, "Peer connected to db"; "peer_id" => peer_id.to_string(), "n_dc" => self.n_dc); } /// Sets a peer as connected with an outgoing connection. @@ -255,7 +289,6 @@ impl PeerDB { self.n_dc = self.n_dc.saturating_sub(1); } info.connection_status.connect_outgoing(); - debug!(self.log, "Peer connected to db"; "peer_id" => peer_id.to_string(), "n_dc" => self.n_dc); } /// Sets the peer as disconnected. A banned peer remains banned @@ -270,7 +303,6 @@ impl PeerDB { info.connection_status.disconnect(); self.n_dc += 1; } - debug!(self.log, "Peer disconnected from db"; "peer_id" => peer_id.to_string(), "n_dc" => self.n_dc); self.shrink_to_fit(); } @@ -302,7 +334,6 @@ impl PeerDB { if info.connection_status.is_disconnected() { self.n_dc = self.n_dc.saturating_sub(1); } - debug!(self.log, "Peer banned"; "peer_id" => peer_id.to_string(), "n_dc" => self.n_dc); info.connection_status.ban(); } @@ -375,7 +406,7 @@ mod tests { } fn get_db() -> PeerDB { - let log = build_log(slog::Level::Debug, true); + let log = build_log(slog::Level::Debug, false); PeerDB::new(&log) } diff --git a/beacon_node/eth2-libp2p/src/rpc/handler.rs b/beacon_node/eth2-libp2p/src/rpc/handler.rs index 9bb8500a3..e60ee28c1 100644 --- a/beacon_node/eth2-libp2p/src/rpc/handler.rs +++ b/beacon_node/eth2-libp2p/src/rpc/handler.rs @@ -3,7 +3,7 @@ use super::methods::{RPCCodedResponse, RequestId, ResponseTermination}; use super::protocol::{Protocol, RPCError, RPCProtocol, RPCRequest}; -use super::RPCEvent; +use super::{RPCReceived, RPCSend}; use crate::rpc::protocol::{InboundFramed, OutboundFramed}; use fnv::FnvHashMap; use futures::prelude::*; @@ -33,12 +33,34 @@ pub const RESPONSE_TIMEOUT: u64 = 10; /// The number of times to retry an outbound upgrade in the case of IO errors. const IO_ERROR_RETRIES: u8 = 3; -/// Inbound requests are given a sequential `RequestId` to keep track of. All inbound streams are -/// identified by their substream ID which is identical to the RPC Id. -type InboundRequestId = RequestId; -/// Outbound requests are associated with an id that is given by the application that sent the -/// request. -type OutboundRequestId = RequestId; +/// Identifier of inbound and outbound substreams from the handler's perspective. +#[derive(Debug, Clone, Copy, Hash, Eq, PartialEq)] +pub struct SubstreamId(usize); + +/// An error encoutered by the handler. +pub enum HandlerErr { + /// An error ocurred for this peer's request. This can occurr during protocol negotiation, + /// message passing, or if the handler identifies that we are sending an error reponse to the peer. + Inbound { + /// Id of the peer's request for which an error occurred. + id: SubstreamId, + /// Information of the negotiated protocol. + proto: Protocol, + /// The error that ocurred. + error: RPCError, + }, + /// An error ocurred for this request. Such error can occurr during protocol negotiation, + /// message passing, or if we successfully received a response from the peer, but this response + /// indicates an error. + Outbound { + /// Application-given Id of the request for which an error occurred. + id: RequestId, + /// Information of the protocol. + proto: Protocol, + /// The error that ocurred. + error: RPCError, + }, +} /// Implementation of `ProtocolsHandler` for the RPC protocol. pub struct RPCHandler @@ -48,11 +70,11 @@ where /// The upgrade for inbound substreams. listen_protocol: SubstreamProtocol>, - /// If something bad happened and we should shut down the handler with an error. - pending_error: Vec<(RequestId, Protocol, RPCError)>, + /// Errors ocurring on outbound and inbound connections queued for reporting back. + pending_errors: Vec, /// Queue of events to produce in `poll()`. - events_out: SmallVec<[RPCEvent; 4]>, + events_out: SmallVec<[RPCReceived; 4]>, /// Queue of outbound substreams to open. dial_queue: SmallVec<[(RequestId, RPCRequest); 4]>, @@ -62,7 +84,7 @@ where /// Current inbound substreams awaiting processing. inbound_substreams: FnvHashMap< - InboundRequestId, + SubstreamId, ( InboundSubstreamState, Option, @@ -71,29 +93,22 @@ where >, /// Inbound substream `DelayQueue` which keeps track of when an inbound substream will timeout. - inbound_substreams_delay: DelayQueue, + inbound_substreams_delay: DelayQueue, - /// Map of outbound substreams that need to be driven to completion. The `RequestId` is - /// maintained by the application sending the request. - /// For Responses with multiple expected response chunks a counter is added to be able to terminate the stream when the expected number has been received - outbound_substreams: FnvHashMap< - OutboundRequestId, - ( - OutboundSubstreamState, - delay_queue::Key, - Protocol, - Option, - ), - >, + /// Map of outbound substreams that need to be driven to completion. + outbound_substreams: FnvHashMap>, /// Inbound substream `DelayQueue` which keeps track of when an inbound substream will timeout. - outbound_substreams_delay: DelayQueue, + outbound_substreams_delay: DelayQueue, /// Map of outbound items that are queued as the stream processes them. - queued_outbound_items: FnvHashMap>>, + queued_outbound_items: FnvHashMap>>, /// Sequential ID for waiting substreams. For inbound substreams, this is also the inbound request ID. - current_inbound_substream_id: RequestId, + current_inbound_substream_id: SubstreamId, + + /// Sequential ID for outbound substreams. + current_outbound_substream_id: SubstreamId, /// Maximum number of concurrent outbound substreams being opened. Value is never modified. max_dial_negotiated: u32, @@ -112,6 +127,23 @@ where log: slog::Logger, } +/// Contains the information the handler keeps on established outbound substreams. +struct OutboundInfo { + /// State of the substream. + state: OutboundSubstreamState, + /// Key to keep track of the substream's timeout via `self.outbound_substreams_delay`. + delay_key: delay_queue::Key, + /// Info over the protocol this substream is handling. + proto: Protocol, + /// Number of chunks to be seen from the peer's response. + // TODO: removing the option could allow clossing the streams after the number of + // expected responses is met for all protocols. + // TODO: the type of this is wrong + remaining_chunks: Option, + /// RequestId as given by the application that sent the request. + req_id: RequestId, +} + pub enum InboundSubstreamState where TSpec: EthSpec, @@ -208,7 +240,7 @@ where } InboundSubstreamState::ResponseIdle(substream) => { *self = InboundSubstreamState::ResponsePendingSend { - substream: substream, + substream, message: error, closing: true, }; @@ -235,7 +267,7 @@ where ) -> Self { RPCHandler { listen_protocol, - pending_error: Vec::new(), + pending_errors: Vec::new(), events_out: SmallVec::new(), dial_queue: SmallVec::new(), dial_negotiated: 0, @@ -244,7 +276,8 @@ where outbound_substreams: FnvHashMap::default(), inbound_substreams_delay: DelayQueue::new(), outbound_substreams_delay: DelayQueue::new(), - current_inbound_substream_id: 1, + current_inbound_substream_id: SubstreamId(0), + current_outbound_substream_id: SubstreamId(0), max_dial_negotiated: 8, keep_alive: KeepAlive::Yes, inactive_timeout, @@ -300,8 +333,8 @@ impl ProtocolsHandler for RPCHandler where TSpec: EthSpec, { - type InEvent = RPCEvent; - type OutEvent = RPCEvent; + type InEvent = RPCSend; + type OutEvent = Result, HandlerErr>; type Error = RPCError; type InboundProtocol = RPCProtocol; type OutboundProtocol = RPCRequest; @@ -316,9 +349,11 @@ where substream: >::Output, ) { let (req, substream) = substream; - // drop the stream and return a 0 id for goodbye "requests" - if let r @ RPCRequest::Goodbye(_) = req { - self.events_out.push(RPCEvent::Request(0, r)); + // drop the stream + if let RPCRequest::Goodbye(_) = req { + self.events_out + .push(RPCReceived::Request(self.current_inbound_substream_id, req)); + self.current_inbound_substream_id.0 += 1; return; } @@ -334,8 +369,8 @@ where ); self.events_out - .push(RPCEvent::Request(self.current_inbound_substream_id, req)); - self.current_inbound_substream_id += 1; + .push(RPCReceived::Request(self.current_inbound_substream_id, req)); + self.current_inbound_substream_id.0 += 1; } fn inject_fully_negotiated_outbound( @@ -346,43 +381,42 @@ where self.dial_negotiated -= 1; // add the stream to substreams if we expect a response, otherwise drop the stream. - let (mut id, request) = request_info; - if request.expect_response() { - // outbound requests can be sent from various aspects of lighthouse which don't - // track request ids. In the future these will be flagged as None, currently they - // are flagged as 0. These can overlap. In this case, we pick the highest request - // Id available - if id == 0 && self.outbound_substreams.get(&id).is_some() { - // have duplicate outbound request with no id. Pick one that will not collide - let mut new_id = std::usize::MAX; - while self.outbound_substreams.get(&new_id).is_some() { - // panic all outbound substreams are full - new_id -= 1; - } - trace!(self.log, "New outbound stream id created"; "id" => new_id); - id = RequestId::from(new_id); - } - + let (id, request) = request_info; + let expected_responses = request.expected_responses(); + if expected_responses > 0 { // new outbound request. Store the stream and tag the output. - let delay_key = self - .outbound_substreams_delay - .insert(id, Duration::from_secs(RESPONSE_TIMEOUT)); - let protocol = request.protocol(); - let response_chunk_count = match request { - RPCRequest::BlocksByRange(ref req) => Some(req.count), - RPCRequest::BlocksByRoot(ref req) => Some(req.block_roots.len() as u64), - _ => None, // Other requests do not have a known response chunk length, - }; + let delay_key = self.outbound_substreams_delay.insert( + self.current_outbound_substream_id, + Duration::from_secs(RESPONSE_TIMEOUT), + ); + let proto = request.protocol(); let awaiting_stream = OutboundSubstreamState::RequestPendingResponse { substream: out, - request: request, + request, }; - if let Some(_) = self.outbound_substreams.insert( - id, - (awaiting_stream, delay_key, protocol, response_chunk_count), - ) { - crit!(self.log, "Duplicate outbound substream id"; "id" => format!("{:?}", id)); + let expected_responses = if expected_responses > 1 { + // Currently enforced only for multiple responses + Some(expected_responses) + } else { + None + }; + if self + .outbound_substreams + .insert( + self.current_outbound_substream_id, + OutboundInfo { + state: awaiting_stream, + delay_key, + proto, + remaining_chunks: expected_responses, + req_id: id, + }, + ) + .is_some() + { + crit!(self.log, "Duplicate outbound substream id"; "id" => format!("{:?}", self.current_outbound_substream_id)); } + self.current_outbound_substream_id.0 += 1; } self.update_keep_alive(); @@ -392,113 +426,124 @@ where // wrong state a response will fail silently. fn inject_event(&mut self, rpc_event: Self::InEvent) { match rpc_event { - RPCEvent::Request(id, req) => self.send_request(id, req), - RPCEvent::Response(rpc_id, response) => { + RPCSend::Request(id, req) => self.send_request(id, req), + RPCSend::Response(inbound_id, response) => { // Variables indicating if the response is an error response or a multi-part // response let res_is_error = response.is_error(); let res_is_multiple = response.multiple_responses(); // check if the stream matching the response still exists - match self.inbound_substreams.get_mut(&rpc_id) { - Some((substream_state, _, protocol)) => { - match std::mem::replace(substream_state, InboundSubstreamState::Poisoned) { - InboundSubstreamState::ResponseIdle(substream) => { - // close the stream if there is no response - match response { - RPCCodedResponse::StreamTermination(_) => { - //trace!(self.log, "Stream termination sent. Ending the stream"); - *substream_state = - InboundSubstreamState::Closing(substream); - } - _ => { - if let Some(error_code) = response.error_code() { - self.pending_error.push(( - rpc_id, - *protocol, - RPCError::ErrorResponse(error_code), - )); - } - // send the response - // if it's a single rpc request or an error, close the stream after - *substream_state = - InboundSubstreamState::ResponsePendingSend { - substream: substream, - message: response, - closing: !res_is_multiple | res_is_error, // close if an error or we are not expecting more responses - }; - } - } - } - InboundSubstreamState::ResponsePendingSend { - substream, - message, - closing, - } if res_is_multiple => { - // the stream is in use, add the request to a pending queue - self.queued_outbound_items - .entry(rpc_id) - .or_insert_with(Vec::new) - .push(response); + let (substream_state, protocol) = match self.inbound_substreams.get_mut(&inbound_id) + { + Some((substream_state, _, protocol)) => (substream_state, protocol), + None => { + warn!(self.log, "Stream has expired. Response not sent"; + "response" => response.to_string(), "id" => inbound_id); + return; + } + }; - // return the state - *substream_state = InboundSubstreamState::ResponsePendingSend { - substream, - message, - closing, - }; - } - InboundSubstreamState::ResponsePendingFlush { substream, closing } - if res_is_multiple => - { - // the stream is in use, add the request to a pending queue - self.queued_outbound_items - .entry(rpc_id) - .or_insert_with(Vec::new) - .push(response); + // If the response we are sending is an error, report back for handling + match response { + RPCCodedResponse::InvalidRequest(ref reason) + | RPCCodedResponse::ServerError(ref reason) + | RPCCodedResponse::Unknown(ref reason) => { + let code = &response + .error_code() + .expect("Error response should map to an error code"); + let err = HandlerErr::Inbound { + id: inbound_id, + proto: *protocol, + error: RPCError::ErrorResponse(*code, reason.clone()), + }; + self.pending_errors.push(err); + } + _ => {} // not an error, continue. + } - // return the state - *substream_state = InboundSubstreamState::ResponsePendingFlush { - substream, - closing, - }; - } - InboundSubstreamState::Closing(substream) => { + match std::mem::replace(substream_state, InboundSubstreamState::Poisoned) { + InboundSubstreamState::ResponseIdle(substream) => { + // close the stream if there is no response + match response { + RPCCodedResponse::StreamTermination(_) => { *substream_state = InboundSubstreamState::Closing(substream); - debug!(self.log, "Response not sent. Stream is closing"; "response" => format!("{}",response)); } - InboundSubstreamState::ResponsePendingSend { - substream, - message, - .. - } => { + _ => { + // send the response + // if it's a single rpc request or an error, close the stream after *substream_state = InboundSubstreamState::ResponsePendingSend { substream, - message, - closing: true, + message: response, + closing: !res_is_multiple | res_is_error, // close if an error or we are not expecting more responses }; - error!(self.log, "Attempted sending multiple responses to a single response request"); - } - InboundSubstreamState::ResponsePendingFlush { substream, .. } => { - *substream_state = InboundSubstreamState::ResponsePendingFlush { - substream, - closing: true, - }; - error!(self.log, "Attempted sending multiple responses to a single response request"); - } - InboundSubstreamState::Poisoned => { - crit!(self.log, "Poisoned inbound substream"); - unreachable!("Coding error: Poisoned substream"); } } } - None => { - warn!(self.log, "Stream has expired. Response not sent"; "response" => response.to_string(), "id" => rpc_id); + InboundSubstreamState::ResponsePendingSend { + substream, + message, + closing, + } if res_is_multiple => { + // the stream is in use, add the request to a pending queue + self.queued_outbound_items + .entry(inbound_id) + .or_insert_with(Vec::new) + .push(response); + + // return the state + *substream_state = InboundSubstreamState::ResponsePendingSend { + substream, + message, + closing, + }; } - }; + InboundSubstreamState::ResponsePendingFlush { substream, closing } + if res_is_multiple => + { + // the stream is in use, add the request to a pending queue + self.queued_outbound_items + .entry(inbound_id) + .or_insert_with(Vec::new) + .push(response); + + // return the state + *substream_state = + InboundSubstreamState::ResponsePendingFlush { substream, closing }; + } + InboundSubstreamState::Closing(substream) => { + *substream_state = InboundSubstreamState::Closing(substream); + debug!(self.log, "Response not sent. Stream is closing"; "response" => format!("{}",response)); + } + InboundSubstreamState::ResponsePendingSend { + substream, message, .. + } => { + *substream_state = InboundSubstreamState::ResponsePendingSend { + substream, + message, + closing: true, + }; + error!( + self.log, + "Attempted sending multiple responses to a single response request" + ); + } + InboundSubstreamState::ResponsePendingFlush { substream, .. } => { + *substream_state = InboundSubstreamState::ResponsePendingFlush { + substream, + closing: true, + }; + error!( + self.log, + "Attempted sending multiple responses to a single response request" + ); + } + InboundSubstreamState::Poisoned => { + crit!(self.log, "Poisoned inbound substream"); + unreachable!("Coding error: Poisoned substream"); + } + } } - // We do not send errors as responses - RPCEvent::Error(..) => {} } } @@ -520,7 +565,7 @@ where self.outbound_io_error_retries = 0; // map the error - let rpc_error = match error { + let error = match error { ProtocolsHandlerUpgrErr::Timer => RPCError::InternalError("Timer failed"), ProtocolsHandlerUpgrErr::Timeout => RPCError::NegotiationTimeout, ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Apply(e)) => e, @@ -541,7 +586,11 @@ where } }, }; - self.pending_error.push((id, req.protocol(), rpc_error)); + self.pending_errors.push(HandlerErr::Outbound { + id, + proto: req.protocol(), + error, + }); } fn connection_keep_alive(&self) -> KeepAlive { @@ -559,16 +608,15 @@ where Self::Error, >, > { - if !self.pending_error.is_empty() { - let (id, protocol, err) = self.pending_error.remove(0); - return Poll::Ready(ProtocolsHandlerEvent::Custom(RPCEvent::Error( - id, protocol, err, - ))); + // report failures + if !self.pending_errors.is_empty() { + let err_info = self.pending_errors.remove(0); + return Poll::Ready(ProtocolsHandlerEvent::Custom(Err(err_info))); } // return any events that need to be reported if !self.events_out.is_empty() { - return Poll::Ready(ProtocolsHandlerEvent::Custom(self.events_out.remove(0))); + return Poll::Ready(ProtocolsHandlerEvent::Custom(Ok(self.events_out.remove(0)))); } else { self.events_out.shrink_to_fit(); } @@ -576,17 +624,23 @@ where // purge expired inbound substreams and send an error loop { match self.inbound_substreams_delay.poll_next_unpin(cx) { - Poll::Ready(Some(Ok(stream_id))) => { + Poll::Ready(Some(Ok(inbound_id))) => { // handle a stream timeout for various states - if let Some((substream_state, delay_key, _)) = - self.inbound_substreams.get_mut(stream_id.get_ref()) + if let Some((substream_state, delay_key, protocol)) = + self.inbound_substreams.get_mut(inbound_id.get_ref()) { // the delay has been removed *delay_key = None; + self.pending_errors.push(HandlerErr::Inbound { + id: *inbound_id.get_ref(), + proto: *protocol, + error: RPCError::StreamTimeout, + }); + let outbound_queue = self .queued_outbound_items - .entry(stream_id.into_inner()) + .entry(inbound_id.into_inner()) .or_insert_with(Vec::new); substream_state.close(outbound_queue); } @@ -605,20 +659,21 @@ where // purge expired outbound substreams loop { match self.outbound_substreams_delay.poll_next_unpin(cx) { - Poll::Ready(Some(Ok(stream_id))) => { - if let Some((_id, _stream, protocol, _)) = - self.outbound_substreams.remove(stream_id.get_ref()) + Poll::Ready(Some(Ok(outbound_id))) => { + if let Some(OutboundInfo { proto, req_id, .. }) = + self.outbound_substreams.remove(outbound_id.get_ref()) { self.update_keep_alive(); + let outbound_err = HandlerErr::Outbound { + id: req_id, + proto, + error: RPCError::StreamTimeout, + }; // notify the user - return Poll::Ready(ProtocolsHandlerEvent::Custom(RPCEvent::Error( - *stream_id.get_ref(), - protocol, - RPCError::StreamTimeout, - ))); + return Poll::Ready(ProtocolsHandlerEvent::Custom(Err(outbound_err))); } else { - crit!(self.log, "timed out substream not in the books"; "stream_id" => stream_id.get_ref()); + crit!(self.log, "timed out substream not in the books"; "stream_id" => outbound_id.get_ref()); } } Poll::Ready(Some(Err(e))) => { @@ -797,155 +852,161 @@ where } // drive outbound streams that need to be processed - for request_id in self.outbound_substreams.keys().copied().collect::>() { - match self.outbound_substreams.entry(request_id) { + for outbound_id in self.outbound_substreams.keys().copied().collect::>() { + // get the state and mark it as poisoned + let (mut entry, state) = match self.outbound_substreams.entry(outbound_id) { Entry::Occupied(mut entry) => { - match std::mem::replace( - &mut entry.get_mut().0, + let state = std::mem::replace( + &mut entry.get_mut().state, OutboundSubstreamState::Poisoned, - ) { - OutboundSubstreamState::RequestPendingResponse { - mut substream, - request, - } => match substream.poll_next_unpin(cx) { - Poll::Ready(Some(Ok(response))) => { - if request.multiple_responses() && !response.is_error() { - let substream_entry = entry.get_mut(); - let delay_key = &substream_entry.1; - // chunks left after this one - let remaining_chunks = substream_entry - .3 - .map(|count| count.saturating_sub(1)) - .unwrap_or_else(|| 0); - if remaining_chunks == 0 { - // this is the last expected message, close the stream as all expected chunks have been received - substream_entry.0 = - OutboundSubstreamState::Closing(substream); - } else { - // If the response chunk was expected update the remaining number of chunks expected and reset the Timeout - substream_entry.0 = - OutboundSubstreamState::RequestPendingResponse { - substream, - request, - }; - substream_entry.3 = Some(remaining_chunks); - self.outbound_substreams_delay.reset( - delay_key, - Duration::from_secs(RESPONSE_TIMEOUT), - ); - } - } else { - // either this is a single response request or we received an - // error - // only expect a single response, close the stream - entry.get_mut().0 = OutboundSubstreamState::Closing(substream); - } + ); + (entry, state) + } + Entry::Vacant(_) => unreachable!(), + }; - return Poll::Ready(ProtocolsHandlerEvent::Custom( - RPCEvent::Response(request_id, response), - )); + match state { + OutboundSubstreamState::RequestPendingResponse { + mut substream, + request, + } => match substream.poll_next_unpin(cx) { + Poll::Ready(Some(Ok(response))) => { + if request.expected_responses() > 1 && !response.is_error() { + let substream_entry = entry.get_mut(); + let delay_key = &substream_entry.delay_key; + // chunks left after this one + let remaining_chunks = substream_entry + .remaining_chunks + .map(|count| count.saturating_sub(1)) + .unwrap_or_else(|| 0); + if remaining_chunks == 0 { + // this is the last expected message, close the stream as all expected chunks have been received + substream_entry.state = OutboundSubstreamState::Closing(substream); + } else { + // If the response chunk was expected update the remaining number of chunks expected and reset the Timeout + substream_entry.state = + OutboundSubstreamState::RequestPendingResponse { + substream, + request, + }; + substream_entry.remaining_chunks = Some(remaining_chunks); + self.outbound_substreams_delay + .reset(delay_key, Duration::from_secs(RESPONSE_TIMEOUT)); } - Poll::Ready(None) => { - // stream closed - // if we expected multiple streams send a stream termination, - // else report the stream terminating only. - //trace!(self.log, "RPC Response - stream closed by remote"); - // drop the stream - let delay_key = &entry.get().1; - self.outbound_substreams_delay.remove(delay_key); - entry.remove_entry(); + } else { + // either this is a single response request or we received an + // error only expect a single response, close the stream + entry.get_mut().state = OutboundSubstreamState::Closing(substream); + } - self.update_keep_alive(); - // notify the application error - if request.multiple_responses() { - // return an end of stream result - return Poll::Ready(ProtocolsHandlerEvent::Custom( - RPCEvent::Response( - request_id, - RPCCodedResponse::StreamTermination( - request.stream_termination(), - ), - ), - )); - } // else we return an error, stream should not have closed early. - return Poll::Ready(ProtocolsHandlerEvent::Custom( - RPCEvent::Error( - request_id, - request.protocol(), - RPCError::IncompleteStream, - ), - )); - } - Poll::Pending => { - entry.get_mut().0 = OutboundSubstreamState::RequestPendingResponse { - substream, - request, - } - } - Poll::Ready(Some(Err(e))) => { - // drop the stream - let delay_key = &entry.get().1; - self.outbound_substreams_delay.remove(delay_key); - let protocol = entry.get().2; - entry.remove_entry(); - self.update_keep_alive(); - return Poll::Ready(ProtocolsHandlerEvent::Custom( - RPCEvent::Error(request_id, protocol, e), - )); - } - }, - OutboundSubstreamState::Closing(mut substream) => { - match Sink::poll_close(Pin::new(&mut substream), cx) { - Poll::Ready(_) => { - // drop the stream and its corresponding timeout - let delay_key = &entry.get().1; - let protocol = entry.get().2; - self.outbound_substreams_delay.remove(delay_key); - entry.remove_entry(); - self.update_keep_alive(); + // Check what type of response we got and report it accordingly + let id = entry.get().req_id; + let proto = entry.get().proto; - // report the stream termination to the user - // - // Streams can be terminated here if a responder tries to - // continue sending responses beyond what we would expect. Here - // we simply terminate the stream and report a stream - // termination to the application - match protocol { - Protocol::BlocksByRange => { - return Poll::Ready(ProtocolsHandlerEvent::Custom( - RPCEvent::Response( - request_id, - RPCCodedResponse::StreamTermination( - ResponseTermination::BlocksByRange, - ), - ), - )); - } - Protocol::BlocksByRoot => { - return Poll::Ready(ProtocolsHandlerEvent::Custom( - RPCEvent::Response( - request_id, - RPCCodedResponse::StreamTermination( - ResponseTermination::BlocksByRoot, - ), - ), - )); - } - _ => {} // all other protocols are do not have multiple responses and we do not inform the user, we simply drop the stream. - } - } - Poll::Pending => { - entry.get_mut().0 = OutboundSubstreamState::Closing(substream); - } + let received = match response { + RPCCodedResponse::StreamTermination(t) => { + Ok(RPCReceived::EndOfStream(id, t)) + } + RPCCodedResponse::Success(resp) => Ok(RPCReceived::Response(id, resp)), + RPCCodedResponse::InvalidRequest(ref r) + | RPCCodedResponse::ServerError(ref r) + | RPCCodedResponse::Unknown(ref r) => { + let code = response.error_code().expect( + "Response indicating and error should map to an error code", + ); + Err(HandlerErr::Outbound { + id, + proto, + error: RPCError::ErrorResponse(code, r.clone()), + }) + } + }; + + return Poll::Ready(ProtocolsHandlerEvent::Custom(received)); + } + Poll::Ready(None) => { + // stream closed + // if we expected multiple streams send a stream termination, + // else report the stream terminating only. + //trace!(self.log, "RPC Response - stream closed by remote"); + // drop the stream + let delay_key = &entry.get().delay_key; + let request_id = *&entry.get().req_id; + self.outbound_substreams_delay.remove(delay_key); + entry.remove_entry(); + self.update_keep_alive(); + // notify the application error + if request.expected_responses() > 1 { + // return an end of stream result + return Poll::Ready(ProtocolsHandlerEvent::Custom(Ok( + RPCReceived::EndOfStream(request_id, request.stream_termination()), + ))); + } + + // else we return an error, stream should not have closed early. + let outbound_err = HandlerErr::Outbound { + id: request_id, + proto: request.protocol(), + error: RPCError::IncompleteStream, + }; + return Poll::Ready(ProtocolsHandlerEvent::Custom(Err(outbound_err))); + } + Poll::Pending => { + entry.get_mut().state = + OutboundSubstreamState::RequestPendingResponse { substream, request } + } + Poll::Ready(Some(Err(e))) => { + // drop the stream + let delay_key = &entry.get().delay_key; + self.outbound_substreams_delay.remove(delay_key); + let outbound_err = HandlerErr::Outbound { + id: entry.get().req_id, + proto: entry.get().proto, + error: e, + }; + entry.remove_entry(); + self.update_keep_alive(); + return Poll::Ready(ProtocolsHandlerEvent::Custom(Err(outbound_err))); + } + }, + OutboundSubstreamState::Closing(mut substream) => { + match Sink::poll_close(Pin::new(&mut substream), cx) { + Poll::Ready(_) => { + // drop the stream and its corresponding timeout + let delay_key = &entry.get().delay_key; + let protocol = entry.get().proto; + let request_id = entry.get().req_id; + self.outbound_substreams_delay.remove(delay_key); + entry.remove_entry(); + self.update_keep_alive(); + + // report the stream termination to the user + // + // Streams can be terminated here if a responder tries to + // continue sending responses beyond what we would expect. Here + // we simply terminate the stream and report a stream + // termination to the application + let termination = match protocol { + Protocol::BlocksByRange => Some(ResponseTermination::BlocksByRange), + Protocol::BlocksByRoot => Some(ResponseTermination::BlocksByRoot), + _ => None, // all other protocols are do not have multiple responses and we do not inform the user, we simply drop the stream. + }; + + if let Some(termination) = termination { + return Poll::Ready(ProtocolsHandlerEvent::Custom(Ok( + RPCReceived::EndOfStream(request_id, termination), + ))); } } - OutboundSubstreamState::Poisoned => { - crit!(self.log, "Poisoned outbound substream"); - unreachable!("Coding Error: Outbound substream is poisoned") + Poll::Pending => { + entry.get_mut().state = OutboundSubstreamState::Closing(substream); } } } - Entry::Vacant(_) => unreachable!(), + OutboundSubstreamState::Poisoned => { + crit!(self.log, "Poisoned outbound substream"); + unreachable!("Coding Error: Outbound substream is poisoned") + } } } @@ -980,7 +1041,7 @@ fn apply_queued_responses( InboundSubstreamState::Closing(substream) } chunk => InboundSubstreamState::ResponsePendingSend { - substream: substream, + substream, message: chunk, closing: false, }, @@ -992,3 +1053,14 @@ fn apply_queued_responses( } } } + +impl slog::Value for SubstreamId { + fn serialize( + &self, + record: &slog::Record, + key: slog::Key, + serializer: &mut dyn slog::Serializer, + ) -> slog::Result { + slog::Value::serialize(&self.0, record, key, serializer) + } +} diff --git a/beacon_node/eth2-libp2p/src/rpc/methods.rs b/beacon_node/eth2-libp2p/src/rpc/methods.rs index 8870ffc03..167198fa8 100644 --- a/beacon_node/eth2-libp2p/src/rpc/methods.rs +++ b/beacon_node/eth2-libp2p/src/rpc/methods.rs @@ -9,7 +9,16 @@ use types::{Epoch, EthSpec, Hash256, SignedBeaconBlock, Slot}; /* Requests */ -pub type RequestId = usize; +/// Identifier of a request. +/// +// NOTE: The handler stores the `RequestId` to inform back of responses and errors, but it's execution +// is independent of the contents on this type. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RequestId { + Router, + Sync(usize), + Behaviour, +} /// The STATUS request/response handshake message. #[derive(Encode, Decode, Clone, Debug, PartialEq)] @@ -194,7 +203,7 @@ pub enum RPCCodedResponse { } /// The code assigned to an erroneous `RPCResponse`. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Copy)] pub enum RPCResponseErrorCode { InvalidRequest, ServerError, @@ -230,6 +239,15 @@ impl RPCCodedResponse { } } + /// Builds an RPCCodedResponse from a response code and an ErrorMessage + pub fn from_error_code(response_code: RPCResponseErrorCode, err: String) -> Self { + match response_code { + RPCResponseErrorCode::InvalidRequest => RPCCodedResponse::InvalidRequest(err), + RPCResponseErrorCode::ServerError => RPCCodedResponse::ServerError(err), + RPCResponseErrorCode::Unknown => RPCCodedResponse::Unknown(err), + } + } + /// Specifies which response allows for multiple chunks for the stream handler. pub fn multiple_responses(&self) -> bool { match self { @@ -333,3 +351,18 @@ impl std::fmt::Display for BlocksByRangeRequest { ) } } + +impl slog::Value for RequestId { + fn serialize( + &self, + record: &slog::Record, + key: slog::Key, + serializer: &mut dyn slog::Serializer, + ) -> slog::Result { + match self { + RequestId::Behaviour => slog::Value::serialize("Behaviour", record, key, serializer), + RequestId::Router => slog::Value::serialize("Router", record, key, serializer), + RequestId::Sync(ref id) => slog::Value::serialize(id, record, key, serializer), + } + } +} diff --git a/beacon_node/eth2-libp2p/src/rpc/mod.rs b/beacon_node/eth2-libp2p/src/rpc/mod.rs index e276bf6b3..1af2389d7 100644 --- a/beacon_node/eth2-libp2p/src/rpc/mod.rs +++ b/beacon_node/eth2-libp2p/src/rpc/mod.rs @@ -11,34 +11,69 @@ use libp2p::swarm::{ PollParameters, SubstreamProtocol, }; use libp2p::{Multiaddr, PeerId}; -pub use methods::{ - MetaData, RPCCodedResponse, RPCResponse, RPCResponseErrorCode, RequestId, ResponseTermination, - StatusMessage, -}; -pub use protocol::{Protocol, RPCError, RPCProtocol, RPCRequest}; use slog::{debug, o}; use std::marker::PhantomData; use std::task::{Context, Poll}; use std::time::Duration; use types::EthSpec; +pub(crate) use handler::HandlerErr; +pub(crate) use methods::{MetaData, Ping, RPCCodedResponse, RPCResponse}; +pub(crate) use protocol::{RPCProtocol, RPCRequest}; + +pub use handler::SubstreamId; +pub use methods::{ + BlocksByRangeRequest, BlocksByRootRequest, GoodbyeReason, RPCResponseErrorCode, RequestId, + ResponseTermination, StatusMessage, +}; +pub use protocol::{Protocol, RPCError}; + pub(crate) mod codec; mod handler; pub mod methods; mod protocol; -/// The return type used in the behaviour and the resultant event from the protocols handler. +/// RPC events sent from Lighthouse. #[derive(Debug, Clone)] -pub enum RPCEvent { - /// An inbound/outbound request for RPC protocol. The first parameter is a sequential - /// id which tracks an awaiting substream for the response. +pub enum RPCSend { + /// A request sent from Lighthouse. + /// + /// The `RequestId` is given by the application making the request. These + /// go over *outbound* connections. Request(RequestId, RPCRequest), - /// A response that is being sent or has been received from the RPC protocol. The first parameter returns - /// that which was sent with the corresponding request, the second is a single chunk of a - /// response. - Response(RequestId, RPCCodedResponse), - /// An Error occurred. - Error(RequestId, Protocol, RPCError), + /// A response sent from Lighthouse. + /// + /// The `SubstreamId` must correspond to the RPC-given ID of the original request received from the + /// peer. The second parameter is a single chunk of a response. These go over *inbound* + /// connections. + Response(SubstreamId, RPCCodedResponse), +} + +/// RPC events received from outside Lighthouse. +#[derive(Debug, Clone)] +pub enum RPCReceived { + /// A request received from the outside. + /// + /// The `SubstreamId` is given by the `RPCHandler` as it identifies this request with the + /// *inbound* substream over which it is managed. + Request(SubstreamId, RPCRequest), + /// A response received from the outside. + /// + /// The `RequestId` corresponds to the application given ID of the original request sent to the + /// peer. The second parameter is a single chunk of a response. These go over *outbound* + /// connections. + Response(RequestId, RPCResponse), + /// Marks a request as completed + EndOfStream(RequestId, ResponseTermination), +} + +impl std::fmt::Display for RPCSend { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RPCSend::Request(id, req) => write!(f, "RPC Request(id: {:?}, {})", id, req), + RPCSend::Response(id, res) => write!(f, "RPC Response(id: {:?}, {})", id, res), + } + } } /// Messages sent to the user from the RPC protocol. @@ -46,38 +81,14 @@ pub struct RPCMessage { /// The peer that sent the message. pub peer_id: PeerId, /// The message that was sent. - pub event: RPCEvent, -} - -impl RPCEvent { - pub fn id(&self) -> usize { - match *self { - RPCEvent::Request(id, _) => id, - RPCEvent::Response(id, _) => id, - RPCEvent::Error(id, _, _) => id, - } - } -} - -impl std::fmt::Display for RPCEvent { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - RPCEvent::Request(id, req) => write!(f, "RPC Request(id: {}, {})", id, req), - RPCEvent::Response(id, res) => write!(f, "RPC Response(id: {}, {})", id, res), - RPCEvent::Error(id, prot, err) => write!( - f, - "RPC Error(id: {}, protocol: {:?} error: {:?})", - id, prot, err - ), - } - } + pub event: as ProtocolsHandler>::OutEvent, } /// Implements the libp2p `NetworkBehaviour` trait and therefore manages network-level /// logic. pub struct RPC { - /// Queue of events to processed. - events: Vec, RPCMessage>>, + /// Queue of events to be processed. + events: Vec, RPCMessage>>, /// Slog logger for RPC behaviour. log: slog::Logger, } @@ -94,11 +105,11 @@ impl RPC { /// Submits an RPC request. /// /// The peer must be connected for this to succeed. - pub fn send_rpc(&mut self, peer_id: PeerId, rpc_event: RPCEvent) { + pub fn send_rpc(&mut self, peer_id: PeerId, event: RPCSend) { self.events.push(NetworkBehaviourAction::NotifyHandler { peer_id, handler: NotifyHandler::Any, - event: rpc_event, + event, }); } } @@ -129,8 +140,7 @@ where fn inject_connected(&mut self, peer_id: &PeerId) { // find the peer's meta-data debug!(self.log, "Requesting new peer's metadata"; "peer_id" => format!("{}",peer_id)); - let rpc_event = - RPCEvent::Request(RequestId::from(0usize), RPCRequest::MetaData(PhantomData)); + let rpc_event = RPCSend::Request(RequestId::Behaviour, RPCRequest::MetaData(PhantomData)); self.events.push(NetworkBehaviourAction::NotifyHandler { peer_id: peer_id.clone(), handler: NotifyHandler::Any, @@ -158,14 +168,14 @@ where fn inject_event( &mut self, - source: PeerId, + peer_id: PeerId, _: ConnectionId, event: ::OutEvent, ) { // send the event to the user self.events .push(NetworkBehaviourAction::GenerateEvent(RPCMessage { - peer_id: source, + peer_id, event, })); } diff --git a/beacon_node/eth2-libp2p/src/rpc/protocol.rs b/beacon_node/eth2-libp2p/src/rpc/protocol.rs index 808f695fa..6f5beb749 100644 --- a/beacon_node/eth2-libp2p/src/rpc/protocol.rs +++ b/beacon_node/eth2-libp2p/src/rpc/protocol.rs @@ -290,32 +290,19 @@ impl RPCRequest { /* These functions are used in the handler for stream management */ - /// This specifies whether a stream should remain open and await a response, given a request. - /// A GOODBYE request has no response. - pub fn expect_response(&self) -> bool { + /// Number of responses expected for this request. + pub fn expected_responses(&self) -> usize { match self { - RPCRequest::Status(_) => true, - RPCRequest::Goodbye(_) => false, - RPCRequest::BlocksByRange(_) => true, - RPCRequest::BlocksByRoot(_) => true, - RPCRequest::Ping(_) => true, - RPCRequest::MetaData(_) => true, - } - } - - /// Returns which methods expect multiple responses from the stream. If this is false and - /// the stream terminates, an error is given. - pub fn multiple_responses(&self) -> bool { - match self { - RPCRequest::Status(_) => false, - RPCRequest::Goodbye(_) => false, - RPCRequest::BlocksByRange(_) => true, - RPCRequest::BlocksByRoot(_) => true, - RPCRequest::Ping(_) => false, - RPCRequest::MetaData(_) => false, + RPCRequest::Status(_) => 1, + RPCRequest::Goodbye(_) => 0, + RPCRequest::BlocksByRange(req) => req.count as usize, + RPCRequest::BlocksByRoot(req) => req.block_roots.len(), + RPCRequest::Ping(_) => 1, + RPCRequest::MetaData(_) => 1, } } + /// Gives the corresponding `Protocol` to this request. pub fn protocol(&self) -> Protocol { match self { RPCRequest::Status(_) => Protocol::Status, @@ -390,7 +377,7 @@ pub enum RPCError { /// IO Error. IoError(String), /// The peer returned a valid response but the response indicated an error. - ErrorResponse(RPCResponseErrorCode), + ErrorResponse(RPCResponseErrorCode, String), /// Timed out waiting for a response. StreamTimeout, /// Peer does not support the protocol. @@ -430,7 +417,11 @@ impl std::fmt::Display for RPCError { RPCError::SSZDecodeError(ref err) => write!(f, "Error while decoding ssz: {:?}", err), RPCError::InvalidData => write!(f, "Peer sent unexpected data"), RPCError::IoError(ref err) => write!(f, "IO Error: {}", err), - RPCError::ErrorResponse(ref code) => write!(f, "RPC response was an error: {}", code), + RPCError::ErrorResponse(ref code, ref reason) => write!( + f, + "RPC response was an error: {} with reason: {}", + code, reason + ), RPCError::StreamTimeout => write!(f, "Stream Timeout"), RPCError::UnsupportedProtocol => write!(f, "Peer does not support the protocol"), RPCError::IncompleteStream => write!(f, "Stream ended unexpectedly"), @@ -451,7 +442,7 @@ impl std::error::Error for RPCError { RPCError::IncompleteStream => None, RPCError::InvalidData => None, RPCError::InternalError(_) => None, - RPCError::ErrorResponse(_) => None, + RPCError::ErrorResponse(_, _) => None, RPCError::NegotiationTimeout => None, } } diff --git a/beacon_node/eth2-libp2p/src/service.rs b/beacon_node/eth2-libp2p/src/service.rs index efb2c062a..e527ef7b6 100644 --- a/beacon_node/eth2-libp2p/src/service.rs +++ b/beacon_node/eth2-libp2p/src/service.rs @@ -1,6 +1,7 @@ -use crate::behaviour::{Behaviour, BehaviourEvent}; +use crate::behaviour::{Behaviour, BehaviourEvent, Request, Response}; use crate::discovery::enr; use crate::multiaddr::Protocol; +use crate::rpc::{RPCResponseErrorCode, RequestId, SubstreamId}; use crate::types::{error, GossipKind}; use crate::EnrExt; use crate::{NetworkConfig, NetworkGlobals}; @@ -84,6 +85,7 @@ pub struct Service { impl Service { pub fn new( + executor: environment::TaskExecutor, config: &NetworkConfig, enr_fork_id: EnrForkId, log: &slog::Logger, @@ -122,15 +124,15 @@ impl Service { let behaviour = Behaviour::new(&local_keypair, config, network_globals.clone(), &log)?; // use the executor for libp2p - struct Executor(tokio::runtime::Handle); + struct Executor(environment::TaskExecutor); impl libp2p::core::Executor for Executor { fn exec(&self, f: Pin + Send>>) { - self.0.spawn(f); + self.0.spawn(f, "libp2p"); } } SwarmBuilder::new(transport, behaviour, local_peer_id.clone()) .peer_connection_limit(MAX_CONNECTIONS_PER_PEER) - .executor(Box::new(Executor(tokio::runtime::Handle::current()))) + .executor(Box::new(Executor(executor))) .build() }; @@ -228,126 +230,154 @@ impl Service { self.peer_ban_timeout.insert(peer_id, timeout); } + /// Sends a request to a peer, with a given Id. + pub fn send_request(&mut self, peer_id: PeerId, request_id: RequestId, request: Request) { + self.swarm.send_request(peer_id, request_id, request); + } + + /// Informs the peer that their request failed. + pub fn respond_with_error( + &mut self, + peer_id: PeerId, + stream_id: SubstreamId, + error: RPCResponseErrorCode, + reason: String, + ) { + self.swarm + ._send_error_reponse(peer_id, stream_id, error, reason); + } + + /// Sends a response to a peer's request. + pub fn send_response( + &mut self, + peer_id: PeerId, + stream_id: SubstreamId, + response: Response, + ) { + self.swarm + .send_successful_response(peer_id, stream_id, response); + } + pub async fn next_event(&mut self) -> Libp2pEvent { loop { tokio::select! { - event = self.swarm.next_event() => { - match event { - SwarmEvent::Behaviour(behaviour) => { - return Libp2pEvent::Behaviour(behaviour) - } - SwarmEvent::ConnectionEstablished { - peer_id, - endpoint, - num_established, - } => { - debug!(self.log, "Connection established"; "peer_id"=> peer_id.to_string(), "connections" => num_established.get()); - // if this is the first connection inform the network layer a new connection - // has been established and update the db - if num_established.get() == 1 { - // update the peerdb - match endpoint { - ConnectedPoint::Listener { .. } => { - self.swarm.peer_manager().connect_ingoing(&peer_id); + event = self.swarm.next_event() => { + match event { + SwarmEvent::Behaviour(behaviour) => { + return Libp2pEvent::Behaviour(behaviour) + } + SwarmEvent::ConnectionEstablished { + peer_id, + endpoint, + num_established, + } => { + debug!(self.log, "Connection established"; "peer_id" => peer_id.to_string(), "connections" => num_established.get()); + // if this is the first connection inform the network layer a new connection + // has been established and update the db + if num_established.get() == 1 { + // update the peerdb + match endpoint { + ConnectedPoint::Listener { .. } => { + self.swarm.peer_manager().connect_ingoing(&peer_id); + } + ConnectedPoint::Dialer { .. } => self + .network_globals + .peers + .write() + .connect_outgoing(&peer_id), } - ConnectedPoint::Dialer { .. } => self - .network_globals - .peers - .write() - .connect_outgoing(&peer_id), + return Libp2pEvent::PeerConnected { peer_id, endpoint }; } - return Libp2pEvent::PeerConnected { peer_id, endpoint }; } - } - SwarmEvent::ConnectionClosed { - peer_id, - cause, - endpoint, - num_established, - } => { - debug!(self.log, "Connection closed"; "peer_id"=> peer_id.to_string(), "cause" => cause.to_string(), "connections" => num_established); - if num_established == 0 { - // update the peer_db - self.swarm.peer_manager().notify_disconnect(&peer_id); - // the peer has disconnected - return Libp2pEvent::PeerDisconnected { - peer_id, - endpoint, - }; + SwarmEvent::ConnectionClosed { + peer_id, + cause, + endpoint, + num_established, + } => { + debug!(self.log, "Connection closed"; "peer_id"=> peer_id.to_string(), "cause" => cause.to_string(), "connections" => num_established); + if num_established == 0 { + // update the peer_db + self.swarm.peer_manager().notify_disconnect(&peer_id); + // the peer has disconnected + return Libp2pEvent::PeerDisconnected { + peer_id, + endpoint, + }; + } + } + SwarmEvent::NewListenAddr(multiaddr) => { + return Libp2pEvent::NewListenAddr(multiaddr) } - } - SwarmEvent::NewListenAddr(multiaddr) => { - return Libp2pEvent::NewListenAddr(multiaddr) - } - SwarmEvent::IncomingConnection { - local_addr, - send_back_addr, - } => { - debug!(self.log, "Incoming connection"; "our_addr" => local_addr.to_string(), "from" => send_back_addr.to_string()) - } - SwarmEvent::IncomingConnectionError { - local_addr, - send_back_addr, - error, - } => { - debug!(self.log, "Failed incoming connection"; "our_addr" => local_addr.to_string(), "from" => send_back_addr.to_string(), "error" => error.to_string()) - } - SwarmEvent::BannedPeer { - peer_id, - endpoint: _, - } => { - debug!(self.log, "Attempted to dial a banned peer"; "peer_id" => peer_id.to_string()) - } - SwarmEvent::UnreachableAddr { - peer_id, - address, - error, - attempts_remaining, - } => { - debug!(self.log, "Failed to dial address"; "peer_id" => peer_id.to_string(), "address" => address.to_string(), "error" => error.to_string(), "attempts_remaining" => attempts_remaining); - self.swarm.peer_manager().notify_disconnect(&peer_id); - } - SwarmEvent::UnknownPeerUnreachableAddr { address, error } => { - debug!(self.log, "Peer not known at dialed address"; "address" => address.to_string(), "error" => error.to_string()); - } - SwarmEvent::ExpiredListenAddr(multiaddr) => { - debug!(self.log, "Listen address expired"; "multiaddr" => multiaddr.to_string()) - } - SwarmEvent::ListenerClosed { addresses, reason } => { - debug!(self.log, "Listener closed"; "addresses" => format!("{:?}", addresses), "reason" => format!("{:?}", reason)) - } - SwarmEvent::ListenerError { error } => { - debug!(self.log, "Listener error"; "error" => format!("{:?}", error.to_string())) - } - SwarmEvent::Dialing(peer_id) => { - debug!(self.log, "Dialing peer"; "peer" => peer_id.to_string()); - self.swarm.peer_manager().dialing_peer(&peer_id); + SwarmEvent::IncomingConnection { + local_addr, + send_back_addr, + } => { + debug!(self.log, "Incoming connection"; "our_addr" => local_addr.to_string(), "from" => send_back_addr.to_string()) + } + SwarmEvent::IncomingConnectionError { + local_addr, + send_back_addr, + error, + } => { + debug!(self.log, "Failed incoming connection"; "our_addr" => local_addr.to_string(), "from" => send_back_addr.to_string(), "error" => error.to_string()) + } + SwarmEvent::BannedPeer { + peer_id, + endpoint: _, + } => { + debug!(self.log, "Attempted to dial a banned peer"; "peer_id" => peer_id.to_string()) + } + SwarmEvent::UnreachableAddr { + peer_id, + address, + error, + attempts_remaining, + } => { + debug!(self.log, "Failed to dial address"; "peer_id" => peer_id.to_string(), "address" => address.to_string(), "error" => error.to_string(), "attempts_remaining" => attempts_remaining); + self.swarm.peer_manager().notify_disconnect(&peer_id); + } + SwarmEvent::UnknownPeerUnreachableAddr { address, error } => { + debug!(self.log, "Peer not known at dialed address"; "address" => address.to_string(), "error" => error.to_string()); + } + SwarmEvent::ExpiredListenAddr(multiaddr) => { + debug!(self.log, "Listen address expired"; "multiaddr" => multiaddr.to_string()) + } + SwarmEvent::ListenerClosed { addresses, reason } => { + debug!(self.log, "Listener closed"; "addresses" => format!("{:?}", addresses), "reason" => format!("{:?}", reason)) + } + SwarmEvent::ListenerError { error } => { + debug!(self.log, "Listener error"; "error" => format!("{:?}", error.to_string())) + } + SwarmEvent::Dialing(peer_id) => { + debug!(self.log, "Dialing peer"; "peer" => peer_id.to_string()); + self.swarm.peer_manager().dialing_peer(&peer_id); + } } } - } - Some(Ok(peer_to_ban)) = self.peers_to_ban.next() => { - let peer_id = peer_to_ban.into_inner(); - Swarm::ban_peer_id(&mut self.swarm, peer_id.clone()); - // TODO: Correctly notify protocols of the disconnect - // TODO: Also remove peer from the DHT: https://github.com/sigp/lighthouse/issues/629 - self.swarm.inject_disconnected(&peer_id); - // inform the behaviour that the peer has been banned - self.swarm.peer_banned(peer_id); - } - Some(Ok(peer_to_unban)) = self.peer_ban_timeout.next() => { - debug!(self.log, "Peer has been unbanned"; "peer" => format!("{:?}", peer_to_unban)); - let unban_peer = peer_to_unban.into_inner(); - self.swarm.peer_unbanned(&unban_peer); - Swarm::unban_peer_id(&mut self.swarm, unban_peer); - } + Some(Ok(peer_to_ban)) = self.peers_to_ban.next() => { + let peer_id = peer_to_ban.into_inner(); + Swarm::ban_peer_id(&mut self.swarm, peer_id.clone()); + // TODO: Correctly notify protocols of the disconnect + // TODO: Also remove peer from the DHT: https://github.com/sigp/lighthouse/issues/629 + self.swarm.inject_disconnected(&peer_id); + // inform the behaviour that the peer has been banned + self.swarm.peer_banned(peer_id); + } + Some(Ok(peer_to_unban)) = self.peer_ban_timeout.next() => { + debug!(self.log, "Peer has been unbanned"; "peer" => format!("{:?}", peer_to_unban)); + let unban_peer = peer_to_unban.into_inner(); + self.swarm.peer_unbanned(&unban_peer); + Swarm::unban_peer_id(&mut self.swarm, unban_peer); + } } } } } -/// The implementation supports TCP/IP, WebSockets over TCP/IP, noise/secio as the encryption layer, and -/// mplex or yamux as the multiplexing layer. +/// The implementation supports TCP/IP, WebSockets over TCP/IP, noise/secio as the encryption +/// layer, and mplex or yamux as the multiplexing layer. fn build_transport( local_private_key: Keypair, ) -> Result, Error> { diff --git a/beacon_node/eth2-libp2p/tests/common/mod.rs b/beacon_node/eth2-libp2p/tests/common/mod.rs index 99857cb1a..e26381f51 100644 --- a/beacon_node/eth2-libp2p/tests/common/mod.rs +++ b/beacon_node/eth2-libp2p/tests/common/mod.rs @@ -12,6 +12,21 @@ use types::{EnrForkId, MinimalEthSpec}; type E = MinimalEthSpec; use tempdir::TempDir; +pub struct Libp2pInstance(LibP2PService, exit_future::Signal); + +impl std::ops::Deref for Libp2pInstance { + type Target = LibP2PService; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl std::ops::DerefMut for Libp2pInstance { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + pub fn build_log(level: slog::Level, enabled: bool) -> slog::Logger { let decorator = slog_term::TermDecorator::new().build(); let drain = slog_term::FullFormat::new(decorator).build().fuse(); @@ -82,13 +97,20 @@ pub fn build_libp2p_instance( boot_nodes: Vec, secret_key: Option, log: slog::Logger, -) -> LibP2PService { +) -> Libp2pInstance { let port = unused_port("tcp").unwrap(); let config = build_config(port, boot_nodes, secret_key); // launch libp2p service - LibP2PService::new(&config, EnrForkId::default(), &log) - .expect("should build libp2p instance") - .1 + + let (signal, exit) = exit_future::signal(); + let executor = + environment::TaskExecutor::new(tokio::runtime::Handle::current(), exit, log.clone()); + Libp2pInstance( + LibP2PService::new(executor, &config, EnrForkId::default(), &log) + .expect("should build libp2p instance") + .1, + signal, + ) } #[allow(dead_code)] @@ -99,8 +121,8 @@ pub fn get_enr(node: &LibP2PService) -> Enr { // Returns `n` libp2p peers in fully connected topology. #[allow(dead_code)] -pub fn build_full_mesh(log: slog::Logger, n: usize) -> Vec> { - let mut nodes: Vec> = (0..n) +pub fn build_full_mesh(log: slog::Logger, n: usize) -> Vec { + let mut nodes: Vec<_> = (0..n) .map(|_| build_libp2p_instance(vec![], None, log.clone())) .collect(); let multiaddrs: Vec = nodes @@ -124,7 +146,7 @@ pub fn build_full_mesh(log: slog::Logger, n: usize) -> Vec> { // Constructs a pair of nodes with separate loggers. The sender dials the receiver. // This returns a (sender, receiver) pair. #[allow(dead_code)] -pub async fn build_node_pair(log: &slog::Logger) -> (LibP2PService, LibP2PService) { +pub async fn build_node_pair(log: &slog::Logger) -> (Libp2pInstance, Libp2pInstance) { let sender_log = log.new(o!("who" => "sender")); let receiver_log = log.new(o!("who" => "receiver")); @@ -168,8 +190,8 @@ pub async fn build_node_pair(log: &slog::Logger) -> (LibP2PService, LibP2PSer // Returns `n` peers in a linear topology #[allow(dead_code)] -pub fn build_linear(log: slog::Logger, n: usize) -> Vec> { - let mut nodes: Vec> = (0..n) +pub fn build_linear(log: slog::Logger, n: usize) -> Vec { + let mut nodes: Vec<_> = (0..n) .map(|_| build_libp2p_instance(vec![], None, log.clone())) .collect(); let multiaddrs: Vec = nodes diff --git a/beacon_node/eth2-libp2p/tests/gossipsub_tests.rs b/beacon_node/eth2-libp2p/tests/gossipsub_tests.rs index 3fcd9b701..809920a56 100644 --- a/beacon_node/eth2-libp2p/tests/gossipsub_tests.rs +++ b/beacon_node/eth2-libp2p/tests/gossipsub_tests.rs @@ -1,3 +1,7 @@ +/* These are temporarily disabled due to their non-deterministic behaviour and impending update to + * gossipsub 1.1. We leave these here as a template for future test upgrades + + #![cfg(test)] use crate::types::GossipEncoding; use ::types::{BeaconBlock, EthSpec, MinimalEthSpec, Signature, SignedBeaconBlock}; @@ -164,3 +168,4 @@ async fn test_gossipsub_full_mesh_publish() { } } } +*/ diff --git a/beacon_node/eth2-libp2p/tests/noise.rs b/beacon_node/eth2-libp2p/tests/noise.rs index ba12c7346..973fb425c 100644 --- a/beacon_node/eth2-libp2p/tests/noise.rs +++ b/beacon_node/eth2-libp2p/tests/noise.rs @@ -136,7 +136,10 @@ async fn test_secio_noise_fallback() { let port = common::unused_port("tcp").unwrap(); let noisy_config = common::build_config(port, vec![], None); - let mut noisy_node = Service::new(&noisy_config, EnrForkId::default(), &log) + let (_signal, exit) = exit_future::signal(); + let executor = + environment::TaskExecutor::new(tokio::runtime::Handle::current(), exit, log.clone()); + let mut noisy_node = Service::new(executor, &noisy_config, EnrForkId::default(), &log) .expect("should build a libp2p instance") .1; diff --git a/beacon_node/eth2-libp2p/tests/rpc_tests.rs b/beacon_node/eth2-libp2p/tests/rpc_tests.rs index 847be0884..7218a64c0 100644 --- a/beacon_node/eth2-libp2p/tests/rpc_tests.rs +++ b/beacon_node/eth2-libp2p/tests/rpc_tests.rs @@ -1,7 +1,6 @@ #![cfg(test)] use eth2_libp2p::rpc::methods::*; -use eth2_libp2p::rpc::*; -use eth2_libp2p::{BehaviourEvent, Libp2pEvent, RPCEvent}; +use eth2_libp2p::{BehaviourEvent, Libp2pEvent, Request, Response}; use slog::{debug, warn, Level}; use std::time::Duration; use tokio::time::delay_for; @@ -26,7 +25,7 @@ async fn test_status_rpc() { let (mut sender, mut receiver) = common::build_node_pair(&log).await; // Dummy STATUS RPC message - let rpc_request = RPCRequest::Status(StatusMessage { + let rpc_request = Request::Status(StatusMessage { fork_digest: [0; 4], finalized_root: Hash256::from_low_u64_be(0), finalized_epoch: Epoch::new(1), @@ -35,7 +34,7 @@ async fn test_status_rpc() { }); // Dummy STATUS RPC message - let rpc_response = RPCResponse::Status(StatusMessage { + let rpc_response = Response::Status(StatusMessage { fork_digest: [0; 4], finalized_root: Hash256::from_low_u64_be(0), finalized_epoch: Epoch::new(1), @@ -52,26 +51,19 @@ async fn test_status_rpc() { debug!(log, "Sending RPC"); sender .swarm - .send_rpc(peer_id, RPCEvent::Request(10, rpc_request.clone())); + .send_request(peer_id, RequestId::Sync(10), rpc_request.clone()); } - Libp2pEvent::Behaviour(BehaviourEvent::RPC(_, event)) => match event { + Libp2pEvent::Behaviour(BehaviourEvent::ResponseReceived { + peer_id: _, + id: RequestId::Sync(10), + response, + }) => { // Should receive the RPC response - RPCEvent::Response(id, response @ RPCCodedResponse::Success(_)) => { - if id == 10 { - debug!(log, "Sender Received"); - let response = { - match response { - RPCCodedResponse::Success(r) => r, - _ => unreachable!(), - } - }; - assert_eq!(response, rpc_response.clone()); - debug!(log, "Sender Completed"); - return; - } - } - _ => {} // Ignore other RPC messages - }, + debug!(log, "Sender Received"); + assert_eq!(response, rpc_response.clone()); + debug!(log, "Sender Completed"); + return; + } _ => {} } } @@ -81,23 +73,17 @@ async fn test_status_rpc() { let receiver_future = async { loop { match receiver.next_event().await { - Libp2pEvent::Behaviour(BehaviourEvent::RPC(peer_id, event)) => { - match event { - // Should receive sent RPC request - RPCEvent::Request(id, request) => { - if request == rpc_request { - // send the response - debug!(log, "Receiver Received"); - receiver.swarm.send_rpc( - peer_id, - RPCEvent::Response( - id, - RPCCodedResponse::Success(rpc_response.clone()), - ), - ); - } - } - _ => {} // Ignore other RPC requests + Libp2pEvent::Behaviour(BehaviourEvent::RequestReceived { + peer_id, + id, + request, + }) => { + if request == rpc_request { + // send the response + debug!(log, "Receiver Received"); + receiver + .swarm + .send_successful_response(peer_id, id, rpc_response.clone()); } } _ => {} // Ignore other events @@ -108,7 +94,7 @@ async fn test_status_rpc() { tokio::select! { _ = sender_future => {} _ = receiver_future => {} - _ = delay_for(Duration::from_millis(800)) => { + _ = delay_for(Duration::from_millis(2000)) => { panic!("Future timed out"); } } @@ -129,7 +115,7 @@ async fn test_blocks_by_range_chunked_rpc() { let (mut sender, mut receiver) = common::build_node_pair(&log).await; // BlocksByRange Request - let rpc_request = RPCRequest::BlocksByRange(BlocksByRangeRequest { + let rpc_request = Request::BlocksByRange(BlocksByRangeRequest { start_slot: 0, count: messages_to_send, step: 0, @@ -142,7 +128,7 @@ async fn test_blocks_by_range_chunked_rpc() { message: empty_block, signature: Signature::empty_signature(), }; - let rpc_response = RPCResponse::BlocksByRange(Box::new(empty_signed)); + let rpc_response = Response::BlocksByRange(Some(Box::new(empty_signed))); // keep count of the number of messages received let mut messages_received = 0; @@ -155,31 +141,29 @@ async fn test_blocks_by_range_chunked_rpc() { debug!(log, "Sending RPC"); sender .swarm - .send_rpc(peer_id, RPCEvent::Request(10, rpc_request.clone())); + .send_request(peer_id, RequestId::Sync(10), rpc_request.clone()); } - Libp2pEvent::Behaviour(BehaviourEvent::RPC(_, event)) => match event { - // Should receive the RPC response - RPCEvent::Response(id, response) => { - if id == 10 { - warn!(log, "Sender received a response"); - match response { - RPCCodedResponse::Success(res) => { - assert_eq!(res, rpc_response.clone()); - messages_received += 1; - warn!(log, "Chunk received"); - } - RPCCodedResponse::StreamTermination(_) => { - // should be exactly 10 messages before terminating - assert_eq!(messages_received, messages_to_send); - // end the test - return; - } - _ => panic!("Invalid RPC received"), - } + Libp2pEvent::Behaviour(BehaviourEvent::ResponseReceived { + peer_id: _, + id: RequestId::Sync(10), + response, + }) => { + warn!(log, "Sender received a response"); + match response { + Response::BlocksByRange(Some(_)) => { + assert_eq!(response, rpc_response.clone()); + messages_received += 1; + warn!(log, "Chunk received"); } + Response::BlocksByRange(None) => { + // should be exactly 10 messages before terminating + assert_eq!(messages_received, messages_to_send); + // end the test + return; + } + _ => panic!("Invalid RPC received"), } - _ => {} // Ignore other RPC messages - }, + } _ => {} // Ignore other behaviour events } } @@ -189,36 +173,27 @@ async fn test_blocks_by_range_chunked_rpc() { let receiver_future = async { loop { match receiver.next_event().await { - Libp2pEvent::Behaviour(BehaviourEvent::RPC(peer_id, event)) => { - match event { - // Should receive sent RPC request - RPCEvent::Request(id, request) => { - if request == rpc_request { - // send the response - warn!(log, "Receiver got request"); - - for _ in 1..=messages_to_send { - receiver.swarm.send_rpc( - peer_id.clone(), - RPCEvent::Response( - id, - RPCCodedResponse::Success(rpc_response.clone()), - ), - ); - } - // send the stream termination - receiver.swarm.send_rpc( - peer_id, - RPCEvent::Response( - id, - RPCCodedResponse::StreamTermination( - ResponseTermination::BlocksByRange, - ), - ), - ); - } + Libp2pEvent::Behaviour(BehaviourEvent::RequestReceived { + peer_id, + id, + request, + }) => { + if request == rpc_request { + // send the response + warn!(log, "Receiver got request"); + for _ in 1..=messages_to_send { + receiver.swarm.send_successful_response( + peer_id.clone(), + id, + rpc_response.clone(), + ); } - _ => {} // Ignore other events + // send the stream termination + receiver.swarm.send_successful_response( + peer_id, + id, + Response::BlocksByRange(None), + ); } } _ => {} // Ignore other events @@ -229,7 +204,7 @@ async fn test_blocks_by_range_chunked_rpc() { tokio::select! { _ = sender_future => {} _ = receiver_future => {} - _ = delay_for(Duration::from_millis(800)) => { + _ = delay_for(Duration::from_millis(2000)) => { panic!("Future timed out"); } } @@ -251,7 +226,7 @@ async fn test_blocks_by_range_chunked_rpc_terminates_correctly() { let (mut sender, mut receiver) = common::build_node_pair(&log).await; // BlocksByRange Request - let rpc_request = RPCRequest::BlocksByRange(BlocksByRangeRequest { + let rpc_request = Request::BlocksByRange(BlocksByRangeRequest { start_slot: 0, count: messages_to_send, step: 0, @@ -264,7 +239,7 @@ async fn test_blocks_by_range_chunked_rpc_terminates_correctly() { message: empty_block, signature: Signature::empty_signature(), }; - let rpc_response = RPCResponse::BlocksByRange(Box::new(empty_signed)); + let rpc_response = Response::BlocksByRange(Some(Box::new(empty_signed))); // keep count of the number of messages received let mut messages_received: u64 = 0; @@ -277,28 +252,29 @@ async fn test_blocks_by_range_chunked_rpc_terminates_correctly() { debug!(log, "Sending RPC"); sender .swarm - .send_rpc(peer_id, RPCEvent::Request(10, rpc_request.clone())); + .send_request(peer_id, RequestId::Sync(10), rpc_request.clone()); } - Libp2pEvent::Behaviour(BehaviourEvent::RPC(_, event)) => match event { - // Should receive the RPC response - RPCEvent::Response(id, response) => { - if id == 10 { - debug!(log, "Sender received a response"); - match response { - RPCCodedResponse::Success(res) => { - assert_eq!(res, rpc_response.clone()); - messages_received += 1; - } - RPCCodedResponse::StreamTermination(_) => { - // should be exactly 10 messages, as requested - assert_eq!(messages_received, messages_to_send); - } - _ => panic!("Invalid RPC received"), - } + Libp2pEvent::Behaviour(BehaviourEvent::ResponseReceived { + peer_id: _, + id: RequestId::Sync(10), + response, + }) => + // Should receive the RPC response + { + debug!(log, "Sender received a response"); + match response { + Response::BlocksByRange(Some(_)) => { + assert_eq!(response, rpc_response.clone()); + messages_received += 1; } + Response::BlocksByRange(None) => { + // should be exactly 10 messages, as requested + assert_eq!(messages_received, messages_to_send); + } + _ => panic!("Invalid RPC received"), } - _ => {} // Ignore other RPC messages - }, + } + _ => {} // Ignore other behaviour events } } @@ -320,21 +296,17 @@ async fn test_blocks_by_range_chunked_rpc_terminates_correctly() { .await { futures::future::Either::Left(( - Libp2pEvent::Behaviour(BehaviourEvent::RPC(peer_id, event)), + Libp2pEvent::Behaviour(BehaviourEvent::RequestReceived { + peer_id, + id, + request, + }), _, )) => { - match event { - // Should receive sent RPC request - RPCEvent::Request(id, request) => { - if request == rpc_request { - // send the response - warn!(log, "Receiver got request"); - message_info = Some((peer_id, id)); - } else { - continue; - } - } - _ => continue, // Ignore other events, don't send messages until ready + if request == rpc_request { + // send the response + warn!(log, "Receiver got request"); + message_info = Some((peer_id, id)); } } futures::future::Either::Right((_, _)) => {} // The timeout hit, send messages if required @@ -344,12 +316,11 @@ async fn test_blocks_by_range_chunked_rpc_terminates_correctly() { // if we need to send messages send them here. This will happen after a delay if message_info.is_some() { messages_sent += 1; - receiver.swarm.send_rpc( - message_info.as_ref().unwrap().0.clone(), - RPCEvent::Response( - message_info.as_ref().unwrap().1.clone(), - RPCCodedResponse::Success(rpc_response.clone()), - ), + let (peer_id, stream_id) = message_info.as_ref().unwrap(); + receiver.swarm.send_successful_response( + peer_id.clone(), + stream_id.clone(), + rpc_response.clone(), ); debug!(log, "Sending message {}", messages_sent); if messages_sent == messages_to_send + extra_messages_to_send { @@ -363,7 +334,7 @@ async fn test_blocks_by_range_chunked_rpc_terminates_correctly() { tokio::select! { _ = sender_future => {} _ = receiver_future => {} - _ = delay_for(Duration::from_millis(50000)) => { + _ = delay_for(Duration::from_millis(2000)) => { panic!("Future timed out"); } } @@ -382,7 +353,7 @@ async fn test_blocks_by_range_single_empty_rpc() { let (mut sender, mut receiver) = common::build_node_pair(&log).await; // BlocksByRange Request - let rpc_request = RPCRequest::BlocksByRange(BlocksByRangeRequest { + let rpc_request = Request::BlocksByRange(BlocksByRangeRequest { start_slot: 0, count: 10, step: 0, @@ -395,7 +366,7 @@ async fn test_blocks_by_range_single_empty_rpc() { message: empty_block, signature: Signature::empty_signature(), }; - let rpc_response = RPCResponse::BlocksByRange(Box::new(empty_signed)); + let rpc_response = Response::BlocksByRange(Some(Box::new(empty_signed))); let messages_to_send = 1; @@ -410,30 +381,25 @@ async fn test_blocks_by_range_single_empty_rpc() { debug!(log, "Sending RPC"); sender .swarm - .send_rpc(peer_id, RPCEvent::Request(10, rpc_request.clone())); + .send_request(peer_id, RequestId::Sync(10), rpc_request.clone()); } - Libp2pEvent::Behaviour(BehaviourEvent::RPC(_, event)) => match event { - // Should receive the RPC response - RPCEvent::Response(id, response) => { - if id == 10 { - warn!(log, "Sender received a response"); - match response { - RPCCodedResponse::Success(res) => { - assert_eq!(res, rpc_response.clone()); - messages_received += 1; - warn!(log, "Chunk received"); - } - RPCCodedResponse::StreamTermination(_) => { - // should be exactly 10 messages before terminating - assert_eq!(messages_received, messages_to_send); - // end the test - return; - } - _ => panic!("Invalid RPC received"), - } - } + Libp2pEvent::Behaviour(BehaviourEvent::ResponseReceived { + peer_id: _, + id: RequestId::Sync(10), + response, + }) => match response { + Response::BlocksByRange(Some(_)) => { + assert_eq!(response, rpc_response.clone()); + messages_received += 1; + warn!(log, "Chunk received"); } - _ => {} // Ignore other RPC messages + Response::BlocksByRange(None) => { + // should be exactly 10 messages before terminating + assert_eq!(messages_received, messages_to_send); + // end the test + return; + } + _ => panic!("Invalid RPC received"), }, _ => {} // Ignore other behaviour events } @@ -444,36 +410,28 @@ async fn test_blocks_by_range_single_empty_rpc() { let receiver_future = async { loop { match receiver.next_event().await { - Libp2pEvent::Behaviour(BehaviourEvent::RPC(peer_id, event)) => { - match event { - // Should receive sent RPC request - RPCEvent::Request(id, request) => { - if request == rpc_request { - // send the response - warn!(log, "Receiver got request"); + Libp2pEvent::Behaviour(BehaviourEvent::RequestReceived { + peer_id, + id, + request, + }) => { + if request == rpc_request { + // send the response + warn!(log, "Receiver got request"); - for _ in 1..=messages_to_send { - receiver.swarm.send_rpc( - peer_id.clone(), - RPCEvent::Response( - id, - RPCCodedResponse::Success(rpc_response.clone()), - ), - ); - } - // send the stream termination - receiver.swarm.send_rpc( - peer_id, - RPCEvent::Response( - id, - RPCCodedResponse::StreamTermination( - ResponseTermination::BlocksByRange, - ), - ), - ); - } + for _ in 1..=messages_to_send { + receiver.swarm.send_successful_response( + peer_id.clone(), + id, + rpc_response.clone(), + ); } - _ => {} // Ignore other events + // send the stream termination + receiver.swarm.send_successful_response( + peer_id, + id, + Response::BlocksByRange(None), + ); } } _ => {} // Ignore other events @@ -508,7 +466,7 @@ async fn test_blocks_by_root_chunked_rpc() { let (mut sender, mut receiver) = common::build_node_pair(&log).await; // BlocksByRoot Request - let rpc_request = RPCRequest::BlocksByRoot(BlocksByRootRequest { + let rpc_request = Request::BlocksByRoot(BlocksByRootRequest { block_roots: vec![ Hash256::from_low_u64_be(0), Hash256::from_low_u64_be(0), @@ -522,7 +480,7 @@ async fn test_blocks_by_root_chunked_rpc() { message: full_block, signature: Signature::empty_signature(), }; - let rpc_response = RPCResponse::BlocksByRoot(Box::new(signed_full_block)); + let rpc_response = Response::BlocksByRoot(Some(Box::new(signed_full_block))); // keep count of the number of messages received let mut messages_received = 0; @@ -535,28 +493,23 @@ async fn test_blocks_by_root_chunked_rpc() { debug!(log, "Sending RPC"); sender .swarm - .send_rpc(peer_id, RPCEvent::Request(10, rpc_request.clone())); + .send_request(peer_id, RequestId::Sync(10), rpc_request.clone()); } - Libp2pEvent::Behaviour(BehaviourEvent::RPC(_, event)) => match event { - // Should receive the RPC response - RPCEvent::Response(id, response) => { - if id == 10 { - debug!(log, "Sender received a response"); - match response { - RPCCodedResponse::Success(res) => { - assert_eq!(res, rpc_response.clone()); - messages_received += 1; - debug!(log, "Chunk received"); - } - RPCCodedResponse::StreamTermination(_) => { - // should be exactly messages_to_send - assert_eq!(messages_received, messages_to_send); - // end the test - return; - } - _ => {} // Ignore other RPC messages - } - } + Libp2pEvent::Behaviour(BehaviourEvent::ResponseReceived { + peer_id: _, + id: RequestId::Sync(10), + response, + }) => match response { + Response::BlocksByRoot(Some(_)) => { + assert_eq!(response, rpc_response.clone()); + messages_received += 1; + debug!(log, "Chunk received"); + } + Response::BlocksByRoot(None) => { + // should be exactly messages_to_send + assert_eq!(messages_received, messages_to_send); + // end the test + return; } _ => {} // Ignore other RPC messages }, @@ -569,38 +522,30 @@ async fn test_blocks_by_root_chunked_rpc() { let receiver_future = async { loop { match receiver.next_event().await { - Libp2pEvent::Behaviour(BehaviourEvent::RPC(peer_id, event)) => { - match event { - // Should receive sent RPC request - RPCEvent::Request(id, request) => { - if request == rpc_request { - // send the response - debug!(log, "Receiver got request"); + Libp2pEvent::Behaviour(BehaviourEvent::RequestReceived { + peer_id, + id, + request, + }) => { + if request == rpc_request { + // send the response + debug!(log, "Receiver got request"); - for _ in 1..=messages_to_send { - receiver.swarm.send_rpc( - peer_id.clone(), - RPCEvent::Response( - id, - RPCCodedResponse::Success(rpc_response.clone()), - ), - ); - debug!(log, "Sending message"); - } - // send the stream termination - receiver.swarm.send_rpc( - peer_id, - RPCEvent::Response( - id, - RPCCodedResponse::StreamTermination( - ResponseTermination::BlocksByRange, - ), - ), - ); - debug!(log, "Send stream term"); - } + for _ in 1..=messages_to_send { + receiver.swarm.send_successful_response( + peer_id.clone(), + id, + rpc_response.clone(), + ); + debug!(log, "Sending message"); } - _ => {} // Ignore other events + // send the stream termination + receiver.swarm.send_successful_response( + peer_id, + id, + Response::BlocksByRange(None), + ); + debug!(log, "Send stream term"); } } _ => {} // Ignore other events @@ -610,7 +555,7 @@ async fn test_blocks_by_root_chunked_rpc() { tokio::select! { _ = sender_future => {} _ = receiver_future => {} - _ = delay_for(Duration::from_millis(1000)) => { + _ = delay_for(Duration::from_millis(2000)) => { panic!("Future timed out"); } } @@ -633,7 +578,7 @@ async fn test_blocks_by_root_chunked_rpc_terminates_correctly() { let (mut sender, mut receiver) = common::build_node_pair(&log).await; // BlocksByRoot Request - let rpc_request = RPCRequest::BlocksByRoot(BlocksByRootRequest { + let rpc_request = Request::BlocksByRoot(BlocksByRootRequest { block_roots: vec![ Hash256::from_low_u64_be(0), Hash256::from_low_u64_be(0), @@ -654,7 +599,7 @@ async fn test_blocks_by_root_chunked_rpc_terminates_correctly() { message: full_block, signature: Signature::empty_signature(), }; - let rpc_response = RPCResponse::BlocksByRoot(Box::new(signed_full_block)); + let rpc_response = Response::BlocksByRoot(Some(Box::new(signed_full_block))); // keep count of the number of messages received let mut messages_received = 0; @@ -667,31 +612,29 @@ async fn test_blocks_by_root_chunked_rpc_terminates_correctly() { debug!(log, "Sending RPC"); sender .swarm - .send_rpc(peer_id, RPCEvent::Request(10, rpc_request.clone())); + .send_request(peer_id, RequestId::Sync(10), rpc_request.clone()); } - Libp2pEvent::Behaviour(BehaviourEvent::RPC(_, event)) => match event { - // Should receive the RPC response - RPCEvent::Response(id, response) => { - if id == 10 { - debug!(log, "Sender received a response"); - match response { - RPCCodedResponse::Success(res) => { - assert_eq!(res, rpc_response.clone()); - messages_received += 1; - debug!(log, "Chunk received"); - } - RPCCodedResponse::StreamTermination(_) => { - // should be exactly messages_to_send - assert_eq!(messages_received, messages_to_send); - // end the test - return; - } - _ => {} // Ignore other RPC messages - } + Libp2pEvent::Behaviour(BehaviourEvent::ResponseReceived { + peer_id: _, + id: RequestId::Sync(10), + response, + }) => { + debug!(log, "Sender received a response"); + match response { + Response::BlocksByRoot(Some(_)) => { + assert_eq!(response, rpc_response.clone()); + messages_received += 1; + debug!(log, "Chunk received"); } + Response::BlocksByRoot(None) => { + // should be exactly messages_to_send + assert_eq!(messages_received, messages_to_send); + // end the test + return; + } + _ => {} // Ignore other RPC messages } - _ => {} // Ignore other RPC messages - }, + } _ => {} // Ignore other behaviour events } } @@ -713,21 +656,17 @@ async fn test_blocks_by_root_chunked_rpc_terminates_correctly() { .await { futures::future::Either::Left(( - Libp2pEvent::Behaviour(BehaviourEvent::RPC(peer_id, event)), + Libp2pEvent::Behaviour(BehaviourEvent::RequestReceived { + peer_id, + id, + request, + }), _, )) => { - match event { - // Should receive sent RPC request - RPCEvent::Request(id, request) => { - if request == rpc_request { - // send the response - warn!(log, "Receiver got request"); - message_info = Some((peer_id, id)); - } else { - continue; - } - } - _ => continue, // Ignore other events, don't send messages until ready + if request == rpc_request { + // send the response + warn!(log, "Receiver got request"); + message_info = Some((peer_id, id)); } } futures::future::Either::Right((_, _)) => {} // The timeout hit, send messages if required @@ -737,12 +676,11 @@ async fn test_blocks_by_root_chunked_rpc_terminates_correctly() { // if we need to send messages send them here. This will happen after a delay if message_info.is_some() { messages_sent += 1; - receiver.swarm.send_rpc( - message_info.as_ref().unwrap().0.clone(), - RPCEvent::Response( - message_info.as_ref().unwrap().1.clone(), - RPCCodedResponse::Success(rpc_response.clone()), - ), + let (peer_id, stream_id) = message_info.as_ref().unwrap(); + receiver.swarm.send_successful_response( + peer_id.clone(), + stream_id.clone(), + rpc_response.clone(), ); debug!(log, "Sending message {}", messages_sent); if messages_sent == messages_to_send + extra_messages_to_send { @@ -756,7 +694,7 @@ async fn test_blocks_by_root_chunked_rpc_terminates_correctly() { tokio::select! { _ = sender_future => {} _ = receiver_future => {} - _ = delay_for(Duration::from_millis(1000)) => { + _ = delay_for(Duration::from_millis(2000)) => { panic!("Future timed out"); } } @@ -775,7 +713,7 @@ async fn test_goodbye_rpc() { let (mut sender, mut receiver) = common::build_node_pair(&log).await; // Goodbye Request - let rpc_request = RPCRequest::Goodbye(GoodbyeReason::ClientShutdown); + let rpc_request = Request::Goodbye(GoodbyeReason::ClientShutdown); // build the sender future let sender_future = async { @@ -786,7 +724,7 @@ async fn test_goodbye_rpc() { debug!(log, "Sending RPC"); sender .swarm - .send_rpc(peer_id, RPCEvent::Request(10, rpc_request.clone())); + .send_request(peer_id, RequestId::Sync(10), rpc_request.clone()); } _ => {} // Ignore other RPC messages } @@ -797,18 +735,14 @@ async fn test_goodbye_rpc() { let receiver_future = async { loop { match receiver.next_event().await { - Libp2pEvent::Behaviour(BehaviourEvent::RPC(_peer_id, event)) => { - match event { - // Should receive sent RPC request - RPCEvent::Request(id, request) => { - if request == rpc_request { - assert_eq!(id, 0); - assert_eq!(rpc_request.clone(), request); // receives the goodbye. Nothing left to do - return; - } - } - _ => {} // Ignore other events - } + Libp2pEvent::Behaviour(BehaviourEvent::RequestReceived { + peer_id: _, + id: _, + request, + }) => { + // Should receive sent RPC request + assert_eq!(rpc_request.clone(), request); // receives the goodbye. Nothing left to do + return; } _ => {} // Ignore other events } @@ -818,7 +752,7 @@ async fn test_goodbye_rpc() { tokio::select! { _ = sender_future => {} _ = receiver_future => {} - _ = delay_for(Duration::from_millis(1000)) => { + _ = delay_for(Duration::from_millis(2000)) => { panic!("Future timed out"); } } diff --git a/beacon_node/genesis/Cargo.toml b/beacon_node/genesis/Cargo.toml index 8b01f7fde..3100f060a 100644 --- a/beacon_node/genesis/Cargo.toml +++ b/beacon_node/genesis/Cargo.toml @@ -18,7 +18,7 @@ merkle_proof = { path = "../../consensus/merkle_proof" } eth2_ssz = "0.1.2" eth2_hashing = "0.1.0" tree_hash = "0.1.0" -tokio = { version = "0.2.20", features = ["full"] } +tokio = { version = "0.2.21", features = ["full"] } parking_lot = "0.10.2" slog = "2.5.2" exit-future = "0.2.0" diff --git a/beacon_node/genesis/tests/tests.rs b/beacon_node/genesis/tests/tests.rs index 7e2131f51..b4f72d8c8 100644 --- a/beacon_node/genesis/tests/tests.rs +++ b/beacon_node/genesis/tests/tests.rs @@ -24,7 +24,7 @@ pub fn new_env() -> Environment { #[test] fn basic() { let mut env = new_env(); - let log = env.core_context().log.clone(); + let log = env.core_context().log().clone(); let mut spec = env.eth2_config().spec.clone(); env.runtime().block_on(async { diff --git a/beacon_node/network/Cargo.toml b/beacon_node/network/Cargo.toml index 56a6b3700..e9181e88b 100644 --- a/beacon_node/network/Cargo.toml +++ b/beacon_node/network/Cargo.toml @@ -10,6 +10,8 @@ genesis = { path = "../genesis" } lazy_static = "1.4.0" matches = "0.1.8" tempfile = "3.1.0" +exit-future = "0.2.0" +assert_approx_eq = "1.1.0" [dependencies] beacon_chain = { path = "../beacon_chain" } @@ -25,7 +27,7 @@ eth2_ssz = "0.1.2" tree_hash = "0.1.0" futures = "0.3.5" error-chain = "0.12.2" -tokio = { version = "0.2.20", features = ["full"] } +tokio = { version = "0.2.21", features = ["full"] } parking_lot = "0.10.2" smallvec = "1.4.0" # TODO: Remove rand crate for mainnet @@ -34,3 +36,4 @@ fnv = "1.0.6" rlp = "0.4.5" lazy_static = "1.4.0" lighthouse_metrics = { path = "../../common/lighthouse_metrics" } +environment = { path = "../../lighthouse/environment" } diff --git a/beacon_node/network/src/attestation_service/mod.rs b/beacon_node/network/src/attestation_service/mod.rs index 90f67629f..2fb8facea 100644 --- a/beacon_node/network/src/attestation_service/mod.rs +++ b/beacon_node/network/src/attestation_service/mod.rs @@ -28,15 +28,19 @@ const MIN_PEER_DISCOVERY_SLOT_LOOK_AHEAD: u64 = 1; const TARGET_PEER_DISCOVERY_SLOT_LOOK_AHEAD: u64 = 6; /// The time (in slots) before a last seen validator is considered absent and we unsubscribe from the random /// gossip topics that we subscribed to due to the validator connection. -const LAST_SEEN_VALIDATOR_TIMEOUT: u32 = 150; // 30 mins at a 12s slot time +const LAST_SEEN_VALIDATOR_TIMEOUT: u32 = 150; +// 30 mins at a 12s slot time /// The fraction of a slot that we subscribe to a subnet before the required slot. /// /// Note: The time is calculated as `time = milliseconds_per_slot / ADVANCE_SUBSCRIPTION_TIME`. const ADVANCE_SUBSCRIBE_TIME: u32 = 3; /// The default number of slots before items in hash delay sets used by this class should expire. -const DEFAULT_EXPIRATION_TIMEOUT: u32 = 3; // 36s at 12s slot time +const DEFAULT_EXPIRATION_TIMEOUT: u32 = 3; +// 36s at 12s slot time +/// The default number of slots before items in hash delay sets used by this class should expire. +const DURATION_DIFFERENCE: Duration = Duration::from_millis(1); -#[derive(Debug, PartialEq, Eq, Clone)] +#[derive(Debug, Eq, Clone)] pub enum AttServiceMessage { /// Subscribe to the specified subnet id. Subscribe(SubnetId), @@ -47,12 +51,45 @@ pub enum AttServiceMessage { /// Remove the `SubnetId` from the ENR bitfield. EnrRemove(SubnetId), /// Discover peers for a particular subnet. - DiscoverPeers(SubnetId), + /// The includes the `Instant` we need the discovered peer until. + DiscoverPeers { + subnet_id: SubnetId, + min_ttl: Option, + }, +} + +impl PartialEq for AttServiceMessage { + fn eq(&self, other: &AttServiceMessage) -> bool { + match (self, other) { + (&AttServiceMessage::Subscribe(a), &AttServiceMessage::Subscribe(b)) => a == b, + (&AttServiceMessage::Unsubscribe(a), &AttServiceMessage::Unsubscribe(b)) => a == b, + (&AttServiceMessage::EnrAdd(a), &AttServiceMessage::EnrAdd(b)) => a == b, + (&AttServiceMessage::EnrRemove(a), &AttServiceMessage::EnrRemove(b)) => a == b, + ( + &AttServiceMessage::DiscoverPeers { subnet_id, min_ttl }, + &AttServiceMessage::DiscoverPeers { + subnet_id: other_subnet_id, + min_ttl: other_min_ttl, + }, + ) => match (min_ttl, other_min_ttl) { + (Some(min_ttl_instant), Some(other_min_ttl_instant)) => { + min_ttl_instant.saturating_duration_since(other_min_ttl_instant) + < DURATION_DIFFERENCE + && other_min_ttl_instant.saturating_duration_since(min_ttl_instant) + < DURATION_DIFFERENCE + && subnet_id == other_subnet_id + } + (None, None) => subnet_id == other_subnet_id, + _ => false, + }, + _ => false, + } + } } /// A particular subnet at a given slot. -#[derive(PartialEq, Eq, Hash, Clone)] -struct ExactSubnet { +#[derive(PartialEq, Eq, Hash, Clone, Debug)] +pub struct ExactSubnet { /// The `SubnetId` associated with this subnet. pub subnet_id: SubnetId, /// The `Slot` associated with this subnet. @@ -244,24 +281,18 @@ impl AttestationService { return Ok(()); } - // check current event log to see if there is a discovery event queued - if self - .events - .iter() - .find(|event| event == &&AttServiceMessage::DiscoverPeers(exact_subnet.subnet_id)) - .is_some() - { - // already queued a discovery event - return Ok(()); - } - // if the slot is more than epoch away, add an event to start looking for peers if exact_subnet.slot < current_slot.saturating_add(TARGET_PEER_DISCOVERY_SLOT_LOOK_AHEAD) { - // then instantly add a discovery request - self.events - .push_back(AttServiceMessage::DiscoverPeers(exact_subnet.subnet_id)); + // add one slot to ensure we keep the peer for the subscription slot + let min_ttl = self + .beacon_chain + .slot_clock + .duration_to_slot(exact_subnet.slot + 1) + .map(|duration| std::time::Instant::now() + duration); + + self.send_or_update_discovery_event(exact_subnet.subnet_id, min_ttl); } else { // Queue the discovery event to be executed for // TARGET_PEER_DISCOVERY_SLOT_LOOK_AHEAD @@ -296,6 +327,52 @@ impl AttestationService { Ok(()) } + /// Checks if we have a discover peers event already and sends a new event if necessary + /// + /// If a message exists for the same subnet, compare the `min_ttl` of the current and + /// existing messages and extend the existing message as necessary. + fn send_or_update_discovery_event(&mut self, subnet_id: SubnetId, min_ttl: Option) { + // track whether this message already exists in the event queue + let mut is_duplicate = false; + + self.events.iter_mut().for_each(|event| { + match event { + AttServiceMessage::DiscoverPeers { + subnet_id: other_subnet_id, + min_ttl: other_min_ttl, + } => { + if subnet_id == *other_subnet_id { + let other_min_ttl_clone = other_min_ttl.clone(); + match (min_ttl, other_min_ttl_clone) { + (Some(min_ttl_instant), Some(other_min_ttl_instant)) => + // only update the min_ttl if it is greater than the existing min_ttl and a DURATION_DIFFERENCE padding + { + if min_ttl_instant.saturating_duration_since(other_min_ttl_instant) + > DURATION_DIFFERENCE + { + *other_min_ttl = min_ttl; + } + } + (None, Some(_)) => { + // Update the min_ttl to None, because the new message is longer-lived. + *other_min_ttl = None; + } + (Some(_), None) => {} // Don't replace this because the existing message is for a longer-lived peer. + (None, None) => {} // Duplicate message, do nothing. + } + is_duplicate = true; + return; + } + } + _ => {} + }; + }); + if !is_duplicate { + self.events + .push_back(AttServiceMessage::DiscoverPeers { subnet_id, min_ttl }); + } + } + /// Checks the current random subnets and subscriptions to determine if a new subscription for this /// subnet is required for the given slot. /// @@ -436,18 +513,17 @@ impl AttestationService { // if we are not already subscribed, then subscribe let topic_kind = &GossipKind::CommitteeIndex(subnet_id); - if let None = self + let already_subscribed = self .network_globals .gossipsub_subscriptions .read() .iter() .find(|topic| topic.kind() == topic_kind) - { - // not already subscribed to the topic + .is_some(); + if !already_subscribed { // send a discovery request and a subscription - self.events - .push_back(AttServiceMessage::DiscoverPeers(subnet_id)); + self.send_or_update_discovery_event(subnet_id, None); self.events .push_back(AttServiceMessage::Subscribe(subnet_id)); } @@ -461,8 +537,15 @@ impl AttestationService { /// Request a discovery query to find peers for a particular subnet. fn handle_discover_peers(&mut self, exact_subnet: ExactSubnet) { debug!(self.log, "Searching for peers for subnet"; "subnet" => *exact_subnet.subnet_id, "target_slot" => exact_subnet.slot); - self.events - .push_back(AttServiceMessage::DiscoverPeers(exact_subnet.subnet_id)); + + // add one slot to ensure we keep the peer for the subscription slot + let min_ttl = self + .beacon_chain + .slot_clock + .duration_to_slot(exact_subnet.slot + 1) + .map(|duration| std::time::Instant::now() + duration); + + self.send_or_update_discovery_event(exact_subnet.subnet_id, min_ttl) } /// A queued subscription is ready. @@ -619,7 +702,7 @@ impl Stream for AttestationService { match self.discover_peers.poll_next_unpin(cx) { Poll::Ready(Some(Ok(exact_subnet))) => self.handle_discover_peers(exact_subnet), Poll::Ready(Some(Err(e))) => { - error!(self.log, "Failed to check for peer discovery requests"; "error"=> format!("{}", e)); + error!(self.log, "Failed to check for peer discovery requests"; "error"=> format ! ("{}", e)); } Poll::Ready(None) | Poll::Pending => {} } diff --git a/beacon_node/network/src/attestation_service/tests.rs b/beacon_node/network/src/attestation_service/tests/mod.rs similarity index 87% rename from beacon_node/network/src/attestation_service/tests.rs rename to beacon_node/network/src/attestation_service/tests/mod.rs index c0d32248f..568a551c9 100644 --- a/beacon_node/network/src/attestation_service/tests.rs +++ b/beacon_node/network/src/attestation_service/tests/mod.rs @@ -16,10 +16,9 @@ mod tests { use slog::Logger; use sloggers::{null::NullLoggerBuilder, Build}; use slot_clock::{SlotClock, SystemTimeSlotClock}; - use std::time::SystemTime; + use std::time::{Duration, SystemTime}; use store::MemoryStore; use tempfile::tempdir; - use tokio::time::Duration; use types::{CommitteeIndex, EnrForkId, EthSpec, MinimalEthSpec}; const SLOT_DURATION_MILLIS: u64 = 200; @@ -192,7 +191,10 @@ mod tests { assert_matches!( events[..3], [ - AttServiceMessage::DiscoverPeers(_any2), + AttServiceMessage::DiscoverPeers { + subnet_id: _any_subnet, + min_ttl: _any_instant + }, AttServiceMessage::Subscribe(_any1), AttServiceMessage::EnrAdd(_any3) ] @@ -240,7 +242,10 @@ mod tests { assert_matches!( events[..3], [ - AttServiceMessage::DiscoverPeers(_any2), + AttServiceMessage::DiscoverPeers { + subnet_id: _any_subnet, + min_ttl: _any_instant + }, AttServiceMessage::Subscribe(_any1), AttServiceMessage::EnrAdd(_any3) ] @@ -278,16 +283,28 @@ mod tests { .validator_subscriptions(subscriptions) .unwrap(); + let min_ttl = Instant::now().checked_add( + attestation_service + .beacon_chain + .slot_clock + .duration_to_slot(current_slot + Slot::new(subscription_slot) + Slot::new(1)) + .unwrap(), + ); + // just discover peers, don't subscribe yet - let expected = vec![AttServiceMessage::DiscoverPeers(SubnetId::new( - validator_index, - ))]; + let expected = vec![AttServiceMessage::DiscoverPeers { + subnet_id: SubnetId::new(validator_index), + min_ttl, + }]; let events = get_events(attestation_service, no_events_expected, 1).await; assert_matches!( events[..3], [ - AttServiceMessage::DiscoverPeers(_any1), + AttServiceMessage::DiscoverPeers { + subnet_id: _any_subnet, + min_ttl: _any_instant + }, AttServiceMessage::Subscribe(_any2), AttServiceMessage::EnrAdd(_any3) ] @@ -325,9 +342,20 @@ mod tests { .validator_subscriptions(subscriptions) .unwrap(); + let min_ttl = Instant::now().checked_add( + attestation_service + .beacon_chain + .slot_clock + .duration_to_slot(current_slot + Slot::new(subscription_slot) + Slot::new(1)) + .unwrap(), + ); + // we should discover peers, wait, then subscribe let expected = vec![ - AttServiceMessage::DiscoverPeers(SubnetId::new(validator_index)), + AttServiceMessage::DiscoverPeers { + subnet_id: SubnetId::new(validator_index), + min_ttl, + }, AttServiceMessage::Subscribe(SubnetId::new(validator_index)), ]; @@ -335,7 +363,10 @@ mod tests { assert_matches!( events[..3], [ - AttServiceMessage::DiscoverPeers(_any1), + AttServiceMessage::DiscoverPeers { + subnet_id: _any_subnet, + min_ttl: _any_instant + }, AttServiceMessage::Subscribe(_any2), AttServiceMessage::EnrAdd(_any3) ] @@ -381,7 +412,10 @@ mod tests { assert_matches!( events[..3], [ - AttServiceMessage::DiscoverPeers(_any1), + AttServiceMessage::DiscoverPeers { + subnet_id: _any_subnet, + min_ttl: _any_instant + }, AttServiceMessage::Subscribe(_any2), AttServiceMessage::EnrAdd(_any3) ] @@ -419,17 +453,29 @@ mod tests { .validator_subscriptions(subscriptions) .unwrap(); + let min_ttl = Instant::now().checked_add( + attestation_service + .beacon_chain + .slot_clock + .duration_to_slot(current_slot + Slot::new(subscription_slot) + Slot::new(1)) + .unwrap(), + ); + // expect discover peers because we will enter TARGET_PEER_DISCOVERY_SLOT_LOOK_AHEAD range - let expected: Vec = vec![AttServiceMessage::DiscoverPeers( - SubnetId::new(validator_index), - )]; + let expected: Vec = vec![AttServiceMessage::DiscoverPeers { + subnet_id: SubnetId::new(validator_index), + min_ttl, + }]; let events = get_events(attestation_service, no_events_expected, 5).await; assert_matches!( events[..3], [ - AttServiceMessage::DiscoverPeers(_any1), + AttServiceMessage::DiscoverPeers { + subnet_id: _any_subnet, + min_ttl: _any_instant + }, AttServiceMessage::Subscribe(_any2), AttServiceMessage::EnrAdd(_any3) ] @@ -470,9 +516,10 @@ mod tests { for event in events { match event { - AttServiceMessage::DiscoverPeers(_any_subnet) => { - discover_peer_count = discover_peer_count + 1 - } + AttServiceMessage::DiscoverPeers { + subnet_id: _any_subnet, + min_ttl: _any_instant, + } => discover_peer_count = discover_peer_count + 1, AttServiceMessage::Subscribe(_any_subnet) => subscribe_count = subscribe_count + 1, AttServiceMessage::EnrAdd(_any_subnet) => enr_add_count = enr_add_count + 1, _ => unexpected_msg_count = unexpected_msg_count + 1, @@ -517,9 +564,10 @@ mod tests { for event in events { match event { - AttServiceMessage::DiscoverPeers(_any_subnet) => { - discover_peer_count = discover_peer_count + 1 - } + AttServiceMessage::DiscoverPeers { + subnet_id: _any_subnet, + min_ttl: _any_instant, + } => discover_peer_count = discover_peer_count + 1, AttServiceMessage::Subscribe(_any_subnet) => subscribe_count = subscribe_count + 1, AttServiceMessage::EnrAdd(_any_subnet) => enr_add_count = enr_add_count + 1, _ => unexpected_msg_count = unexpected_msg_count + 1, diff --git a/beacon_node/network/src/router/mod.rs b/beacon_node/network/src/router/mod.rs index 78e1b8b2a..025914d6b 100644 --- a/beacon_node/network/src/router/mod.rs +++ b/beacon_node/network/src/router/mod.rs @@ -10,8 +10,8 @@ use crate::error; use crate::service::NetworkMessage; use beacon_chain::{BeaconChain, BeaconChainTypes, BlockError}; use eth2_libp2p::{ - rpc::{RPCCodedResponse, RPCRequest, RPCResponse, RequestId, ResponseTermination}, - MessageId, NetworkGlobals, PeerId, PubsubMessage, RPCEvent, + rpc::{RPCError, RequestId, SubstreamId}, + MessageId, NetworkGlobals, PeerId, PubsubMessage, Request, Response, }; use futures::prelude::*; use processor::Processor; @@ -43,8 +43,24 @@ pub enum RouterMessage { PeerDialed(PeerId), /// Peer has disconnected, PeerDisconnected(PeerId), - /// An RPC response/request has been received. - RPC(PeerId, RPCEvent), + /// An RPC request has been received. + RPCRequestReceived { + peer_id: PeerId, + stream_id: SubstreamId, + request: Request, + }, + /// An RPC response has been received. + RPCResponseReceived { + peer_id: PeerId, + request_id: RequestId, + response: Response, + }, + /// An RPC request failed + RPCFailed { + peer_id: PeerId, + request_id: RequestId, + error: RPCError, + }, /// A gossip message has been received. The fields are: message id, the peer that sent us this /// message and the message itself. PubsubMessage(MessageId, PeerId, PubsubMessage), @@ -58,7 +74,7 @@ impl Router { beacon_chain: Arc>, network_globals: Arc>, network_send: mpsc::UnboundedSender>, - runtime_handle: &tokio::runtime::Handle, + executor: environment::TaskExecutor, log: slog::Logger, ) -> error::Result>> { let message_handler_log = log.new(o!("service"=> "router")); @@ -68,7 +84,7 @@ impl Router { // Initialise a message instance, which itself spawns the syncing thread. let processor = Processor::new( - runtime_handle, + executor.clone(), beacon_chain, network_globals.clone(), network_send.clone(), @@ -84,12 +100,15 @@ impl Router { }; // spawn handler task and move the message handler instance into the spawned thread - runtime_handle.spawn(async move { - handler_recv - .for_each(move |msg| future::ready(handler.handle_message(msg))) - .await; - debug!(log, "Network message handler terminated."); - }); + executor.spawn( + async move { + debug!(log, "Network message router started"); + handler_recv + .for_each(move |msg| future::ready(handler.handle_message(msg))) + .await; + }, + "router", + ); Ok(handler_send) } @@ -106,11 +125,32 @@ impl Router { RouterMessage::PeerDisconnected(peer_id) => { self.processor.on_disconnect(peer_id); } - // An RPC message request/response has been received - RouterMessage::RPC(peer_id, rpc_event) => { - self.handle_rpc_message(peer_id, rpc_event); + RouterMessage::RPCRequestReceived { + peer_id, + stream_id, + request, + } => { + self.handle_rpc_request(peer_id, stream_id, request); + } + RouterMessage::RPCResponseReceived { + peer_id, + request_id, + response, + } => { + self.handle_rpc_response(peer_id, request_id, response); + } + RouterMessage::RPCFailed { + peer_id, + request_id, + error, + } => { + warn!(self.log, "RPC Error"; + "peer_id" => peer_id.to_string(), + "request_id" => request_id, + "error" => error.to_string(), + "client" => self.network_globals.client(&peer_id).to_string()); + self.processor.on_rpc_error(peer_id, request_id); } - // An RPC message request/response has been received RouterMessage::PubsubMessage(id, peer_id, gossip) => { self.handle_gossip(id, peer_id, gossip); } @@ -119,32 +159,14 @@ impl Router { /* RPC - Related functionality */ - /// Handle RPC messages - fn handle_rpc_message(&mut self, peer_id: PeerId, rpc_message: RPCEvent) { - match rpc_message { - RPCEvent::Request(id, req) => self.handle_rpc_request(peer_id, id, req), - RPCEvent::Response(id, resp) => self.handle_rpc_response(peer_id, id, resp), - RPCEvent::Error(id, _protocol, error) => { - warn!(self.log, "RPC Error"; "peer_id" => peer_id.to_string(), "request_id" => id, "error" => error.to_string(), - "client" => self.network_globals.client(&peer_id).to_string()); - self.processor.on_rpc_error(peer_id, id); - } - } - } - /// A new RPC request has been received from the network. - fn handle_rpc_request( - &mut self, - peer_id: PeerId, - request_id: RequestId, - request: RPCRequest, - ) { + fn handle_rpc_request(&mut self, peer_id: PeerId, stream_id: SubstreamId, request: Request) { match request { - RPCRequest::Status(status_message) => { + Request::Status(status_message) => { self.processor - .on_status_request(peer_id, request_id, status_message) + .on_status_request(peer_id, stream_id, status_message) } - RPCRequest::Goodbye(goodbye_reason) => { + Request::Goodbye(goodbye_reason) => { debug!( self.log, "Peer sent Goodbye"; "peer_id" => peer_id.to_string(), @@ -153,14 +175,12 @@ impl Router { ); self.processor.on_disconnect(peer_id); } - RPCRequest::BlocksByRange(request) => self + Request::BlocksByRange(request) => self .processor - .on_blocks_by_range_request(peer_id, request_id, request), - RPCRequest::BlocksByRoot(request) => self + .on_blocks_by_range_request(peer_id, stream_id, request), + Request::BlocksByRoot(request) => self .processor - .on_blocks_by_root_request(peer_id, request_id, request), - RPCRequest::Ping(_) => unreachable!("Ping MUST be handled in the behaviour"), - RPCRequest::MetaData(_) => unreachable!("MetaData MUST be handled in the behaviour"), + .on_blocks_by_root_request(peer_id, stream_id, request), } } @@ -170,71 +190,20 @@ impl Router { &mut self, peer_id: PeerId, request_id: RequestId, - error_response: RPCCodedResponse, + response: Response, ) { // an error could have occurred. - match error_response { - RPCCodedResponse::InvalidRequest(error) => { - warn!(self.log, "RPC Invalid Request"; - "peer_id" => peer_id.to_string(), - "request_id" => request_id, - "error" => error.to_string(), - "client" => self.network_globals.client(&peer_id).to_string()); - self.processor.on_rpc_error(peer_id, request_id); + match response { + Response::Status(status_message) => { + self.processor.on_status_response(peer_id, status_message); } - RPCCodedResponse::ServerError(error) => { - warn!(self.log, "RPC Server Error" ; - "peer_id" => peer_id.to_string(), - "request_id" => request_id, - "error" => error.to_string(), - "client" => self.network_globals.client(&peer_id).to_string()); - self.processor.on_rpc_error(peer_id, request_id); + Response::BlocksByRange(beacon_block) => { + self.processor + .on_blocks_by_range_response(peer_id, request_id, beacon_block); } - RPCCodedResponse::Unknown(error) => { - warn!(self.log, "RPC Unknown Error"; - "peer_id" => peer_id.to_string(), - "request_id" => request_id, - "error" => error.to_string(), - "client" => self.network_globals.client(&peer_id).to_string()); - self.processor.on_rpc_error(peer_id, request_id); - } - RPCCodedResponse::Success(response) => match response { - RPCResponse::Status(status_message) => { - self.processor.on_status_response(peer_id, status_message); - } - RPCResponse::BlocksByRange(beacon_block) => { - self.processor.on_blocks_by_range_response( - peer_id, - request_id, - Some(beacon_block), - ); - } - RPCResponse::BlocksByRoot(beacon_block) => { - self.processor.on_blocks_by_root_response( - peer_id, - request_id, - Some(beacon_block), - ); - } - RPCResponse::Pong(_) => { - unreachable!("Ping must be handled in the behaviour"); - } - RPCResponse::MetaData(_) => { - unreachable!("Meta data must be handled in the behaviour"); - } - }, - RPCCodedResponse::StreamTermination(response_type) => { - // have received a stream termination, notify the processing functions - match response_type { - ResponseTermination::BlocksByRange => { - self.processor - .on_blocks_by_range_response(peer_id, request_id, None); - } - ResponseTermination::BlocksByRoot => { - self.processor - .on_blocks_by_root_response(peer_id, request_id, None); - } - } + Response::BlocksByRoot(beacon_block) => { + self.processor + .on_blocks_by_root_response(peer_id, request_id, beacon_block); } } } diff --git a/beacon_node/network/src/router/processor.rs b/beacon_node/network/src/router/processor.rs index 7cab8ad31..1fd6927c5 100644 --- a/beacon_node/network/src/router/processor.rs +++ b/beacon_node/network/src/router/processor.rs @@ -7,14 +7,13 @@ use beacon_chain::{ }, BeaconChain, BeaconChainTypes, BlockError, BlockProcessingOutcome, GossipVerifiedBlock, }; -use eth2_libp2p::rpc::methods::*; -use eth2_libp2p::rpc::{RPCCodedResponse, RPCEvent, RPCRequest, RPCResponse, RequestId}; -use eth2_libp2p::{NetworkGlobals, PeerId}; +use eth2_libp2p::rpc::*; +use eth2_libp2p::{NetworkGlobals, PeerId, Request, Response}; use slog::{debug, error, o, trace, warn}; use ssz::Encode; use std::sync::Arc; use store::Store; -use tokio::sync::{mpsc, oneshot}; +use tokio::sync::mpsc; use types::{ Attestation, ChainSpec, Epoch, EthSpec, Hash256, SignedAggregateAndProof, SignedBeaconBlock, Slot, @@ -33,8 +32,6 @@ pub struct Processor { chain: Arc>, /// A channel to the syncing thread. sync_send: mpsc::UnboundedSender>, - /// A oneshot channel for destroying the sync thread. - _sync_exit: oneshot::Sender<()>, /// A network context to return and handle RPC requests. network: HandlerNetworkContext, /// The `RPCHandler` logger. @@ -44,7 +41,7 @@ pub struct Processor { impl Processor { /// Instantiate a `Processor` instance pub fn new( - runtime_handle: &tokio::runtime::Handle, + executor: environment::TaskExecutor, beacon_chain: Arc>, network_globals: Arc>, network_send: mpsc::UnboundedSender>, @@ -53,8 +50,8 @@ impl Processor { let sync_logger = log.new(o!("service"=> "sync")); // spawn the sync thread - let (sync_send, _sync_exit) = crate::sync::manager::spawn( - runtime_handle, + let sync_send = crate::sync::manager::spawn( + executor, beacon_chain.clone(), network_globals, network_send.clone(), @@ -64,7 +61,6 @@ impl Processor { Processor { chain: beacon_chain, sync_send, - _sync_exit, network: HandlerNetworkContext::new(network_send, log.clone()), log: log.clone(), } @@ -89,7 +85,10 @@ impl Processor { /// An error occurred during an RPC request. The state is maintained by the sync manager, so /// this function notifies the sync manager of the error. pub fn on_rpc_error(&mut self, peer_id: PeerId, request_id: RequestId) { - self.send_to_sync(SyncMessage::RPCError(peer_id, request_id)); + // Check if the failed RPC belongs to sync + if let RequestId::Sync(id) = request_id { + self.send_to_sync(SyncMessage::RPCError(peer_id, id)); + } } /// Sends a `Status` message to the peer. @@ -109,7 +108,7 @@ impl Processor { "head_slot" => format!("{}", status_message.head_slot), ); self.network - .send_rpc_request(peer_id, RPCRequest::Status(status_message)); + .send_processor_request(peer_id, Request::Status(status_message)); } } @@ -119,7 +118,7 @@ impl Processor { pub fn on_status_request( &mut self, peer_id: PeerId, - request_id: RequestId, + request_id: SubstreamId, status: StatusMessage, ) { debug!( @@ -136,10 +135,10 @@ impl Processor { // ignore status responses if we are shutting down if let Some(status_message) = status_message(&self.chain) { // Say status back. - self.network.send_rpc_response( + self.network.send_response( peer_id.clone(), + Response::Status(status_message), request_id, - RPCResponse::Status(status_message), ); } @@ -284,16 +283,16 @@ impl Processor { pub fn on_blocks_by_root_request( &mut self, peer_id: PeerId, - request_id: RequestId, + request_id: SubstreamId, request: BlocksByRootRequest, ) { let mut send_block_count = 0; for root in request.block_roots.iter() { if let Ok(Some(block)) = self.chain.store.get_block(root) { - self.network.send_rpc_response( + self.network.send_response( peer_id.clone(), + Response::BlocksByRoot(Some(Box::new(block))), request_id, - RPCResponse::BlocksByRoot(Box::new(block)), ); send_block_count += 1; } else { @@ -314,18 +313,15 @@ impl Processor { ); // send stream termination - self.network.send_rpc_error_response( - peer_id, - request_id, - RPCCodedResponse::StreamTermination(ResponseTermination::BlocksByRoot), - ); + self.network + .send_response(peer_id, Response::BlocksByRoot(None), request_id); } /// Handle a `BlocksByRange` request from the peer. pub fn on_blocks_by_range_request( &mut self, peer_id: PeerId, - request_id: RequestId, + request_id: SubstreamId, req: BlocksByRangeRequest, ) { debug!( @@ -391,10 +387,10 @@ impl Processor { && block.slot() < req.start_slot + req.count * req.step { blocks_sent += 1; - self.network.send_rpc_response( + self.network.send_response( peer_id.clone(), + Response::BlocksByRange(Some(Box::new(block))), request_id, - RPCResponse::BlocksByRange(Box::new(block)), ); } } else { @@ -428,11 +424,8 @@ impl Processor { } // send the stream terminator - self.network.send_rpc_error_response( - peer_id, - request_id, - RPCCodedResponse::StreamTermination(ResponseTermination::BlocksByRange), - ); + self.network + .send_response(peer_id, Response::BlocksByRange(None), request_id); } /// Handle a `BlocksByRange` response from the peer. @@ -449,11 +442,18 @@ impl Processor { "peer" => format!("{:?}", peer_id), ); - self.send_to_sync(SyncMessage::BlocksByRangeResponse { - peer_id, - request_id, - beacon_block, - }); + if let RequestId::Sync(id) = request_id { + self.send_to_sync(SyncMessage::BlocksByRangeResponse { + peer_id, + request_id: id, + beacon_block, + }); + } else { + debug!( + self.log, + "All blocks by range responses should belong to sync" + ); + } } /// Handle a `BlocksByRoot` response from the peer. @@ -469,11 +469,18 @@ impl Processor { "peer" => format!("{:?}", peer_id), ); - self.send_to_sync(SyncMessage::BlocksByRootResponse { - peer_id, - request_id, - beacon_block, - }); + if let RequestId::Sync(id) = request_id { + self.send_to_sync(SyncMessage::BlocksByRootResponse { + peer_id, + request_id: id, + beacon_block, + }); + } else { + debug!( + self.log, + "All Blocks by Root responses should belong to sync" + ) + } } /// Template function to be called on a block to determine if the block should be propagated @@ -905,8 +912,6 @@ pub(crate) fn status_message( /// Wraps a Network Channel to employ various RPC related network functionality for the /// processor. -/// The Processor doesn't manage it's own request Id's and can therefore only send -/// responses or requests with 0 request Ids. pub struct HandlerNetworkContext { /// The network channel to relay messages to the Network service. network_send: mpsc::UnboundedSender>, @@ -919,6 +924,12 @@ impl HandlerNetworkContext { Self { network_send, log } } + fn inform_network(&mut self, msg: NetworkMessage) { + self.network_send + .send(msg) + .unwrap_or_else(|_| warn!(self.log, "Could not send message to the network service")) + } + pub fn disconnect(&mut self, peer_id: PeerId, reason: GoodbyeReason) { warn!( &self.log, @@ -926,55 +937,42 @@ impl HandlerNetworkContext { "reason" => format!("{:?}", reason), "peer_id" => format!("{:?}", peer_id), ); - self.send_rpc_request(peer_id.clone(), RPCRequest::Goodbye(reason)); - self.network_send - .send(NetworkMessage::Disconnect { peer_id }) - .unwrap_or_else(|_| { - warn!( - self.log, - "Could not send a Disconnect to the network service" - ) - }); + self.send_processor_request(peer_id.clone(), Request::Goodbye(reason)); + self.inform_network(NetworkMessage::Disconnect { peer_id }); } - pub fn send_rpc_request(&mut self, peer_id: PeerId, rpc_request: RPCRequest) { - // the message handler cannot send requests with ids. Id's are managed by the sync - // manager. - let request_id = 0; - self.send_rpc_event(peer_id, RPCEvent::Request(request_id, rpc_request)); - } - - /// Convenience function to wrap successful RPC Responses. - pub fn send_rpc_response( - &mut self, - peer_id: PeerId, - request_id: RequestId, - rpc_response: RPCResponse, - ) { - self.send_rpc_event( + pub fn send_processor_request(&mut self, peer_id: PeerId, request: Request) { + self.inform_network(NetworkMessage::SendRequest { peer_id, - RPCEvent::Response(request_id, RPCCodedResponse::Success(rpc_response)), - ); + request_id: RequestId::Router, + request, + }) } - /// Send an RPCCodedResponse. This handles errors and stream terminations. - pub fn send_rpc_error_response( + pub fn send_response( &mut self, peer_id: PeerId, - request_id: RequestId, - rpc_error_response: RPCCodedResponse, + response: Response, + stream_id: SubstreamId, ) { - self.send_rpc_event(peer_id, RPCEvent::Response(request_id, rpc_error_response)); + self.inform_network(NetworkMessage::SendResponse { + peer_id, + stream_id, + response, + }) } - - fn send_rpc_event(&mut self, peer_id: PeerId, rpc_event: RPCEvent) { - self.network_send - .send(NetworkMessage::RPC(peer_id, rpc_event)) - .unwrap_or_else(|_| { - warn!( - self.log, - "Could not send RPC message to the network service" - ) - }); + pub fn _send_error_response( + &mut self, + peer_id: PeerId, + substream_id: SubstreamId, + error: RPCResponseErrorCode, + reason: String, + ) { + self.inform_network(NetworkMessage::SendError { + peer_id, + error, + substream_id, + reason, + }) } } diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index ff9063ea5..efabaa23b 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -7,15 +7,17 @@ use crate::{ use crate::{error, metrics}; use beacon_chain::{BeaconChain, BeaconChainTypes}; use eth2_libp2p::Service as LibP2PService; -use eth2_libp2p::{rpc::RPCRequest, BehaviourEvent, Enr, MessageId, NetworkGlobals, PeerId}; -use eth2_libp2p::{Libp2pEvent, PubsubMessage, RPCEvent}; +use eth2_libp2p::{ + rpc::{RPCResponseErrorCode, RequestId, SubstreamId}, + Libp2pEvent, PubsubMessage, Request, Response, +}; +use eth2_libp2p::{BehaviourEvent, Enr, MessageId, NetworkGlobals, PeerId}; use futures::prelude::*; use rest_types::ValidatorSubscription; use slog::{debug, error, info, o, trace}; use std::sync::Arc; use std::time::Duration; -use tokio::runtime::Handle; -use tokio::sync::{mpsc, oneshot}; +use tokio::sync::mpsc; use tokio::time::Delay; use types::EthSpec; @@ -53,13 +55,12 @@ impl NetworkService { pub fn start( beacon_chain: Arc>, config: &NetworkConfig, - runtime_handle: &Handle, - network_log: slog::Logger, + executor: environment::TaskExecutor, ) -> error::Result<( Arc>, mpsc::UnboundedSender>, - oneshot::Sender<()>, )> { + let network_log = executor.log().clone(); // build the network channel let (network_send, network_recv) = mpsc::unbounded_channel::>(); // get a reference to the beacon chain store @@ -75,7 +76,7 @@ impl NetworkService { // launch libp2p service let (network_globals, mut libp2p) = - runtime_handle.enter(|| LibP2PService::new(config, enr_fork_id, &network_log))?; + LibP2PService::new(executor.clone(), config, enr_fork_id, &network_log)?; for enr in load_dht::(store.clone()) { libp2p.swarm.add_enr(enr); @@ -88,7 +89,7 @@ impl NetworkService { beacon_chain.clone(), network_globals.clone(), network_send.clone(), - runtime_handle, + executor.clone(), network_log.clone(), )?; @@ -111,19 +112,23 @@ impl NetworkService { propagation_percentage, }; - let network_exit = runtime_handle.enter(|| spawn_service(network_service))?; + spawn_service(executor, network_service)?; - Ok((network_globals, network_send, network_exit)) + Ok((network_globals, network_send)) } } fn spawn_service( + executor: environment::TaskExecutor, mut service: NetworkService, -) -> error::Result> { - let (network_exit, mut exit_rx) = tokio::sync::oneshot::channel(); +) -> error::Result<()> { + let mut exit_rx = executor.exit(); // spawn on the current executor - tokio::spawn(async move { + executor.spawn_without_exit(async move { + // TODO: there is something with this code that prevents cargo fmt from doing anything at + // all. Ok, it is worse, the compiler doesn't show errors over this code beyond ast + // checking loop { // build the futures to check simultaneously tokio::select! { @@ -151,13 +156,18 @@ fn spawn_service( info!(service.log, "Network service shutdown"); return; - } - // handle a message sent to the network - Some(message) = service.network_recv.recv() => { - match message { - NetworkMessage::RPC(peer_id, rpc_event) => { - trace!(service.log, "Sending RPC"; "rpc" => format!("{}", rpc_event)); - service.libp2p.swarm.send_rpc(peer_id, rpc_event); + } + // handle a message sent to the network + Some(message) = service.network_recv.recv() => { + match message { + NetworkMessage::SendRequest{ peer_id, request, request_id } => { + service.libp2p.send_request(peer_id, request_id, request); + } + NetworkMessage::SendResponse{ peer_id, response, stream_id } => { + service.libp2p.send_response(peer_id, stream_id, response); + } + NetworkMessage::SendError{ peer_id, error, substream_id, reason } => { + service.libp2p.respond_with_error(peer_id, substream_id, error, reason); } NetworkMessage::Propagate { propagation_source, @@ -178,8 +188,8 @@ fn spawn_service( info!(service.log, "Random filter did not propagate message"); } else { trace!(service.log, "Propagating gossipsub message"; - "propagation_peer" => format!("{:?}", propagation_source), - "message_id" => message_id.to_string(), + "propagation_peer" => format!("{:?}", propagation_source), + "message_id" => message_id.to_string(), ); service .libp2p @@ -230,124 +240,143 @@ fn spawn_service( .attestation_service .validator_subscriptions(subscriptions); } - } - } - // process any attestation service events - Some(attestation_service_message) = service.attestation_service.next() => { - match attestation_service_message { - // TODO: Implement - AttServiceMessage::Subscribe(subnet_id) => { - service.libp2p.swarm.subscribe_to_subnet(subnet_id); - } - AttServiceMessage::Unsubscribe(subnet_id) => { - service.libp2p.swarm.subscribe_to_subnet(subnet_id); - } - AttServiceMessage::EnrAdd(subnet_id) => { - service.libp2p.swarm.update_enr_subnet(subnet_id, true); - } - AttServiceMessage::EnrRemove(subnet_id) => { - service.libp2p.swarm.update_enr_subnet(subnet_id, false); - } - AttServiceMessage::DiscoverPeers(subnet_id) => { - service.libp2p.swarm.peers_request(subnet_id); } } - } - libp2p_event = service.libp2p.next_event() => { - // poll the swarm - match libp2p_event { - Libp2pEvent::Behaviour(event) => match event { - BehaviourEvent::RPC(peer_id, rpc_event) => { - // if we received a Goodbye message, drop and ban the peer - if let RPCEvent::Request(_, RPCRequest::Goodbye(_)) = rpc_event { - //peers_to_ban.push(peer_id.clone()); - service.libp2p.disconnect_and_ban_peer( - peer_id.clone(), - std::time::Duration::from_secs(BAN_PEER_TIMEOUT), - ); - }; - let _ = service - .router_send - .send(RouterMessage::RPC(peer_id, rpc_event)) - .map_err(|_| { - debug!(service.log, "Failed to send RPC to router"); - }); + // process any attestation service events + Some(attestation_service_message) = service.attestation_service.next() => { + match attestation_service_message { + // TODO: Implement + AttServiceMessage::Subscribe(subnet_id) => { + service.libp2p.swarm.subscribe_to_subnet(subnet_id); } - BehaviourEvent::StatusPeer(peer_id) => { - let _ = service - .router_send - .send(RouterMessage::StatusPeer(peer_id)) - .map_err(|_| { - debug!(service.log, "Failed to send re-status peer to router"); - }); + AttServiceMessage::Unsubscribe(subnet_id) => { + service.libp2p.swarm.subscribe_to_subnet(subnet_id); } - BehaviourEvent::PubsubMessage { - id, - source, - message, - .. - } => { - // Update prometheus metrics. - expose_receive_metrics(&message); - match message { - // attestation information gets processed in the attestation service - PubsubMessage::Attestation(ref subnet_and_attestation) => { - let subnet = &subnet_and_attestation.0; - let attestation = &subnet_and_attestation.1; - // checks if we have an aggregator for the slot. If so, we process - // the attestation - if service.attestation_service.should_process_attestation( - &id, - &source, - subnet, - attestation, - ) { + AttServiceMessage::EnrAdd(subnet_id) => { + service.libp2p.swarm.update_enr_subnet(subnet_id, true); + } + AttServiceMessage::EnrRemove(subnet_id) => { + service.libp2p.swarm.update_enr_subnet(subnet_id, false); + } + AttServiceMessage::DiscoverPeers{subnet_id, min_ttl} => { + service.libp2p.swarm.discover_subnet_peers(subnet_id, min_ttl); + } + } + } + libp2p_event = service.libp2p.next_event() => { + // poll the swarm + match libp2p_event { + Libp2pEvent::Behaviour(event) => match event { + BehaviourEvent::RequestReceived{peer_id, id, request} => { + if let Request::Goodbye(_) = request { + // if we received a Goodbye message, drop and ban the peer + //peers_to_ban.push(peer_id.clone()); + // TODO: remove this: https://github.com/sigp/lighthouse/issues/1240 + service.libp2p.disconnect_and_ban_peer( + peer_id.clone(), + std::time::Duration::from_secs(BAN_PEER_TIMEOUT), + ); + }; + let _ = service + .router_send + .send(RouterMessage::RPCRequestReceived{peer_id, stream_id:id, request}) + .map_err(|_| { + debug!(service.log, "Failed to send RPC to router"); + }); + } + BehaviourEvent::ResponseReceived{peer_id, id, response} => { + let _ = service + .router_send + .send(RouterMessage::RPCResponseReceived{ peer_id, request_id:id, response }) + .map_err(|_| { + debug!(service.log, "Failed to send RPC to router"); + }); + + } + BehaviourEvent::RPCFailed{id, peer_id, error} => { + let _ = service + .router_send + .send(RouterMessage::RPCFailed{ peer_id, request_id:id, error }) + .map_err(|_| { + debug!(service.log, "Failed to send RPC to router"); + }); + + } + BehaviourEvent::StatusPeer(peer_id) => { + let _ = service + .router_send + .send(RouterMessage::StatusPeer(peer_id)) + .map_err(|_| { + debug!(service.log, "Failed to send re-status peer to router"); + }); + } + BehaviourEvent::PubsubMessage { + id, + source, + message, + .. + } => { + // Update prometheus metrics. + expose_receive_metrics(&message); + match message { + // attestation information gets processed in the attestation service + PubsubMessage::Attestation(ref subnet_and_attestation) => { + let subnet = &subnet_and_attestation.0; + let attestation = &subnet_and_attestation.1; + // checks if we have an aggregator for the slot. If so, we process + // the attestation + if service.attestation_service.should_process_attestation( + &id, + &source, + subnet, + attestation, + ) { + let _ = service + .router_send + .send(RouterMessage::PubsubMessage(id, source, message)) + .map_err(|_| { + debug!(service.log, "Failed to send pubsub message to router"); + }); + } else { + metrics::inc_counter(&metrics::GOSSIP_UNAGGREGATED_ATTESTATIONS_IGNORED) + } + } + _ => { + // all else is sent to the router let _ = service .router_send .send(RouterMessage::PubsubMessage(id, source, message)) .map_err(|_| { debug!(service.log, "Failed to send pubsub message to router"); }); - } else { - metrics::inc_counter(&metrics::GOSSIP_UNAGGREGATED_ATTESTATIONS_IGNORED) } } - _ => { - // all else is sent to the router - let _ = service - .router_send - .send(RouterMessage::PubsubMessage(id, source, message)) - .map_err(|_| { - debug!(service.log, "Failed to send pubsub message to router"); - }); - } + } + BehaviourEvent::PeerSubscribed(_, _) => {}, + } + Libp2pEvent::NewListenAddr(multiaddr) => { + service.network_globals.listen_multiaddrs.write().push(multiaddr); + } + Libp2pEvent::PeerConnected{ peer_id, endpoint,} => { + debug!(service.log, "Peer Connected"; "peer_id" => peer_id.to_string(), "endpoint" => format!("{:?}", endpoint)); + if let eth2_libp2p::ConnectedPoint::Dialer { .. } = endpoint { + let _ = service + .router_send + .send(RouterMessage::PeerDialed(peer_id)) + .map_err(|_| { + debug!(service.log, "Failed to send peer dialed to router"); }); } } - BehaviourEvent::PeerSubscribed(_, _) => {}, + Libp2pEvent::PeerDisconnected{ peer_id, endpoint,} => { + debug!(service.log, "Peer Disconnected"; "peer_id" => peer_id.to_string(), "endpoint" => format!("{:?}", endpoint)); + let _ = service + .router_send + .send(RouterMessage::PeerDisconnected(peer_id)) + .map_err(|_| { + debug!(service.log, "Failed to send peer disconnect to router"); + }); + } } - Libp2pEvent::NewListenAddr(multiaddr) => { - service.network_globals.listen_multiaddrs.write().push(multiaddr); - } - Libp2pEvent::PeerConnected{ peer_id, endpoint,} => { - debug!(service.log, "Peer Connected"; "peer_id" => peer_id.to_string(), "endpoint" => format!("{:?}", endpoint)); - if let eth2_libp2p::ConnectedPoint::Dialer { .. } = endpoint { - let _ = service - .router_send - .send(RouterMessage::PeerDialed(peer_id)) - .map_err(|_| { - debug!(service.log, "Failed to send peer dialed to router"); }); - } - } - Libp2pEvent::PeerDisconnected{ peer_id, endpoint,} => { - debug!(service.log, "Peer Disconnected"; "peer_id" => peer_id.to_string(), "endpoint" => format!("{:?}", endpoint)); - let _ = service - .router_send - .send(RouterMessage::PeerDisconnected(peer_id)) - .map_err(|_| { - debug!(service.log, "Failed to send peer disconnect to router"); - }); - } - } } } @@ -361,9 +390,9 @@ fn spawn_service( } } } - }); + }, "network"); - Ok(network_exit) + Ok(()) } /// Returns a `Delay` that triggers shortly after the next change in the beacon chain fork version. @@ -385,8 +414,27 @@ pub enum NetworkMessage { Subscribe { subscriptions: Vec, }, - /// Send an RPC message to the libp2p service. - RPC(PeerId, RPCEvent), + /// Send an RPC request to the libp2p service. + SendRequest { + peer_id: PeerId, + request: Request, + request_id: RequestId, + }, + /// Send a successful Response to the libp2p service. + SendResponse { + peer_id: PeerId, + response: Response, + stream_id: SubstreamId, + }, + /// Respond to a peer's request with an error. + SendError { + // TODO: note that this is never used, we just say goodbye without nicely clossing the + // stream assigned to the request + peer_id: PeerId, + error: RPCResponseErrorCode, + reason: String, + substream_id: SubstreamId, + }, /// Publish a list of messages to the gossipsub protocol. Publish { messages: Vec> }, /// Propagate a received gossipsub message. diff --git a/beacon_node/network/src/service/tests.rs b/beacon_node/network/src/service/tests.rs index a33bd9eeb..48aa038c8 100644 --- a/beacon_node/network/src/service/tests.rs +++ b/beacon_node/network/src/service/tests.rs @@ -32,7 +32,9 @@ mod tests { let enrs = vec![enr1, enr2]; let runtime = Runtime::new().unwrap(); - let handle = runtime.handle().clone(); + + let (signal, exit) = exit_future::signal(); + let executor = environment::TaskExecutor::new(runtime.handle().clone(), exit, log.clone()); let mut config = NetworkConfig::default(); config.libp2p_port = 21212; @@ -42,8 +44,8 @@ mod tests { // Create a new network service which implicitly gets dropped at the // end of the block. - let _ = - NetworkService::start(beacon_chain.clone(), &config, &handle, log.clone()).unwrap(); + let _ = NetworkService::start(beacon_chain.clone(), &config, executor).unwrap(); + drop(signal); }); runtime.shutdown_timeout(tokio::time::Duration::from_millis(300)); diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index ce8e26b3e..e5c035b7f 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -37,9 +37,10 @@ use super::block_processor::{spawn_block_processor, BatchProcessResult, ProcessI use super::network_context::SyncNetworkContext; use super::peer_sync_info::{PeerSyncInfo, PeerSyncType}; use super::range_sync::{BatchId, ChainId, RangeSync}; +use super::RequestId; use crate::service::NetworkMessage; use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProcessingOutcome}; -use eth2_libp2p::rpc::{methods::*, RequestId}; +use eth2_libp2p::rpc::BlocksByRootRequest; use eth2_libp2p::types::NetworkGlobals; use eth2_libp2p::PeerId; use fnv::FnvHashMap; @@ -48,7 +49,7 @@ use smallvec::SmallVec; use std::boxed::Box; use std::ops::Sub; use std::sync::Arc; -use tokio::sync::{mpsc, oneshot}; +use tokio::sync::mpsc; use types::{EthSpec, Hash256, SignedBeaconBlock, Slot}; /// The number of slots ahead of us that is allowed before requesting a long-range (batch) Sync @@ -181,17 +182,12 @@ impl SingleBlockRequest { /// chain. This allows the chain to be /// dropped during the syncing process which will gracefully end the `SyncManager`. pub fn spawn( - runtime_handle: &tokio::runtime::Handle, + executor: environment::TaskExecutor, beacon_chain: Arc>, network_globals: Arc>, network_send: mpsc::UnboundedSender>, log: slog::Logger, -) -> ( - mpsc::UnboundedSender>, - oneshot::Sender<()>, -) { - // generate the exit channel - let (sync_exit, exit_rx) = tokio::sync::oneshot::channel(); +) -> mpsc::UnboundedSender> { // generate the message channel let (sync_send, sync_recv) = mpsc::unbounded_channel::>(); @@ -215,11 +211,8 @@ pub fn spawn( // spawn the sync manager thread debug!(log, "Sync Manager started"); - runtime_handle.spawn(async move { - futures::future::select(Box::pin(sync_manager.main()), exit_rx).await; - info!(log.clone(), "Sync Manager shutdown"); - }); - (sync_send, sync_exit) + executor.spawn(async move { Box::pin(sync_manager.main()).await }, "sync"); + sync_send } impl SyncManager { diff --git a/beacon_node/network/src/sync/mod.rs b/beacon_node/network/src/sync/mod.rs index 2e68dc6e8..2c0fcabb2 100644 --- a/beacon_node/network/src/sync/mod.rs +++ b/beacon_node/network/src/sync/mod.rs @@ -9,3 +9,6 @@ mod range_sync; pub use manager::SyncMessage; pub use peer_sync_info::PeerSyncInfo; + +/// Type of id of rpc requests sent by sync +pub type RequestId = usize; diff --git a/beacon_node/network/src/sync/network_context.rs b/beacon_node/network/src/sync/network_context.rs index a5813ff96..039c5db6d 100644 --- a/beacon_node/network/src/sync/network_context.rs +++ b/beacon_node/network/src/sync/network_context.rs @@ -4,9 +4,8 @@ use crate::router::processor::status_message; use crate::service::NetworkMessage; use beacon_chain::{BeaconChain, BeaconChainTypes}; -use eth2_libp2p::rpc::methods::*; -use eth2_libp2p::rpc::{RPCEvent, RPCRequest, RequestId}; -use eth2_libp2p::{Client, NetworkGlobals, PeerId}; +use eth2_libp2p::rpc::{BlocksByRangeRequest, BlocksByRootRequest, GoodbyeReason, RequestId}; +use eth2_libp2p::{Client, NetworkGlobals, PeerId, Request}; use slog::{debug, trace, warn}; use std::sync::Arc; use tokio::sync::mpsc; @@ -22,7 +21,7 @@ pub struct SyncNetworkContext { network_globals: Arc>, /// A sequential ID for all RPC requests. - request_id: RequestId, + request_id: usize, /// Logger for the `SyncNetworkContext`. log: slog::Logger, } @@ -68,7 +67,7 @@ impl SyncNetworkContext { "head_slot" => format!("{}", status_message.head_slot), ); - let _ = self.send_rpc_request(peer_id, RPCRequest::Status(status_message)); + let _ = self.send_rpc_request(peer_id, Request::Status(status_message)); } } @@ -76,7 +75,7 @@ impl SyncNetworkContext { &mut self, peer_id: PeerId, request: BlocksByRangeRequest, - ) -> Result { + ) -> Result { trace!( self.log, "Sending BlocksByRange Request"; @@ -84,14 +83,14 @@ impl SyncNetworkContext { "count" => request.count, "peer" => format!("{:?}", peer_id) ); - self.send_rpc_request(peer_id, RPCRequest::BlocksByRange(request)) + self.send_rpc_request(peer_id, Request::BlocksByRange(request)) } pub fn blocks_by_root_request( &mut self, peer_id: PeerId, request: BlocksByRootRequest, - ) -> Result { + ) -> Result { trace!( self.log, "Sending BlocksByRoot Request"; @@ -99,7 +98,7 @@ impl SyncNetworkContext { "count" => request.block_roots.len(), "peer" => format!("{:?}", peer_id) ); - self.send_rpc_request(peer_id, RPCRequest::BlocksByRoot(request)) + self.send_rpc_request(peer_id, Request::BlocksByRoot(request)) } pub fn downvote_peer(&mut self, peer_id: PeerId) { @@ -109,6 +108,10 @@ impl SyncNetworkContext { "peer" => format!("{:?}", peer_id) ); // TODO: Implement reputation + // TODO: what if we first close the channel sending a response + // RPCResponseErrorCode::InvalidRequest (or something) + // and then disconnect the peer? either request dc or let the behaviour have that logic + // itself self.disconnect(peer_id, GoodbyeReason::Fault); } @@ -121,7 +124,7 @@ impl SyncNetworkContext { ); // ignore the error if the channel send fails - let _ = self.send_rpc_request(peer_id.clone(), RPCRequest::Goodbye(reason)); + let _ = self.send_rpc_request(peer_id.clone(), Request::Goodbye(reason)); self.network_send .send(NetworkMessage::Disconnect { peer_id }) .unwrap_or_else(|_| { @@ -135,27 +138,22 @@ impl SyncNetworkContext { pub fn send_rpc_request( &mut self, peer_id: PeerId, - rpc_request: RPCRequest, - ) -> Result { + request: Request, + ) -> Result { let request_id = self.request_id; self.request_id += 1; - self.send_rpc_event(peer_id, RPCEvent::Request(request_id, rpc_request))?; + self.send_network_msg(NetworkMessage::SendRequest { + peer_id, + request_id: RequestId::Sync(request_id), + request, + })?; Ok(request_id) } - fn send_rpc_event( - &mut self, - peer_id: PeerId, - rpc_event: RPCEvent, - ) -> Result<(), &'static str> { - self.network_send - .send(NetworkMessage::RPC(peer_id, rpc_event)) - .map_err(|_| { - debug!( - self.log, - "Could not send RPC message to the network service" - ); - "Network channel send Failed" - }) + fn send_network_msg(&mut self, msg: NetworkMessage) -> Result<(), &'static str> { + self.network_send.send(msg).map_err(|_| { + debug!(self.log, "Could not send message to the network service"); + "Network channel send Failed" + }) } } diff --git a/beacon_node/network/src/sync/peer_sync_info.rs b/beacon_node/network/src/sync/peer_sync_info.rs index f03d1a1df..78d91282a 100644 --- a/beacon_node/network/src/sync/peer_sync_info.rs +++ b/beacon_node/network/src/sync/peer_sync_info.rs @@ -1,7 +1,7 @@ use super::manager::SLOT_IMPORT_TOLERANCE; use crate::router::processor::status_message; use beacon_chain::{BeaconChain, BeaconChainTypes}; -use eth2_libp2p::rpc::methods::*; +use eth2_libp2p::rpc::*; use eth2_libp2p::SyncInfo; use std::ops::Sub; use std::sync::Arc; diff --git a/beacon_node/network/src/sync/range_sync/batch.rs b/beacon_node/network/src/sync/range_sync/batch.rs index bd8b604e3..684c75126 100644 --- a/beacon_node/network/src/sync/range_sync/batch.rs +++ b/beacon_node/network/src/sync/range_sync/batch.rs @@ -1,6 +1,5 @@ use super::chain::EPOCHS_PER_BATCH; use eth2_libp2p::rpc::methods::*; -use eth2_libp2p::rpc::RequestId; use eth2_libp2p::PeerId; use fnv::FnvHashMap; use ssz::Encode; @@ -112,9 +111,9 @@ impl PartialOrd for Batch { /// This is used to optimise searches for idle peers (peers that have no outbound batch requests). pub struct PendingBatches { /// The current pending batches. - batches: FnvHashMap>, + batches: FnvHashMap>, /// A mapping of peers to the number of pending requests. - peer_requests: HashMap>, + peer_requests: HashMap>, } impl PendingBatches { @@ -125,7 +124,7 @@ impl PendingBatches { } } - pub fn insert(&mut self, request_id: RequestId, batch: Batch) -> Option> { + pub fn insert(&mut self, request_id: usize, batch: Batch) -> Option> { let peer_request = batch.current_peer.clone(); self.peer_requests .entry(peer_request) @@ -134,7 +133,7 @@ impl PendingBatches { self.batches.insert(request_id, batch) } - pub fn remove(&mut self, request_id: RequestId) -> Option> { + pub fn remove(&mut self, request_id: usize) -> Option> { if let Some(batch) = self.batches.remove(&request_id) { if let Entry::Occupied(mut entry) = self.peer_requests.entry(batch.current_peer.clone()) { @@ -157,7 +156,7 @@ impl PendingBatches { /// Adds a block to the batches if the request id exists. Returns None if there is no batch /// matching the request id. - pub fn add_block(&mut self, request_id: RequestId, block: SignedBeaconBlock) -> Option<()> { + pub fn add_block(&mut self, request_id: usize, block: SignedBeaconBlock) -> Option<()> { let batch = self.batches.get_mut(&request_id)?; batch.downloaded_blocks.push(block); Some(()) diff --git a/beacon_node/network/src/sync/range_sync/chain.rs b/beacon_node/network/src/sync/range_sync/chain.rs index 3a98adcac..05abf6ea3 100644 --- a/beacon_node/network/src/sync/range_sync/chain.rs +++ b/beacon_node/network/src/sync/range_sync/chain.rs @@ -1,9 +1,8 @@ use super::batch::{Batch, BatchId, PendingBatches}; use crate::sync::block_processor::{spawn_block_processor, BatchProcessResult, ProcessId}; use crate::sync::network_context::SyncNetworkContext; -use crate::sync::SyncMessage; +use crate::sync::{RequestId, SyncMessage}; use beacon_chain::{BeaconChain, BeaconChainTypes}; -use eth2_libp2p::rpc::RequestId; use eth2_libp2p::PeerId; use rand::prelude::*; use slog::{crit, debug, warn}; diff --git a/beacon_node/network/src/sync/range_sync/range.rs b/beacon_node/network/src/sync/range_sync/range.rs index ca827082e..f6a1d80e4 100644 --- a/beacon_node/network/src/sync/range_sync/range.rs +++ b/beacon_node/network/src/sync/range_sync/range.rs @@ -47,8 +47,8 @@ use crate::sync::block_processor::BatchProcessResult; use crate::sync::manager::SyncMessage; use crate::sync::network_context::SyncNetworkContext; use crate::sync::PeerSyncInfo; +use crate::sync::RequestId; use beacon_chain::{BeaconChain, BeaconChainTypes}; -use eth2_libp2p::rpc::RequestId; use eth2_libp2p::{NetworkGlobals, PeerId}; use slog::{debug, error, trace}; use std::collections::HashSet; diff --git a/beacon_node/operation_pool/src/lib.rs b/beacon_node/operation_pool/src/lib.rs index 79eb3f890..d91aa2cc5 100644 --- a/beacon_node/operation_pool/src/lib.rs +++ b/beacon_node/operation_pool/src/lib.rs @@ -20,6 +20,7 @@ use state_processing::per_block_processing::{ }; use std::collections::{hash_map, HashMap, HashSet}; use std::marker::PhantomData; +use std::ptr; use types::{ typenum::Unsigned, Attestation, AttesterSlashing, BeaconState, BeaconStateError, ChainSpec, EthSpec, Fork, Hash256, ProposerSlashing, RelativeEpoch, SignedVoluntaryExit, Validator, @@ -408,6 +409,9 @@ fn prune_validator_hash_map( /// Compare two operation pools. impl PartialEq for OperationPool { fn eq(&self, other: &Self) -> bool { + if ptr::eq(self, other) { + return true; + } *self.attestations.read() == *other.attestations.read() && *self.attester_slashings.read() == *other.attester_slashings.read() && *self.proposer_slashings.read() == *other.proposer_slashings.read() diff --git a/beacon_node/rest_api/Cargo.toml b/beacon_node/rest_api/Cargo.toml index 382f1c30c..f8cdbfe4f 100644 --- a/beacon_node/rest_api/Cargo.toml +++ b/beacon_node/rest_api/Cargo.toml @@ -25,7 +25,7 @@ state_processing = { path = "../../consensus/state_processing" } types = { path = "../../consensus/types" } http = "0.2.1" hyper = "0.13.5" -tokio = { version = "0.2", features = ["sync"] } +tokio = { version = "0.2.21", features = ["sync"] } url = "2.1.1" lazy_static = "1.4.0" eth2_config = { path = "../../common/eth2_config" } @@ -36,6 +36,9 @@ parking_lot = "0.10.2" futures = "0.3.5" operation_pool = { path = "../operation_pool" } rayon = "1.3.0" +environment = { path = "../../lighthouse/environment" } +uhttp_sse = "0.5.1" +bus = "2.2.3" [dev-dependencies] assert_matches = "1.3.0" diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index 7d41535b3..f0040cfef 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -3,16 +3,22 @@ use crate::response_builder::ResponseBuilder; use crate::validator::get_state_for_epoch; use crate::{ApiError, ApiResult, UrlQuery}; use beacon_chain::{BeaconChain, BeaconChainTypes, StateSkipConfig}; -use hyper::{Body, Request}; +use bus::BusReader; +use futures::executor::block_on; +use hyper::body::Bytes; +use hyper::{Body, Request, Response}; use rest_types::{ BlockResponse, CanonicalHeadResponse, Committee, HeadBeaconBlock, StateResponse, ValidatorRequest, ValidatorResponse, }; +use std::io::Write; use std::sync::Arc; use store::Store; + +use slog::{error, Logger}; use types::{ AttesterSlashing, BeaconState, EthSpec, Hash256, ProposerSlashing, PublicKeyBytes, - RelativeEpoch, Slot, + RelativeEpoch, SignedBeaconBlockHash, Slot, }; /// HTTP handler to return a `BeaconBlock` at a given `root` or `slot`. @@ -122,6 +128,48 @@ pub fn get_block_root( ResponseBuilder::new(&req)?.body(&root) } +fn make_sse_response_chunk(new_head_hash: SignedBeaconBlockHash) -> std::io::Result { + let mut buffer = Vec::new(); + { + let mut sse_message = uhttp_sse::SseMessage::new(&mut buffer); + let untyped_hash: Hash256 = new_head_hash.into(); + write!(sse_message.data()?, "{:?}", untyped_hash)?; + } + let bytes: Bytes = buffer.into(); + Ok(bytes) +} + +pub fn stream_forks( + log: Logger, + mut events: BusReader, +) -> ApiResult { + let (mut sender, body) = Body::channel(); + std::thread::spawn(move || { + while let Ok(new_head_hash) = events.recv() { + let chunk = match make_sse_response_chunk(new_head_hash) { + Ok(chunk) => chunk, + Err(e) => { + error!(log, "Failed to make SSE chunk"; "error" => e.to_string()); + sender.abort(); + break; + } + }; + if let Err(bytes) = block_on(sender.send_data(chunk)) { + error!(log, "Couldn't stream piece {:?}", bytes); + } + } + }); + let response = Response::builder() + .status(200) + .header("Content-Type", "text/event-stream") + .header("Connection", "Keep-Alive") + .header("Cache-Control", "no-cache") + .header("Access-Control-Allow-Origin", "*") + .body(body) + .map_err(|e| ApiError::ServerError(format!("Failed to build response: {:?}", e)))?; + Ok(response) +} + /// HTTP handler to return the `Fork` of the current head. pub fn get_fork( req: Request, diff --git a/beacon_node/rest_api/src/error.rs b/beacon_node/rest_api/src/error.rs index 0897e62fe..b6ce5182d 100644 --- a/beacon_node/rest_api/src/error.rs +++ b/beacon_node/rest_api/src/error.rs @@ -71,6 +71,12 @@ impl From for ApiError { } } +impl From for ApiError { + fn from(e: std::io::Error) -> ApiError { + ApiError::ServerError(format!("IO error: {:?}", e)) + } +} + impl StdError for ApiError { fn cause(&self) -> Option<&dyn StdError> { None diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index c05cb66ab..00043d7b9 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -21,6 +21,7 @@ mod url_query; mod validator; use beacon_chain::{BeaconChain, BeaconChainTypes}; +use bus::Bus; use client_network::NetworkMessage; pub use config::ApiEncodingFormat; use error::{ApiError, ApiResult}; @@ -30,12 +31,13 @@ use futures::future::TryFutureExt; use hyper::server::conn::AddrStream; use hyper::service::{make_service_fn, service_fn}; use hyper::{Body, Request, Server}; +use parking_lot::Mutex; use slog::{info, warn}; use std::net::SocketAddr; -use std::ops::Deref; use std::path::PathBuf; use std::sync::Arc; -use tokio::sync::{mpsc, oneshot}; +use tokio::sync::mpsc; +use types::SignedBeaconBlockHash; use url_query::UrlQuery; pub use crate::helpers::parse_pubkey_bytes; @@ -51,14 +53,16 @@ pub struct NetworkInfo { // Allowing more than 7 arguments. #[allow(clippy::too_many_arguments)] pub fn start_server( + executor: environment::TaskExecutor, config: &Config, beacon_chain: Arc>, network_info: NetworkInfo, db_path: PathBuf, freezer_db_path: PathBuf, eth2_config: Eth2Config, - log: slog::Logger, -) -> Result<(oneshot::Sender<()>, SocketAddr), hyper::Error> { + events: Arc>>, +) -> Result { + let log = executor.log(); let inner_log = log.clone(); let eth2_config = Arc::new(eth2_config); @@ -71,6 +75,7 @@ pub fn start_server( let network_channel = network_info.network_chan.clone(); let db_path = db_path.clone(); let freezer_db_path = freezer_db_path.clone(); + let events = events.clone(); async move { Ok::<_, hyper::Error>(service_fn(move |req: Request| { @@ -83,6 +88,7 @@ pub fn start_server( log.clone(), db_path.clone(), freezer_db_path.clone(), + events.clone(), ) })) } @@ -98,7 +104,7 @@ pub fn start_server( let actual_listen_addr = server.local_addr(); // Build a channel to kill the HTTP server. - let (exit_signal, exit) = oneshot::channel::<()>(); + let exit = executor.exit(); let inner_log = log.clone(); let server_exit = async move { let _ = exit.await; @@ -116,7 +122,8 @@ pub fn start_server( inner_log, "HTTP server failed to start, Unable to bind"; "address" => format!("{:?}", e) ) - }); + }) + .unwrap_or_else(|_| ()); info!( log, @@ -125,18 +132,7 @@ pub fn start_server( "port" => actual_listen_addr.port(), ); - tokio::spawn(server_future); + executor.spawn_without_exit(server_future, "http"); - Ok((exit_signal, actual_listen_addr)) -} - -#[derive(Clone)] -pub struct DBPath(PathBuf); - -impl Deref for DBPath { - type Target = PathBuf; - - fn deref(&self) -> &Self::Target { - &self.0 - } + Ok(actual_listen_addr) } diff --git a/beacon_node/rest_api/src/router.rs b/beacon_node/rest_api/src/router.rs index 722092715..d7e41feab 100644 --- a/beacon_node/rest_api/src/router.rs +++ b/beacon_node/rest_api/src/router.rs @@ -3,14 +3,16 @@ use crate::{ spec, validator, NetworkChannel, }; use beacon_chain::{BeaconChain, BeaconChainTypes}; +use bus::Bus; use eth2_config::Eth2Config; use eth2_libp2p::NetworkGlobals; use hyper::{Body, Error, Method, Request, Response}; +use parking_lot::Mutex; use slog::debug; use std::path::PathBuf; use std::sync::Arc; use std::time::Instant; -use types::Slot; +use types::{SignedBeaconBlockHash, Slot}; // Allowing more than 7 arguments. #[allow(clippy::too_many_arguments)] @@ -23,6 +25,7 @@ pub async fn route( local_log: slog::Logger, db_path: PathBuf, freezer_db_path: PathBuf, + events: Arc>>, ) -> Result, Error> { metrics::inc_counter(&metrics::REQUEST_COUNT); let timer = metrics::start_timer(&metrics::REQUEST_RESPONSE_TIME); @@ -63,6 +66,10 @@ pub async fn route( (&Method::GET, "/beacon/block") => beacon::get_block::(req, beacon_chain), (&Method::GET, "/beacon/block_root") => beacon::get_block_root::(req, beacon_chain), (&Method::GET, "/beacon/fork") => beacon::get_fork::(req, beacon_chain), + (&Method::GET, "/beacon/fork/stream") => { + let reader = events.lock().add_rx(); + beacon::stream_forks::(log, reader) + } (&Method::GET, "/beacon/genesis_time") => beacon::get_genesis_time::(req, beacon_chain), (&Method::GET, "/beacon/genesis_validators_root") => { beacon::get_genesis_validators_root::(req, beacon_chain) diff --git a/beacon_node/src/lib.rs b/beacon_node/src/lib.rs index ca3162a12..67bd28327 100644 --- a/beacon_node/src/lib.rs +++ b/beacon_node/src/lib.rs @@ -10,10 +10,10 @@ pub use client::{Client, ClientBuilder, ClientConfig, ClientGenesis}; pub use config::{get_data_dir, get_eth2_testnet_config, get_testnet_dir}; pub use eth2_config::Eth2Config; +use beacon_chain::events::TeeEventHandler; use beacon_chain::migrate::{BackgroundMigrator, HotColdDB}; use beacon_chain::{ - builder::Witness, eth1_chain::CachingEth1Backend, events::WebSocketSender, - slot_clock::SystemTimeSlotClock, + builder::Witness, eth1_chain::CachingEth1Backend, slot_clock::SystemTimeSlotClock, }; use clap::ArgMatches; use config::get_config; @@ -30,7 +30,7 @@ pub type ProductionClient = Client< SystemTimeSlotClock, CachingEth1Backend>, E, - WebSocketSender, + TeeEventHandler, >, >; @@ -58,7 +58,7 @@ impl ProductionBeaconNode { &matches, &context.eth2_config.spec_constants, &context.eth2_config().spec, - context.log.clone(), + context.log().clone(), )?; Self::new(context, client_config).await } @@ -75,7 +75,7 @@ impl ProductionBeaconNode { let client_config_1 = client_config.clone(); let client_genesis = client_config.genesis.clone(); let store_config = client_config.store.clone(); - let log = context.log.clone(); + let log = context.log().clone(); let db_path = client_config.create_db_path()?; let freezer_db_path_res = client_config.create_freezer_db_path(); @@ -113,15 +113,17 @@ impl ProductionBeaconNode { builder.no_eth1_backend()? }; - let builder = builder + let (builder, events) = builder .system_time_slot_clock()? - .websocket_event_handler(client_config.websocket_server.clone())? + .tee_event_handler(client_config.websocket_server.clone())?; + + let builder = builder .build_beacon_chain()? .network(&mut client_config.network)? .notifier()?; let builder = if client_config.rest_api.enabled { - builder.http_server(&client_config, &http_eth2_config)? + builder.http_server(&client_config, &http_eth2_config, events)? } else { builder }; diff --git a/beacon_node/store/Cargo.toml b/beacon_node/store/Cargo.toml index d651ec804..3782ab205 100644 --- a/beacon_node/store/Cargo.toml +++ b/beacon_node/store/Cargo.toml @@ -16,7 +16,7 @@ rayon = "1.3.0" [dependencies] db-key = "0.0.5" -leveldb = "0.8.4" +leveldb = "0.8.5" parking_lot = "0.10.2" itertools = "0.9.0" eth2_ssz = "0.1.2" @@ -29,4 +29,4 @@ serde = "1.0.110" serde_derive = "1.0.110" lazy_static = "1.4.0" lighthouse_metrics = { path = "../../common/lighthouse_metrics" } -lru = "0.4.3" +lru = "0.5.1" diff --git a/beacon_node/tests/test.rs b/beacon_node/tests/test.rs index 7a7f5a759..a845acf04 100644 --- a/beacon_node/tests/test.rs +++ b/beacon_node/tests/test.rs @@ -51,4 +51,5 @@ fn http_server_genesis_state() { api_state, db_state, "genesis state from api should match that from the DB" ); + env.fire_signal(); } diff --git a/beacon_node/timer/Cargo.toml b/beacon_node/timer/Cargo.toml index 2c183db53..4ac22ad2e 100644 --- a/beacon_node/timer/Cargo.toml +++ b/beacon_node/timer/Cargo.toml @@ -8,7 +8,8 @@ edition = "2018" beacon_chain = { path = "../beacon_chain" } types = { path = "../../consensus/types" } slot_clock = { path = "../../common/slot_clock" } -tokio = { version = "0.2.20", features = ["full"] } +tokio = { version = "0.2.21", features = ["full"] } slog = "2.5.2" parking_lot = "0.10.2" futures = "0.3.5" +environment = { path = "../../lighthouse/environment" } diff --git a/beacon_node/timer/src/lib.rs b/beacon_node/timer/src/lib.rs index 26f8bb60e..67aca9c27 100644 --- a/beacon_node/timer/src/lib.rs +++ b/beacon_node/timer/src/lib.rs @@ -3,23 +3,20 @@ //! This service allows task execution on the beacon node for various functionality. use beacon_chain::{BeaconChain, BeaconChainTypes}; -use futures::future; use futures::stream::StreamExt; +use slog::info; use slot_clock::SlotClock; use std::sync::Arc; use std::time::Duration; use tokio::time::{interval_at, Instant}; /// Spawns a timer service which periodically executes tasks for the beacon chain -/// TODO: We might not need a `Handle` to the runtime since this function should be -/// called from the context of a runtime and we can simply spawn using task::spawn. -/// Check for issues without the Handle. -pub fn spawn( +pub fn spawn_timer( + executor: environment::TaskExecutor, beacon_chain: Arc>, milliseconds_per_slot: u64, -) -> Result, &'static str> { - let (exit_signal, exit) = tokio::sync::oneshot::channel(); - +) -> Result<(), &'static str> { + let log = executor.log(); let start_instant = Instant::now() + beacon_chain .slot_clock @@ -27,14 +24,15 @@ pub fn spawn( .ok_or_else(|| "slot_notifier unable to determine time to next slot")?; // Warning: `interval_at` panics if `milliseconds_per_slot` = 0. - let timer_future = interval_at(start_instant, Duration::from_millis(milliseconds_per_slot)) - .for_each(move |_| { + let mut interval = interval_at(start_instant, Duration::from_millis(milliseconds_per_slot)); + let timer_future = async move { + while interval.next().await.is_some() { beacon_chain.per_slot_task(); - future::ready(()) - }); + } + }; - let future = futures::future::select(timer_future, exit); - tokio::spawn(future); + executor.spawn(timer_future, "timer"); + info!(log, "Timer service started"); - Ok(exit_signal) + Ok(()) } diff --git a/beacon_node/websocket_server/Cargo.toml b/beacon_node/websocket_server/Cargo.toml index a470a427d..8e22fa7d4 100644 --- a/beacon_node/websocket_server/Cargo.toml +++ b/beacon_node/websocket_server/Cargo.toml @@ -12,6 +12,7 @@ serde = "1.0.110" serde_derive = "1.0.110" serde_json = "1.0.52" slog = "2.5.2" -tokio = { version = "0.2.20", features = ["full"] } +tokio = { version = "0.2.21", features = ["full"] } types = { path = "../../consensus/types" } ws = "0.9.1" +environment = { path = "../../lighthouse/environment" } diff --git a/beacon_node/websocket_server/src/lib.rs b/beacon_node/websocket_server/src/lib.rs index 7ffff1b89..f9ed3e97e 100644 --- a/beacon_node/websocket_server/src/lib.rs +++ b/beacon_node/websocket_server/src/lib.rs @@ -1,4 +1,4 @@ -use slog::{debug, error, info, warn, Logger}; +use slog::{debug, error, info, warn}; use std::marker::PhantomData; use std::net::SocketAddr; use types::EthSpec; @@ -34,16 +34,10 @@ impl WebSocketSender { } pub fn start_server( + executor: environment::TaskExecutor, config: &Config, - log: &Logger, -) -> Result< - ( - WebSocketSender, - tokio::sync::oneshot::Sender<()>, - SocketAddr, - ), - String, -> { +) -> Result<(WebSocketSender, SocketAddr), String> { + let log = executor.log(); let server_string = format!("{}:{}", config.listen_address, config.port); // Create a server that simply ignores any incoming messages. @@ -67,31 +61,26 @@ pub fn start_server( let broadcaster = server.broadcaster(); // Produce a signal/channel that can gracefully shutdown the websocket server. - let exit_channel = { - let (exit_channel, exit) = tokio::sync::oneshot::channel(); - - let log_inner = log.clone(); - let broadcaster_inner = server.broadcaster(); - let exit_future = async move { - let _ = exit.await; - if let Err(e) = broadcaster_inner.shutdown() { - warn!( - log_inner, - "Websocket server errored on shutdown"; - "error" => format!("{:?}", e) - ); - } else { - info!(log_inner, "Websocket server shutdown"); - } - }; - - // Place a future on the handle that will shutdown the websocket server when the - // application exits. - tokio::spawn(exit_future); - - exit_channel + let exit = executor.exit(); + let log_inner = log.clone(); + let broadcaster_inner = server.broadcaster(); + let exit_future = async move { + let _ = exit.await; + if let Err(e) = broadcaster_inner.shutdown() { + warn!( + log_inner, + "Websocket server errored on shutdown"; + "error" => format!("{:?}", e) + ); + } else { + info!(log_inner, "Websocket server shutdown"); + } }; + // Place a future on the handle that will shutdown the websocket server when the + // application exits. + executor.runtime_handle().spawn(exit_future); + let log_inner = log.clone(); let _ = std::thread::spawn(move || match server.run() { @@ -122,7 +111,6 @@ pub fn start_server( sender: Some(broadcaster), _phantom: PhantomData, }, - exit_channel, actual_listen_addr, )) } diff --git a/common/eth2_testnet_config/Cargo.toml b/common/eth2_testnet_config/Cargo.toml index a531cb754..e90d9413f 100644 --- a/common/eth2_testnet_config/Cargo.toml +++ b/common/eth2_testnet_config/Cargo.toml @@ -16,5 +16,5 @@ tempdir = "0.3.7" serde = "1.0.110" serde_yaml = "0.8.11" types = { path = "../../consensus/types"} -eth2-libp2p = { path = "../../beacon_node/eth2-libp2p"} +enr = { version = "0.1.0", features = ["libsecp256k1", "ed25519"] } eth2_ssz = "0.1.2" diff --git a/common/eth2_testnet_config/src/lib.rs b/common/eth2_testnet_config/src/lib.rs index 79cc75098..ca29245bf 100644 --- a/common/eth2_testnet_config/src/lib.rs +++ b/common/eth2_testnet_config/src/lib.rs @@ -7,7 +7,7 @@ //! //! https://github.com/sigp/lighthouse/pull/605 -use eth2_libp2p::Enr; +use enr::{CombinedKey, Enr}; use ssz::{Decode, Encode}; use std::fs::{create_dir_all, File}; use std::io::{Read, Write}; @@ -39,7 +39,7 @@ pub const HARDCODED_BOOT_ENR: &[u8] = include_bytes!("../witti-v0-11-3/boot_enr. pub struct Eth2TestnetConfig { pub deposit_contract_address: String, pub deposit_contract_deploy_block: u64, - pub boot_enr: Option>, + pub boot_enr: Option>>, pub genesis_state: Option>, pub yaml_config: Option, } @@ -246,7 +246,7 @@ mod tests { } fn do_test( - boot_enr: Option>, + boot_enr: Option>>, genesis_state: Option>, yaml_config: Option, ) { diff --git a/common/hashset_delay/Cargo.toml b/common/hashset_delay/Cargo.toml index 6470479b0..cba08662d 100644 --- a/common/hashset_delay/Cargo.toml +++ b/common/hashset_delay/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" [dependencies] futures = "0.3.5" -tokio = { version = "0.2.20", features = ["time"] } +tokio = { version = "0.2.21", features = ["time"] } [dev-dependencies] -tokio = { version = "0.2.20", features = ["time", "rt-threaded", "macros"] } +tokio = { version = "0.2.21", features = ["time", "rt-threaded", "macros"] } diff --git a/common/lighthouse_metrics/src/lib.rs b/common/lighthouse_metrics/src/lib.rs index 2874ddc50..c6314bdb4 100644 --- a/common/lighthouse_metrics/src/lib.rs +++ b/common/lighthouse_metrics/src/lib.rs @@ -56,7 +56,9 @@ use prometheus::{HistogramOpts, HistogramTimer, Opts}; -pub use prometheus::{Encoder, Gauge, Histogram, IntCounter, IntGauge, Result, TextEncoder}; +pub use prometheus::{ + Encoder, Gauge, Histogram, HistogramVec, IntCounter, IntGauge, IntGaugeVec, Result, TextEncoder, +}; /// Collect all the metrics for reporting. pub fn gather() -> Vec { @@ -99,6 +101,48 @@ pub fn try_create_histogram(name: &str, help: &str) -> Result { Ok(histogram) } +/// Attempts to crate a `HistogramVec`, returning `Err` if the registry does not accept the counter +/// (potentially due to naming conflict). +pub fn try_create_histogram_vec( + name: &str, + help: &str, + label_names: &[&str], +) -> Result { + let opts = HistogramOpts::new(name, help); + let histogram_vec = HistogramVec::new(opts, label_names)?; + prometheus::register(Box::new(histogram_vec.clone()))?; + Ok(histogram_vec) +} + +/// Attempts to crate a `IntGaugeVec`, returning `Err` if the registry does not accept the gauge +/// (potentially due to naming conflict). +pub fn try_create_int_gauge_vec( + name: &str, + help: &str, + label_names: &[&str], +) -> Result { + let opts = Opts::new(name, help); + let counter_vec = IntGaugeVec::new(opts, label_names)?; + prometheus::register(Box::new(counter_vec.clone()))?; + Ok(counter_vec) +} + +pub fn get_int_gauge(int_gauge_vec: &Result, name: &[&str]) -> Option { + if let Ok(int_gauge_vec) = int_gauge_vec { + Some(int_gauge_vec.get_metric_with_label_values(name).ok()?) + } else { + None + } +} + +pub fn get_histogram(histogram_vec: &Result, name: &[&str]) -> Option { + if let Ok(histogram_vec) = histogram_vec { + Some(histogram_vec.get_metric_with_label_values(name).ok()?) + } else { + None + } +} + /// Starts a timer for the given `Histogram`, stopping when it gets dropped or given to `stop_timer(..)`. pub fn start_timer(histogram: &Result) -> Option { if let Ok(histogram) = histogram { @@ -133,6 +177,18 @@ pub fn set_gauge(gauge: &Result, value: i64) { } } +pub fn inc_gauge(gauge: &Result) { + if let Ok(gauge) = gauge { + gauge.inc(); + } +} + +pub fn dec_gauge(gauge: &Result) { + if let Ok(gauge) = gauge { + gauge.dec(); + } +} + pub fn maybe_set_gauge(gauge: &Result, value_opt: Option) { if let Some(value) = value_opt { set_gauge(gauge, value) diff --git a/consensus/proto_array_fork_choice/src/proto_array_fork_choice.rs b/consensus/proto_array_fork_choice/src/proto_array_fork_choice.rs index 8fc390003..2ce28483b 100644 --- a/consensus/proto_array_fork_choice/src/proto_array_fork_choice.rs +++ b/consensus/proto_array_fork_choice/src/proto_array_fork_choice.rs @@ -5,6 +5,7 @@ use parking_lot::{RwLock, RwLockReadGuard}; use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use std::collections::HashMap; +use std::ptr; use types::{Epoch, Hash256, Slot}; pub const DEFAULT_PRUNE_THRESHOLD: usize = 256; @@ -51,6 +52,9 @@ pub struct ProtoArrayForkChoice { impl PartialEq for ProtoArrayForkChoice { fn eq(&self, other: &Self) -> bool { + if ptr::eq(self, other) { + return true; + } *self.proto_array.read() == *other.proto_array.read() && *self.votes.read() == *other.votes.read() && *self.balances.read() == *other.balances.read() diff --git a/lcli/Cargo.toml b/lcli/Cargo.toml index 9bc1bbdd3..afd8d1f38 100644 --- a/lcli/Cargo.toml +++ b/lcli/Cargo.toml @@ -27,7 +27,7 @@ dirs = "2.0.2" genesis = { path = "../beacon_node/genesis" } deposit_contract = { path = "../common/deposit_contract" } tree_hash = "0.1.0" -tokio = { version = "0.2.20", features = ["full"] } +tokio = { version = "0.2.21", features = ["full"] } clap_utils = { path = "../common/clap_utils" } eth2-libp2p = { path = "../beacon_node/eth2-libp2p" } validator_dir = { path = "../common/validator_dir", features = ["insecure_keys"] } diff --git a/lcli/src/eth1_genesis.rs b/lcli/src/eth1_genesis.rs index 1d267e326..2e4fce0e0 100644 --- a/lcli/src/eth1_genesis.rs +++ b/lcli/src/eth1_genesis.rs @@ -46,7 +46,7 @@ pub fn run(mut env: Environment, matches: &ArgMatches<'_>) -> Res config.lowest_cached_block_number = eth2_testnet_config.deposit_contract_deploy_block; config.follow_distance = spec.eth1_follow_distance / 2; - let genesis_service = Eth1GenesisService::new(config, env.core_context().log.clone()); + let genesis_service = Eth1GenesisService::new(config, env.core_context().log().clone()); env.runtime().block_on(async { let _ = genesis_service diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index 296162ce0..f4abcddb6 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -9,7 +9,7 @@ write_ssz_files = ["beacon_node/write_ssz_files"] # Writes debugging .ssz files [dependencies] beacon_node = { "path" = "../beacon_node" } -tokio = "0.2.20" +tokio = "0.2.21" slog = { version = "2.5.2", features = ["max_level_trace"] } sloggers = "1.0.0" types = { "path" = "../consensus/types" } diff --git a/lighthouse/environment/Cargo.toml b/lighthouse/environment/Cargo.toml index 441f0e275..bd9a1b981 100644 --- a/lighthouse/environment/Cargo.toml +++ b/lighthouse/environment/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" [dependencies] clap = "2.33.0" -tokio = "0.2.20" +tokio = "0.2.21" slog = { version = "2.5.2", features = ["max_level_trace"] } sloggers = "1.0.0" types = { "path" = "../../consensus/types" } @@ -20,6 +20,9 @@ ctrlc = { version = "3.1.4", features = ["termination"] } futures = "0.3.5" parking_lot = "0.10.2" slog-json = "2.3.0" +exit-future = "0.2.0" +lazy_static = "1.4.0" +lighthouse_metrics = { path = "../../common/lighthouse_metrics" } [dev-dependencies] beacon_node = { path = "../../beacon_node" } diff --git a/lighthouse/environment/src/executor.rs b/lighthouse/environment/src/executor.rs new file mode 100644 index 000000000..feb95eeb1 --- /dev/null +++ b/lighthouse/environment/src/executor.rs @@ -0,0 +1,128 @@ +use crate::metrics; +use futures::prelude::*; +use slog::{debug, trace}; +use tokio::runtime::Handle; + +/// A wrapper over a runtime handle which can spawn async and blocking tasks. +#[derive(Clone)] +pub struct TaskExecutor { + /// The handle to the runtime on which tasks are spawned + pub(crate) handle: Handle, + /// The receiver exit future which on receiving shuts down the task + pub(crate) exit: exit_future::Exit, + pub(crate) log: slog::Logger, +} + +impl TaskExecutor { + /// Create a new task executor. + /// + /// Note: this function is mainly useful in tests. A `TaskExecutor` should be normally obtained from + /// a [`RuntimeContext`](struct.RuntimeContext.html) + pub fn new(handle: Handle, exit: exit_future::Exit, log: slog::Logger) -> Self { + Self { handle, exit, log } + } + + /// Spawn a future on the tokio runtime wrapped in an `exit_future::Exit`. The task is canceled + /// when the corresponding exit_future `Signal` is fired/dropped. + /// + /// This function generates prometheus metrics on number of tasks and task duration. + pub fn spawn(&self, task: impl Future + Send + 'static, name: &'static str) { + let exit = self.exit.clone(); + let log = self.log.clone(); + + if let Some(int_gauge) = metrics::get_int_gauge(&metrics::ASYNC_TASKS_COUNT, &[name]) { + // Task is shutdown before it completes if `exit` receives + let int_gauge_1 = int_gauge.clone(); + let future = future::select(Box::pin(task), exit).then(move |either| { + match either { + future::Either::Left(_) => trace!(log, "Async task completed"; "task" => name), + future::Either::Right(_) => { + debug!(log, "Async task shutdown, exit received"; "task" => name) + } + } + int_gauge_1.dec(); + futures::future::ready(()) + }); + + int_gauge.inc(); + self.handle.spawn(future); + } + } + + /// Spawn a future on the tokio runtime. This function does not wrap the task in an `exit_future::Exit` + /// like [spawn](#method.spawn). + /// The caller of this function is responsible for wrapping up the task with an `exit_future::Exit` to + /// ensure that the task gets canceled appropriately. + /// This function generates prometheus metrics on number of tasks and task duration. + /// + /// This is useful in cases where the future to be spawned needs to do additional cleanup work when + /// the task is completed/canceled (e.g. writing local variables to disk) or the task is created from + /// some framework which does its own cleanup (e.g. a hyper server). + pub fn spawn_without_exit( + &self, + task: impl Future + Send + 'static, + name: &'static str, + ) { + if let Some(int_gauge) = metrics::get_int_gauge(&metrics::ASYNC_TASKS_COUNT, &[name]) { + let int_gauge_1 = int_gauge.clone(); + let future = task.then(move |_| { + int_gauge_1.dec(); + futures::future::ready(()) + }); + + int_gauge.inc(); + self.handle.spawn(future); + } + } + + /// Spawn a blocking task on a dedicated tokio thread pool wrapped in an exit future. + /// This function generates prometheus metrics on number of tasks and task duration. + pub fn spawn_blocking(&self, task: F, name: &'static str) + where + F: FnOnce() -> () + Send + 'static, + { + let exit = self.exit.clone(); + let log = self.log.clone(); + + if let Some(metric) = metrics::get_histogram(&metrics::BLOCKING_TASKS_HISTOGRAM, &[name]) { + if let Some(int_gauge) = metrics::get_int_gauge(&metrics::BLOCKING_TASKS_COUNT, &[name]) + { + let int_gauge_1 = int_gauge.clone(); + let timer = metric.start_timer(); + let join_handle = self.handle.spawn_blocking(task); + + let future = future::select(join_handle, exit).then(move |either| { + match either { + future::Either::Left(_) => { + trace!(log, "Blocking task completed"; "task" => name) + } + future::Either::Right(_) => { + debug!(log, "Blocking task shutdown, exit received"; "task" => name) + } + } + timer.observe_duration(); + int_gauge_1.dec(); + futures::future::ready(()) + }); + + int_gauge.inc(); + self.handle.spawn(future); + } + } + } + + /// Returns the underlying runtime handle. + pub fn runtime_handle(&self) -> Handle { + self.handle.clone() + } + + /// Returns a copy of the `exit_future::Exit`. + pub fn exit(&self) -> exit_future::Exit { + self.exit.clone() + } + + /// Returns a reference to the logger. + pub fn log(&self) -> &slog::Logger { + &self.log + } +} diff --git a/lighthouse/environment/src/lib.rs b/lighthouse/environment/src/lib.rs index b20e5f3eb..aeec4dbed 100644 --- a/lighthouse/environment/src/lib.rs +++ b/lighthouse/environment/src/lib.rs @@ -10,6 +10,8 @@ use eth2_config::Eth2Config; use eth2_testnet_config::Eth2TestnetConfig; use futures::channel::oneshot; + +pub use executor::TaskExecutor; use slog::{info, o, Drain, Level, Logger}; use sloggers::{null::NullLoggerBuilder, Build}; use std::cell::RefCell; @@ -17,8 +19,10 @@ use std::ffi::OsStr; use std::fs::{rename as FsRename, OpenOptions}; use std::path::PathBuf; use std::time::{SystemTime, UNIX_EPOCH}; -use tokio::runtime::{Builder as RuntimeBuilder, Handle, Runtime}; +use tokio::runtime::{Builder as RuntimeBuilder, Runtime}; use types::{EthSpec, InteropEthSpec, MainnetEthSpec, MinimalEthSpec}; +mod executor; +mod metrics; pub const ETH2_CONFIG_FILENAME: &str = "eth2-spec.toml"; @@ -184,10 +188,13 @@ impl EnvironmentBuilder { /// Consumes the builder, returning an `Environment`. pub fn build(self) -> Result, String> { + let (signal, exit) = exit_future::signal(); Ok(Environment { runtime: self .runtime .ok_or_else(|| "Cannot build environment without runtime".to_string())?, + signal: Some(signal), + exit, log: self .log .ok_or_else(|| "Cannot build environment without log".to_string())?, @@ -204,8 +211,7 @@ impl EnvironmentBuilder { /// `Runtime`, instead it only has access to a `Runtime`. #[derive(Clone)] pub struct RuntimeContext { - pub runtime_handle: Handle, - pub log: Logger, + pub executor: TaskExecutor, pub eth_spec_instance: E, pub eth2_config: Eth2Config, } @@ -216,8 +222,11 @@ impl RuntimeContext { /// The generated service will have the `service_name` in all it's logs. pub fn service_context(&self, service_name: String) -> Self { Self { - runtime_handle: self.runtime_handle.clone(), - log: self.log.new(o!("service" => service_name)), + executor: TaskExecutor { + handle: self.executor.handle.clone(), + exit: self.executor.exit.clone(), + log: self.executor.log.new(o!("service" => service_name)), + }, eth_spec_instance: self.eth_spec_instance.clone(), eth2_config: self.eth2_config.clone(), } @@ -227,12 +236,19 @@ impl RuntimeContext { pub fn eth2_config(&self) -> &Eth2Config { &self.eth2_config } + + /// Returns a reference to the logger for this service. + pub fn log(&self) -> &slog::Logger { + self.executor.log() + } } /// An environment where Lighthouse services can run. Used to start a production beacon node or /// validator client, or to run tests that involve logging and async task execution. pub struct Environment { runtime: Runtime, + signal: Option, + exit: exit_future::Exit, log: Logger, eth_spec_instance: E, pub eth2_config: Eth2Config, @@ -251,8 +267,11 @@ impl Environment { /// Returns a `Context` where no "service" has been added to the logger output. pub fn core_context(&mut self) -> RuntimeContext { RuntimeContext { - runtime_handle: self.runtime.handle().clone(), - log: self.log.clone(), + executor: TaskExecutor { + exit: self.exit.clone(), + handle: self.runtime().handle().clone(), + log: self.log.clone(), + }, eth_spec_instance: self.eth_spec_instance.clone(), eth2_config: self.eth2_config.clone(), } @@ -261,8 +280,11 @@ impl Environment { /// Returns a `Context` where the `service_name` is added to the logger output. pub fn service_context(&mut self, service_name: String) -> RuntimeContext { RuntimeContext { - runtime_handle: self.runtime.handle().clone(), - log: self.log.new(o!("service" => service_name)), + executor: TaskExecutor { + exit: self.exit.clone(), + handle: self.runtime().handle().clone(), + log: self.log.new(o!("service" => service_name.clone())), + }, eth_spec_instance: self.eth_spec_instance.clone(), eth2_config: self.eth2_config.clone(), } @@ -291,6 +313,13 @@ impl Environment { .shutdown_timeout(std::time::Duration::from_secs(2)) } + /// Fire exit signal which shuts down all spawned services + pub fn fire_signal(&mut self) { + if let Some(signal) = self.signal.take() { + let _ = signal.fire(); + } + } + /// Sets the logger (and all child loggers) to log to a file. pub fn log_to_json_file( &mut self, diff --git a/lighthouse/environment/src/metrics.rs b/lighthouse/environment/src/metrics.rs new file mode 100644 index 000000000..54f4b93c8 --- /dev/null +++ b/lighthouse/environment/src/metrics.rs @@ -0,0 +1,21 @@ +/// Handles async task metrics +use lazy_static::lazy_static; +pub use lighthouse_metrics::*; + +lazy_static! { + pub static ref ASYNC_TASKS_COUNT: Result = try_create_int_gauge_vec( + "async_tasks_count", + "Total number of async tasks spawned using spawn", + &["async_task_count"] + ); + pub static ref BLOCKING_TASKS_COUNT: Result = try_create_int_gauge_vec( + "blocking_tasks_count", + "Total number of async tasks spawned using spawn_blocking", + &["blocking_task_count"] + ); + pub static ref BLOCKING_TASKS_HISTOGRAM: Result = try_create_histogram_vec( + "blocking_tasks_histogram", + "Time taken by blocking tasks", + &["blocking_task_hist"] + ); +} diff --git a/lighthouse/src/main.rs b/lighthouse/src/main.rs index e2554fc3a..5d30ae957 100644 --- a/lighthouse/src/main.rs +++ b/lighthouse/src/main.rs @@ -142,7 +142,7 @@ fn run( .optional_eth2_testnet_config(optional_testnet_config)? .build()?; - let log = environment.core_context().log; + let log = environment.core_context().log().clone(); if let Some(log_path) = matches.value_of("logfile") { let path = log_path @@ -217,11 +217,15 @@ fn run( )) .map_err(|e| format!("Failed to init validator client: {}", e))?; - environment.core_context().runtime_handle.enter(|| { - validator - .start_service() - .map_err(|e| format!("Failed to start validator client service: {}", e)) - })?; + environment + .core_context() + .executor + .runtime_handle() + .enter(|| { + validator + .start_service() + .map_err(|e| format!("Failed to start validator client service: {}", e)) + })?; Some(validator) } else { @@ -235,9 +239,9 @@ fn run( // Block this thread until Crtl+C is pressed. environment.block_until_ctrl_c()?; - info!(log, "Shutting down.."); + environment.fire_signal(); drop(beacon_node); drop(validator_client); diff --git a/testing/eth1_test_rig/Cargo.toml b/testing/eth1_test_rig/Cargo.toml index e93c98288..12d13b451 100644 --- a/testing/eth1_test_rig/Cargo.toml +++ b/testing/eth1_test_rig/Cargo.toml @@ -5,8 +5,8 @@ authors = ["Paul Hauner "] edition = "2018" [dependencies] +tokio = { version = "0.2.21", features = ["time"] } web3 = "0.11.0" -tokio = { version = "0.2.20", features = ["time"] } futures = { version = "0.3.5", features = ["compat"] } types = { path = "../../consensus/types"} serde_json = "1.0.52" diff --git a/testing/simulator/Cargo.toml b/testing/simulator/Cargo.toml index 6874f935b..fd2323c01 100644 --- a/testing/simulator/Cargo.toml +++ b/testing/simulator/Cargo.toml @@ -12,7 +12,7 @@ types = { path = "../../consensus/types" } validator_client = { path = "../../validator_client" } parking_lot = "0.10.2" futures = "0.3.5" -tokio = "0.2.20" +tokio = "0.2.21" eth1_test_rig = { path = "../eth1_test_rig" } env_logger = "0.7.1" clap = "2.33.0" diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index d5ba403e4..ea0c99c62 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -9,7 +9,7 @@ name = "validator_client" path = "src/lib.rs" [dev-dependencies] -tokio = {version = "0.2.20", features = ["time", "rt-threaded", "macros"]} +tokio = { version = "0.2.21", features = ["time", "rt-threaded", "macros"] } [dependencies] eth2_ssz = "0.1.2" @@ -27,7 +27,7 @@ serde_json = "1.0.52" slog = { version = "2.5.2", features = ["max_level_trace", "release_max_level_trace"] } slog-async = "2.5.0" slog-term = "2.5.0" -tokio = {version = "0.2.20", features = ["time"]} +tokio = { version = "0.2.21", features = ["time"] } error-chain = "0.12.2" bincode = "1.2.1" futures = { version = "0.3.5", features = ["compat"] } diff --git a/validator_client/src/attestation_service.rs b/validator_client/src/attestation_service.rs index 76ec305b7..b2426192b 100644 --- a/validator_client/src/attestation_service.rs +++ b/validator_client/src/attestation_service.rs @@ -3,8 +3,7 @@ use crate::{ validator_store::ValidatorStore, }; use environment::RuntimeContext; -use exit_future::Signal; -use futures::{FutureExt, StreamExt}; +use futures::StreamExt; use remote_beacon_node::{PublishStatus, RemoteBeaconNode}; use slog::{crit, debug, info, trace}; use slot_clock::SlotClock; @@ -118,8 +117,8 @@ impl Deref for AttestationService { impl AttestationService { /// Starts the service which periodically produces attestations. - pub fn start_update_service(self, spec: &ChainSpec) -> Result { - let log = self.context.log.clone(); + pub fn start_update_service(self, spec: &ChainSpec) -> Result<(), String> { + let log = self.context.log().clone(); let slot_duration = Duration::from_millis(spec.milliseconds_per_slot); let duration_to_next_slot = self @@ -141,13 +140,11 @@ impl AttestationService { ) }; - let (exit_signal, exit_fut) = exit_future::signal(); - - let runtime_handle = self.context.runtime_handle.clone(); + let executor = self.context.executor.clone(); let interval_fut = async move { while interval.next().await.is_some() { - let log = &self.context.log; + let log = self.context.log(); if let Err(e) = self.spawn_attestation_tasks(slot_duration) { crit!( @@ -164,13 +161,8 @@ impl AttestationService { } }; - let future = futures::future::select( - Box::pin(interval_fut), - exit_fut.map(move |_| info!(log, "Shutdown complete")), - ); - runtime_handle.spawn(future); - - Ok(exit_signal) + executor.spawn(interval_fut, "attestation_service"); + Ok(()) } /// For each each required attestation, spawn a new task that downloads, signs and uploads the @@ -214,7 +206,7 @@ impl AttestationService { .into_iter() .for_each(|(committee_index, validator_duties)| { // Spawn a separate task for each attestation. - self.inner.context.runtime_handle.spawn( + self.inner.context.executor.runtime_handle().spawn( self.clone().publish_attestations_and_aggregates( slot, committee_index, @@ -243,7 +235,7 @@ impl AttestationService { validator_duties: Vec, aggregate_production_instant: Instant, ) -> Result<(), ()> { - let log = &self.context.log; + let log = self.context.log(); // There's not need to produce `Attestation` or `SignedAggregateAndProof` if we do not have // any validators for the given `slot` and `committee_index`. @@ -314,7 +306,7 @@ impl AttestationService { committee_index: CommitteeIndex, validator_duties: &[DutyAndProof], ) -> Result>, String> { - let log = &self.context.log; + let log = self.context.log(); if validator_duties.is_empty() { return Ok(None); @@ -448,7 +440,7 @@ impl AttestationService { attestation: Attestation, validator_duties: &[DutyAndProof], ) -> Result<(), String> { - let log = &self.context.log; + let log = self.context.log(); let aggregated_attestation = self .beacon_node @@ -548,6 +540,7 @@ impl AttestationService { #[cfg(test)] mod tests { use super::*; + use futures::future::FutureExt; use parking_lot::RwLock; /// This test is to ensure that a `tokio_timer::Delay` with an instant in the past will still diff --git a/validator_client/src/block_service.rs b/validator_client/src/block_service.rs index b0bc1860a..faecb63dd 100644 --- a/validator_client/src/block_service.rs +++ b/validator_client/src/block_service.rs @@ -1,7 +1,6 @@ use crate::{duties_service::DutiesService, validator_store::ValidatorStore}; use environment::RuntimeContext; -use exit_future::Signal; -use futures::{FutureExt, StreamExt, TryFutureExt}; +use futures::{StreamExt, TryFutureExt}; use remote_beacon_node::{PublishStatus, RemoteBeaconNode}; use slog::{crit, error, info, trace}; use slot_clock::SlotClock; @@ -113,8 +112,8 @@ impl Deref for BlockService { impl BlockService { /// Starts the service that periodically attempts to produce blocks. - pub fn start_update_service(self, spec: &ChainSpec) -> Result { - let log = self.context.log.clone(); + pub fn start_update_service(self, spec: &ChainSpec) -> Result<(), String> { + let log = self.context.log().clone(); let duration_to_next_slot = self .slot_clock @@ -136,7 +135,7 @@ impl BlockService { ) }; - let runtime_handle = self.inner.context.runtime_handle.clone(); + let executor = self.inner.context.executor.clone(); let interval_fut = async move { while interval.next().await.is_some() { @@ -144,20 +143,14 @@ impl BlockService { } }; - let (exit_signal, exit_fut) = exit_future::signal(); + executor.spawn(interval_fut, "block_service"); - let future = futures::future::select( - Box::pin(interval_fut), - exit_fut.map(move |_| info!(log, "Shutdown complete")), - ); - runtime_handle.spawn(future); - - Ok(exit_signal) + Ok(()) } /// Attempt to produce a block for any block producers in the `ValidatorStore`. async fn do_update(&self) -> Result<(), ()> { - let log = &self.context.log; + let log = self.context.log(); let slot = self.slot_clock.now().ok_or_else(move || { crit!(log, "Duties manager failed to read slot clock"); @@ -190,7 +183,7 @@ impl BlockService { iter.for_each(|validator_pubkey| { let service = self.clone(); let log = log.clone(); - self.inner.context.runtime_handle.spawn( + self.inner.context.executor.runtime_handle().spawn( service .publish_block(slot, validator_pubkey) .map_err(move |e| { @@ -208,7 +201,7 @@ impl BlockService { /// Produce a block at the given slot for validator_pubkey async fn publish_block(self, slot: Slot, validator_pubkey: PublicKey) -> Result<(), String> { - let log = &self.context.log; + let log = self.context.log(); let current_slot = self .slot_clock diff --git a/validator_client/src/duties_service.rs b/validator_client/src/duties_service.rs index a1e8c93c0..b7c0c0876 100644 --- a/validator_client/src/duties_service.rs +++ b/validator_client/src/duties_service.rs @@ -1,11 +1,10 @@ use crate::{is_synced::is_synced, validator_store::ValidatorStore}; use environment::RuntimeContext; -use exit_future::Signal; -use futures::{FutureExt, StreamExt}; +use futures::StreamExt; use parking_lot::RwLock; use remote_beacon_node::{PublishStatus, RemoteBeaconNode}; use rest_types::{ValidatorDuty, ValidatorDutyBytes, ValidatorSubscription}; -use slog::{debug, error, info, trace, warn}; +use slog::{debug, error, trace, warn}; use slot_clock::SlotClock; use std::collections::HashMap; use std::convert::TryInto; @@ -439,9 +438,7 @@ impl DutiesService { } /// Start the service that periodically polls the beacon node for validator duties. - pub fn start_update_service(self, spec: &ChainSpec) -> Result { - let log = self.context.log.clone(); - + pub fn start_update_service(self, spec: &ChainSpec) -> Result<(), String> { let duration_to_next_slot = self .slot_clock .duration_to_next_slot() @@ -456,15 +453,14 @@ impl DutiesService { ) }; - let (exit_signal, exit_fut) = exit_future::signal(); - // Run an immediate update before starting the updater service. self.inner .context - .runtime_handle + .executor + .runtime_handle() .spawn(self.clone().do_update()); - let runtime_handle = self.inner.context.runtime_handle.clone(); + let executor = self.inner.context.executor.clone(); let interval_fut = async move { while interval.next().await.is_some() { @@ -472,18 +468,14 @@ impl DutiesService { } }; - let future = futures::future::select( - Box::pin(interval_fut), - exit_fut.map(move |_| info!(log, "Shutdown complete")), - ); - runtime_handle.spawn(future); + executor.spawn(interval_fut, "duties_service"); - Ok(exit_signal) + Ok(()) } /// Attempt to download the duties of all managed validators for this epoch and the next. async fn do_update(self) -> Result<(), ()> { - let log = &self.context.log; + let log = self.context.log(); if !is_synced(&self.beacon_node, &self.slot_clock, None).await && !self.allow_unsynced_beacon_node @@ -550,7 +542,7 @@ impl DutiesService { .await .map_err(move |e| format!("Failed to get duties for epoch {}: {:?}", epoch, e))?; - let log = self.context.log.clone(); + let log = self.context.log().clone(); let mut new_validator = 0; let mut new_epoch = 0; @@ -652,7 +644,7 @@ impl DutiesService { ) } - let log = self.context.log.clone(); + let log = self.context.log().clone(); let count = validator_subscriptions.len(); if count == 0 { diff --git a/validator_client/src/fork_service.rs b/validator_client/src/fork_service.rs index ae979d4fd..1407ed3c5 100644 --- a/validator_client/src/fork_service.rs +++ b/validator_client/src/fork_service.rs @@ -1,9 +1,8 @@ use environment::RuntimeContext; -use exit_future::Signal; -use futures::{FutureExt, StreamExt}; +use futures::StreamExt; use parking_lot::RwLock; use remote_beacon_node::RemoteBeaconNode; -use slog::{debug, info, trace}; +use slog::{debug, trace}; use slot_clock::SlotClock; use std::ops::Deref; use std::sync::Arc; @@ -100,9 +99,7 @@ impl ForkService { } /// Starts the service that periodically polls for the `Fork`. - pub fn start_update_service(self, spec: &ChainSpec) -> Result { - let log = self.context.log.clone(); - + pub fn start_update_service(self, spec: &ChainSpec) -> Result<(), String> { let duration_to_next_epoch = self .slot_clock .duration_to_next_epoch(E::slots_per_epoch()) @@ -117,15 +114,14 @@ impl ForkService { ) }; - let (exit_signal, exit_fut) = exit_future::signal(); - // Run an immediate update before starting the updater service. self.inner .context - .runtime_handle + .executor + .runtime_handle() .spawn(self.clone().do_update()); - let runtime_handle = self.inner.context.runtime_handle.clone(); + let executor = self.inner.context.executor.clone(); let interval_fut = async move { while interval.next().await.is_some() { @@ -133,18 +129,14 @@ impl ForkService { } }; - let future = futures::future::select( - Box::pin(interval_fut), - exit_fut.map(move |_| info!(log, "Shutdown complete")), - ); - runtime_handle.spawn(future); + executor.spawn(interval_fut, "fork_service"); - Ok(exit_signal) + Ok(()) } /// Attempts to download the `Fork` from the server. async fn do_update(self) -> Result<(), ()> { - let log = &self.context.log; + let log = self.context.log(); let fork = self .inner diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 17d30df75..c7afbd7c4 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -17,7 +17,6 @@ use clap::ArgMatches; use config::SLASHING_PROTECTION_FILENAME; use duties_service::{DutiesService, DutiesServiceBuilder}; use environment::RuntimeContext; -use exit_future::Signal; use fork_service::{ForkService, ForkServiceBuilder}; use notifier::spawn_notifier; use remote_beacon_node::RemoteBeaconNode; @@ -41,7 +40,6 @@ pub struct ProductionValidatorClient { fork_service: ForkService, block_service: BlockService, attestation_service: AttestationService, - exit_signals: Vec, config: Config, } @@ -60,10 +58,10 @@ impl ProductionValidatorClient { /// Instantiates the validator client, _without_ starting the timers to trigger block /// and attestation production. pub async fn new(mut context: RuntimeContext, config: Config) -> Result { - let log_1 = context.log.clone(); - let log_2 = context.log.clone(); - let log_3 = context.log.clone(); - let log_4 = context.log.clone(); + let log_1 = context.log().clone(); + let log_2 = context.log().clone(); + let log_3 = context.log().clone(); + let log_4 = context.log().clone(); info!( log_1, @@ -217,46 +215,32 @@ impl ProductionValidatorClient { fork_service, block_service, attestation_service, - exit_signals: vec![], config, }) } pub fn start_service(&mut self) -> Result<(), String> { - let duties_exit = self - .duties_service + self.duties_service .clone() .start_update_service(&self.context.eth2_config.spec) .map_err(|e| format!("Unable to start duties service: {}", e))?; - let fork_exit = self - .fork_service + self.fork_service .clone() .start_update_service(&self.context.eth2_config.spec) .map_err(|e| format!("Unable to start fork service: {}", e))?; - let block_exit = self - .block_service + self.block_service .clone() .start_update_service(&self.context.eth2_config.spec) .map_err(|e| format!("Unable to start block service: {}", e))?; - let attestation_exit = self - .attestation_service + self.attestation_service .clone() .start_update_service(&self.context.eth2_config.spec) .map_err(|e| format!("Unable to start attestation service: {}", e))?; - let notifier_exit = - spawn_notifier(self).map_err(|e| format!("Failed to start notifier: {}", e))?; - - self.exit_signals = vec![ - duties_exit, - fork_exit, - block_exit, - attestation_exit, - notifier_exit, - ]; + spawn_notifier(self).map_err(|e| format!("Failed to start notifier: {}", e))?; Ok(()) } diff --git a/validator_client/src/notifier.rs b/validator_client/src/notifier.rs index 9d9aa9731..d9ee7faec 100644 --- a/validator_client/src/notifier.rs +++ b/validator_client/src/notifier.rs @@ -1,16 +1,14 @@ use crate::{is_synced::is_synced, ProductionValidatorClient}; -use exit_future::Signal; -use futures::{FutureExt, StreamExt}; +use futures::StreamExt; use slog::{error, info}; use slot_clock::SlotClock; use tokio::time::{interval_at, Duration, Instant}; use types::EthSpec; /// Spawns a notifier service which periodically logs information about the node. -pub fn spawn_notifier(client: &ProductionValidatorClient) -> Result { +pub fn spawn_notifier(client: &ProductionValidatorClient) -> Result<(), String> { let context = client.context.service_context("notifier".into()); - let runtime_handle = context.runtime_handle.clone(); - let log = context.log.clone(); + let executor = context.executor.clone(); let duties_service = client.duties_service.clone(); let allow_unsynced_beacon_node = client.config.allow_unsynced_beacon_node; @@ -25,7 +23,7 @@ pub fn spawn_notifier(client: &ProductionValidatorClient) -> Resu let mut interval = interval_at(start_instant, slot_duration); let interval_fut = async move { - let log = &context.log; + let log = context.log(); while interval.next().await.is_some() { if !is_synced( @@ -83,12 +81,6 @@ pub fn spawn_notifier(client: &ProductionValidatorClient) -> Resu } }; - let (exit_signal, exit) = exit_future::signal(); - let future = futures::future::select( - Box::pin(interval_fut), - exit.map(move |_| info!(log, "Shutdown complete")), - ); - runtime_handle.spawn(future); - - Ok(exit_signal) + executor.spawn(interval_fut, "validator_notifier"); + Ok(()) }