From b276af98b75bfd4f2f92e1da022864ee49dcbdc3 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Sat, 25 Mar 2023 03:00:41 +0530 Subject: [PATCH] Rework block processing (#4092) * introduce availability pending block * add intoavailableblock trait * small fixes * add 'gossip blob cache' and start to clean up processing and transition types * shard memory blob cache * Initial commit * Fix after rebase * Add gossip verification conditions * cache cleanup * general chaos * extended chaos * cargo fmt * more progress * more progress * tons of changes, just tryna compile * everything, everywhere, all at once * Reprocess an ExecutedBlock on unavailable blobs * Add sus gossip verification for blobs * Merge stuff * Remove reprocessing cache stuff * lint * Add a wrapper to allow construction of only valid `AvailableBlock`s * rename blob arc list to blob list * merge cleanuo * Revert "merge cleanuo" This reverts commit 5e98326878c77528d0c4668c5a4db4a4b0fbaeaa. * Revert "Revert "merge cleanuo"" This reverts commit 3a4009443a5812b3028abe855079307436dc5419. * fix rpc methods * move beacon block and blob to eth2/types * rename gossip blob cache to data availability checker * lots of changes * fix some compilation issues * fix compilation issues * fix compilation issues * fix compilation issues * fix compilation issues * fix compilation issues * cargo fmt * use a common data structure for block import types * fix availability check on proposal import * refactor the blob cache and split the block wrapper into two types * add type conversion for signed block and block wrapper * fix beacon chain tests and do some renaming, add some comments * Partial processing (#4) * move beacon block and blob to eth2/types * rename gossip blob cache to data availability checker * lots of changes * fix some compilation issues * fix compilation issues * fix compilation issues * fix compilation issues * fix compilation issues * fix compilation issues * cargo fmt * use a common data structure for block import types * fix availability check on proposal import * refactor the blob cache and split the block wrapper into two types * add type conversion for signed block and block wrapper * fix beacon chain tests and do some renaming, add some comments * cargo update (#6) --------- Co-authored-by: realbigsean Co-authored-by: realbigsean --- Cargo.lock | 1032 +++++++++-------- beacon_node/beacon_chain/src/beacon_chain.rs | 402 ++++--- .../beacon_chain/src/blob_verification.rs | 825 ++++++------- .../beacon_chain/src/block_verification.rs | 190 ++- beacon_node/beacon_chain/src/builder.rs | 16 +- .../src/data_availability_checker.rs | 516 +++++++++ .../beacon_chain/src/early_attester_cache.rs | 4 +- beacon_node/beacon_chain/src/lib.rs | 10 +- beacon_node/beacon_chain/src/test_utils.rs | 6 +- .../tests/attestation_production.rs | 27 +- .../beacon_chain/tests/block_verification.rs | 3 +- .../tests/payload_invalidation.rs | 4 + beacon_node/beacon_chain/tests/tests.rs | 27 +- .../src/engine_api/json_structures.rs | 5 +- beacon_node/execution_layer/src/lib.rs | 5 +- beacon_node/http_api/src/block_id.rs | 5 +- .../http_api/src/build_block_contents.rs | 6 +- beacon_node/http_api/src/publish_blocks.rs | 71 +- .../lighthouse_network/src/rpc/protocol.rs | 13 +- beacon_node/network/Cargo.toml | 2 +- .../network/src/beacon_processor/mod.rs | 10 +- .../work_reprocessing_queue.rs | 43 +- .../beacon_processor/worker/gossip_methods.rs | 88 +- .../beacon_processor/worker/rpc_methods.rs | 5 +- .../beacon_processor/worker/sync_methods.rs | 33 +- beacon_node/network/src/router.rs | 1 - .../network/src/sync/block_lookups/mod.rs | 3 +- .../src/sync/block_lookups/parent_lookup.rs | 3 +- .../sync/block_lookups/single_block_lookup.rs | 3 +- beacon_node/network/src/sync/manager.rs | 14 +- beacon_node/store/src/hot_cold_store.rs | 3 +- .../store/src/impls/execution_payload.rs | 4 +- beacon_node/store/src/lib.rs | 1 + common/eth2/src/types.rs | 29 + .../state_processing/src/consensus_context.rs | 2 +- .../src/beacon_block_and_blob_sidecars.rs | 37 - consensus/types/src/beacon_block_body.rs | 4 +- consensus/types/src/blob_sidecar.rs | 11 +- consensus/types/src/blobs_sidecar.rs | 62 - consensus/types/src/lib.rs | 8 - consensus/types/src/signed_beacon_block.rs | 30 - consensus/types/src/signed_blob.rs | 41 +- consensus/types/src/signed_block_and_blobs.rs | 35 - lcli/src/parse_ssz.rs | 2 +- testing/ef_tests/src/type_name.rs | 6 +- testing/ef_tests/tests/tests.rs | 7 - 46 files changed, 2167 insertions(+), 1487 deletions(-) create mode 100644 beacon_node/beacon_chain/src/data_availability_checker.rs delete mode 100644 consensus/types/src/beacon_block_and_blob_sidecars.rs delete mode 100644 consensus/types/src/blobs_sidecar.rs delete mode 100644 consensus/types/src/signed_block_and_blobs.rs diff --git a/Cargo.lock b/Cargo.lock index 2303b4839..37bdb4294 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -88,6 +88,16 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "aead" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c192eb8f11fc081b0fe4259ba5af04217d4e0faddd02417310a927911abd7c8" +dependencies = [ + "crypto-common", + "generic-array", +] + [[package]] name = "aes" version = "0.6.0" @@ -113,17 +123,14 @@ dependencies = [ ] [[package]] -name = "aes-gcm" -version = "0.8.0" +name = "aes" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5278b5fabbb9bd46e24aa69b2fdea62c99088e0a950a9be40e3e0101298f88da" +checksum = "433cfd6710c9986c576a25ca913c39d66a6474107b406f34f91d4a8923395241" dependencies = [ - "aead 0.3.2", - "aes 0.6.0", - "cipher 0.2.5", - "ctr 0.6.0", - "ghash 0.3.1", - "subtle", + "cfg-if", + "cipher 0.4.4", + "cpufeatures", ] [[package]] @@ -140,6 +147,20 @@ dependencies = [ "subtle", ] +[[package]] +name = "aes-gcm" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82e1366e0c69c9f927b1fa5ce2c7bf9eafc8f9268c0b9800729e8b267612447c" +dependencies = [ + "aead 0.5.1", + "aes 0.8.2", + "cipher 0.4.4", + "ctr 0.9.2", + "ghash 0.5.0", + "subtle", +] + [[package]] name = "aes-soft" version = "0.6.4" @@ -205,9 +226,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.69" +version = "1.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" +checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" [[package]] name = "arbitrary" @@ -225,9 +246,9 @@ checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" [[package]] name = "arrayref" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" +checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" [[package]] name = "arrayvec" @@ -248,14 +269,14 @@ dependencies = [ "num-traits", "rusticata-macros", "thiserror", - "time 0.3.17", + "time 0.3.20", ] [[package]] name = "asn1-rs" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf6690c370453db30743b373a60ba498fc0d6d83b11f4abfd87a84a075db5dd4" +checksum = "7f6fd5ddaf0351dff5b8da21b2fb4ff8e08ddd02857f0bf69c47639106c0fff0" dependencies = [ "asn1-rs-derive 0.4.0", "asn1-rs-impl", @@ -264,7 +285,7 @@ dependencies = [ "num-traits", "rusticata-macros", "thiserror", - "time 0.3.17", + "time 0.3.20", ] [[package]] @@ -275,7 +296,7 @@ checksum = "db8b7511298d5b7784b40b092d9e9dcd3a627a5707e4b5e507931ab0d44eeebf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", "synstructure", ] @@ -287,7 +308,7 @@ checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", "synstructure", ] @@ -299,7 +320,7 @@ checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -310,64 +331,64 @@ checksum = "e22d1f4b888c298a027c99dc9048015fac177587de20fc30232a057dfbe24a21" [[package]] name = "async-io" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c374dda1ed3e7d8f0d9ba58715f924862c63eae6849c92d3a18e7fbde9e2794" +checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" dependencies = [ "async-lock", "autocfg 1.1.0", + "cfg-if", "concurrent-queue", "futures-lite", - "libc", "log", "parking", "polling", + "rustix 0.37.3", "slab", "socket2", "waker-fn", - "windows-sys 0.42.0", ] [[package]] name = "async-lock" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8101efe8695a6c17e02911402145357e718ac92d3ff88ae8419e84b1707b685" +checksum = "fa24f727524730b077666307f2734b4a1a1c57acb79193127dcc8914d5242dd7" dependencies = [ "event-listener", - "futures-lite", ] [[package]] name = "async-stream" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" +checksum = "ad445822218ce64be7a341abfb0b1ea43b5c23aa83902542a4542e78309d8e5e" dependencies = [ "async-stream-impl", "futures-core", + "pin-project-lite 0.2.9", ] [[package]] name = "async-stream-impl" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" +checksum = "e4655ae1a7b0cdf149156f780c5bf3f1352bc53cbd9e0a361a7ef7b22947e965" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "async-trait" -version = "0.1.64" +version = "0.1.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" +checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.9", ] [[package]] @@ -431,7 +452,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -537,14 +558,14 @@ checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" [[package]] name = "base64ct" -version = "1.5.3" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b645a089122eccb6111b4f81cbc1a49f5900ac4666bb93ac027feaecf15607bf" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "beacon-api-client" version = "0.1.0" -source = "git+https://github.com/ralexstokes/beacon-api-client#53690a711e33614d59d4d44fb09762b4699e2a4e" +source = "git+https://github.com/ralexstokes/beacon-api-client#30679e9e25d61731cde54e14cd8a3688a39d8e5b" dependencies = [ "ethereum-consensus", "http", @@ -734,9 +755,9 @@ dependencies = [ [[package]] name = "block-buffer" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ "generic-array", ] @@ -980,9 +1001,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.23" +version = "0.4.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" +checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" dependencies = [ "iana-time-zone", "js-sys", @@ -1012,10 +1033,20 @@ dependencies = [ ] [[package]] -name = "clang-sys" -version = "1.4.0" +name = "cipher" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa2e27ae6ab525c3d369ded447057bca5438d86dc3a68f6faafb8269ba82ebf3" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "clang-sys" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ed9a53e5d4d9c573ae844bfac6872b159cb1d1585a83b29e7a64b7eef7332a" dependencies = [ "glob", "libc", @@ -1088,7 +1119,7 @@ dependencies = [ "state_processing", "store", "task_executor", - "time 0.3.17", + "time 0.3.20", "timer", "tokio", "types", @@ -1125,7 +1156,7 @@ name = "compare_fields_derive" version = "0.2.0" dependencies = [ "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1149,9 +1180,9 @@ dependencies = [ [[package]] name = "const-oid" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cec318a675afcb6a1ea1d4340e2d377e56e47c266f28043ceccbf4412ddfdd3b" +checksum = "520fbf3c07483f94e3e3ca9d0cfd913d7718ef2483d2cfd91c0d9e91474ab913" [[package]] name = "convert_case" @@ -1193,12 +1224,6 @@ dependencies = [ "libc", ] -[[package]] -name = "cpuid-bool" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcb25d077389e53838a8158c8e99174c5a9d902dee4904320db714f3c653ffba" - [[package]] name = "crc" version = "3.0.1" @@ -1261,9 +1286,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" dependencies = [ "cfg-if", "crossbeam-utils", @@ -1271,9 +1296,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ "cfg-if", "crossbeam-epoch", @@ -1282,22 +1307,22 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.13" +version = "0.9.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" +checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" dependencies = [ "autocfg 1.1.0", "cfg-if", "crossbeam-utils", - "memoffset 0.7.1", + "memoffset 0.8.0", "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.8.14" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" dependencies = [ "cfg-if", ] @@ -1327,6 +1352,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] @@ -1340,16 +1366,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "crypto-mac" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bff07008ec701e8028e2ceb8f83f0e4274ee62bd2dbdc4fefff2e9a91824081a" -dependencies = [ - "generic-array", - "subtle", -] - [[package]] name = "crypto-mac" version = "0.11.1" @@ -1362,9 +1378,9 @@ dependencies = [ [[package]] name = "csv" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af91f40b7355f82b0a891f50e70399475945bb0b0da4f1700ce60761c9d3e359" +checksum = "0b015497079b9a9d69c02ad25de6c0a6edef051ea6360a327d0bd05802ef64ad" dependencies = [ "csv-core", "itoa", @@ -1381,15 +1397,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "ctr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb4a30d54f7443bf3d6191dcd486aca19e67cb3c49fa7a06a319966346707e7f" -dependencies = [ - "cipher 0.2.5", -] - [[package]] name = "ctr" version = "0.8.0" @@ -1399,6 +1406,15 @@ dependencies = [ "cipher 0.3.0", ] +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher 0.4.4", +] + [[package]] name = "ctrlc" version = "3.2.5" @@ -1424,9 +1440,9 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.0.0-rc.0" +version = "4.0.0-rc.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da00a7a9a4eb92a0a0f8e75660926d48f0d0f3c537e455c457bcdaa1e16b1ac" +checksum = "8d4ba9852b42210c7538b75484f9daa0655e9a3ac04f693747bb0f02cf3cfe16" dependencies = [ "cfg-if", "fiat-crypto", @@ -1438,9 +1454,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.90" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90d59d9acd2a682b4e40605a242f6670eaa58c5957471cbf85e8aa6a0b97a5e8" +checksum = "a9c00419335c41018365ddf7e4d5f1c12ee3659ddcf3e01974650ba1de73d038" dependencies = [ "cc", "cxxbridge-flags", @@ -1450,9 +1466,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.90" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebfa40bda659dd5c864e65f4c9a2b0aff19bea56b017b9b77c73d3766a453a38" +checksum = "fb8307ad413a98fff033c8545ecf133e3257747b3bae935e7602aab8aa92d4ca" dependencies = [ "cc", "codespan-reporting", @@ -1460,24 +1476,24 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn", + "syn 2.0.9", ] [[package]] name = "cxxbridge-flags" -version = "1.0.90" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "457ce6757c5c70dc6ecdbda6925b958aae7f959bda7d8fb9bde889e34a09dc03" +checksum = "edc52e2eb08915cb12596d29d55f0b5384f00d697a646dbd269b6ecb0fbd9d31" [[package]] name = "cxxbridge-macro" -version = "1.0.90" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebf883b7aacd7b2aeb2a7b338648ee19f57c140d4ee8e52c68979c6b2f7f2263" +checksum = "631569015d0d8d54e6c241733f944042623ab6df7bc3be7466874b05fcdb1c5f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.9", ] [[package]] @@ -1492,12 +1508,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0808e1bd8671fb44a113a14e13497557533369847788fa2ae912b6ebfce9fa8" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" dependencies = [ - "darling_core 0.14.3", - "darling_macro 0.14.3", + "darling_core 0.14.4", + "darling_macro 0.14.4", ] [[package]] @@ -1511,21 +1527,21 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.10.0", - "syn", + "syn 1.0.109", ] [[package]] name = "darling_core" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "001d80444f28e193f30c2f293455da62dcf9a6b29918a4253152ae2b1de592cb" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim 0.10.0", - "syn", + "syn 1.0.109", ] [[package]] @@ -1536,18 +1552,18 @@ checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ "darling_core 0.13.4", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "darling_macro" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b36230598a2d5de7ec1c6f51f72d8a99a9208daff41de2084d06e3fd3ea56685" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" dependencies = [ - "darling_core 0.14.3", + "darling_core 0.14.4", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1593,7 +1609,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5bbed42daaa95e780b60a50546aa345b8413a1e46f9a40a12907d3598f038db" dependencies = [ "data-encoding", - "syn", + "syn 1.0.109", ] [[package]] @@ -1671,11 +1687,11 @@ dependencies = [ [[package]] name = "der-parser" -version = "8.1.0" +version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d4bc9b0db0a0df9ae64634ac5bdefb7afcb534e182275ca0beadbe486701c1" +checksum = "dbd676fbbab537128ef0278adb5576cf363cff6aa22a7b24effe97347cfab61e" dependencies = [ - "asn1-rs 0.5.1", + "asn1-rs 0.5.2", "displaydoc", "nom 7.1.3", "num-bigint", @@ -1691,7 +1707,7 @@ checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1699,10 +1715,10 @@ name = "derive_arbitrary" version = "1.2.2" source = "git+https://github.com/michaelsproul/arbitrary?rev=a572fd8743012a4f1ada5ee5968b1b3619c427ba#a572fd8743012a4f1ada5ee5968b1b3619c427ba" dependencies = [ - "darling 0.14.3", + "darling 0.14.4", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1720,10 +1736,10 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f91d4cfa921f1c05904dc3c57b4a32c38aed3340cce209f3a6fd1478babafc4" dependencies = [ - "darling 0.14.3", + "darling 0.14.4", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1733,7 +1749,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f0314b72bed045f3a68671b3c86328386762c93f82d98c65c3cb5e5f573dd68" dependencies = [ "derive_builder_core", - "syn", + "syn 1.0.109", ] [[package]] @@ -1746,7 +1762,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version 0.4.0", - "syn", + "syn 1.0.109", ] [[package]] @@ -1764,7 +1780,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" dependencies = [ - "block-buffer 0.10.3", + "block-buffer 0.10.4", "crypto-common", "subtle", ] @@ -1861,14 +1877,14 @@ checksum = "3bf95dc3f046b9da4f2d51833c0d3547d8564ef6910f5c1ed130306a75b92886" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "dtoa" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c00704156a7de8df8da0911424e30c2049957b0a714542a44e05fe693dd85313" +checksum = "65d09067bfacaa79114679b279d7f5885b53295b1e2cfb4e79c8e4bd3d633169" [[package]] name = "ecdsa" @@ -2021,7 +2037,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2080,6 +2096,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "errno" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d6a0976c999d473fe89ad888d5a284e55366d9dc9038b1ba2aa15128c4afa0" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys 0.45.0", +] + [[package]] name = "errno-dragonfly" version = "0.1.2" @@ -2285,7 +2312,7 @@ dependencies = [ "eth2_ssz", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2402,7 +2429,7 @@ dependencies = [ "hex", "integer-sqrt", "multiaddr 0.14.0", - "multihash", + "multihash 0.16.3", "rand 0.8.5", "serde", "serde_json", @@ -2634,18 +2661,18 @@ checksum = "ec54ac60a7f2ee9a97cad9946f9bf629a3bc6a7ae59e68983dc9318f5a54b81a" [[package]] name = "fiat-crypto" -version = "0.1.17" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a214f5bb88731d436478f3ae1f8a277b62124089ba9fb67f4f93fb100ef73c90" +checksum = "93ace6ec7cc19c8ed33a32eaa9ea692d7faea05006b5356b9e2b668ec4bc3955" [[package]] name = "field-offset" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e1c54951450cbd39f3dbcf1005ac413b49487dabf18a720ad2383eccfeffb92" +checksum = "a3cf3a800ff6e860c863ca6d4b16fd999db8b752819c1606884047b73e468535" dependencies = [ - "memoffset 0.6.5", - "rustc_version 0.3.3", + "memoffset 0.8.0", + "rustc_version 0.4.0", ] [[package]] @@ -2767,9 +2794,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84" +checksum = "531ac96c6ff5fd7c62263c5e3c67a603af4fcaee2e1a0ae5565ba3a11e69e549" dependencies = [ "futures-channel", "futures-core", @@ -2782,9 +2809,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5" +checksum = "164713a5a0dcc3e7b4b1ed7d3b433cabc18025386f9339346e8daf15963cf7ac" dependencies = [ "futures-core", "futures-sink", @@ -2792,15 +2819,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" +checksum = "86d7a0c1aa76363dac491de0ee99faf6941128376f1cf96f07db7603b7de69dd" [[package]] name = "futures-executor" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e" +checksum = "1997dd9df74cdac935c76252744c1ed5794fac083242ea4fe77ef3ed60ba0f83" dependencies = [ "futures-core", "futures-task", @@ -2810,9 +2837,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531" +checksum = "89d422fa3cbe3b40dca574ab087abb5bc98258ea57eea3fd6f1fa7162c778b91" [[package]] name = "futures-lite" @@ -2831,13 +2858,13 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" +checksum = "3eb14ed937631bd8b8b8977f2c198443447a8355b6e3ca599f38c975e5a963b6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2853,15 +2880,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" +checksum = "ec93083a4aecafb2a80a885c9de1f0ccae9dbd32c2bb54b0c3a65690e0b8d2f2" [[package]] name = "futures-task" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366" +checksum = "fd65540d33b37b16542a0438c12e6aeead10d4ac5d05bd3f805b8f35ab592879" [[package]] name = "futures-timer" @@ -2871,9 +2898,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" [[package]] name = "futures-util" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1" +checksum = "3ef6b17e481503ec85211fed8f39d1970f128935ca1f814cd32ac4a6842e84ab" dependencies = [ "futures-channel", "futures-core", @@ -2953,16 +2980,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "ghash" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97304e4cd182c3846f7575ced3890c53012ce534ad9114046b0a9e00bb30a375" -dependencies = [ - "opaque-debug", - "polyval 0.4.5", -] - [[package]] name = "ghash" version = "0.4.4" @@ -2974,10 +2991,20 @@ dependencies = [ ] [[package]] -name = "gimli" -version = "0.27.1" +name = "ghash" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "221996f774192f0f718773def8201c4ae31f02616a54ccfc2d358bb0e5cefdec" +checksum = "d930750de5717d2dd0b8c0d42c076c0e884c81a73e6cab859bbd2339c71e3e40" +dependencies = [ + "opaque-debug", + "polyval 0.6.0", +] + +[[package]] +name = "gimli" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" [[package]] name = "git-version" @@ -2998,7 +3025,7 @@ dependencies = [ "proc-macro-hack", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -3020,9 +3047,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" dependencies = [ "bytes", "fnv", @@ -3152,6 +3179,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + [[package]] name = "hex" version = "0.4.3" @@ -3183,16 +3216,6 @@ dependencies = [ "digest 0.9.0", ] -[[package]] -name = "hmac" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" -dependencies = [ - "crypto-mac 0.10.1", - "digest 0.9.0", -] - [[package]] name = "hmac" version = "0.11.0" @@ -3236,9 +3259,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ "bytes", "fnv", @@ -3349,9 +3372,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.24" +version = "0.14.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c" +checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" dependencies = [ "bytes", "futures-channel", @@ -3399,16 +3422,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.53" +version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +checksum = "0c17cc76786e99f8d2f055c11159e7f0091c42474dcc3189fbab96072e873e6d" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "winapi", + "windows 0.46.0", ] [[package]] @@ -3495,7 +3518,7 @@ dependencies = [ "rtnetlink", "system-configuration", "tokio", - "windows", + "windows 0.34.0", ] [[package]] @@ -3564,7 +3587,7 @@ checksum = "11d7a9f6330b71fea57921c9b61c47ee6e84f72d394754eff6163ae67e7395eb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -3577,6 +3600,15 @@ dependencies = [ "hashbrown 0.12.3", ] +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "generic-array", +] + [[package]] name = "instant" version = "0.1.12" @@ -3628,10 +3660,11 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.5" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3" +checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" dependencies = [ + "hermit-abi 0.3.1", "libc", "windows-sys 0.45.0", ] @@ -3665,9 +3698,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "jemalloc-ctl" @@ -3726,11 +3759,11 @@ dependencies = [ [[package]] name = "jsonwebtoken" -version = "8.2.0" +version = "8.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f4f04699947111ec1733e71778d763555737579e44b85844cae8e1940a1828" +checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" dependencies = [ - "base64 0.13.1", + "base64 0.21.0", "pem", "ring", "serde", @@ -3866,15 +3899,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.139" +version = "0.2.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" [[package]] name = "libflate" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05605ab2bce11bcfc0e9c635ff29ef8b2ea83f29be257ee7d730cac3ee373093" +checksum = "97822bf791bd4d5b403713886a5fbe8bf49520fe78e323b0dc480ca1a03e50b0" dependencies = [ "adler32", "crc32fast", @@ -3883,9 +3916,9 @@ dependencies = [ [[package]] name = "libflate_lz77" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39a734c0493409afcd49deee13c006a04e3586b9761a03543c6272c9c51f2f5a" +checksum = "a52d3a8bfc85f250440e4424db7d857e241a3aebbbe301f3eb606ab15c39acbf" dependencies = [ "rle-decode-fast", ] @@ -3929,9 +3962,9 @@ dependencies = [ [[package]] name = "libp2p" -version = "0.50.0" +version = "0.50.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e0a0d2f693675f49ded13c5d510c48b78069e23cbd9108d7ccd59f6dc568819" +checksum = "9c7b0104790be871edcf97db9bd2356604984e623a08d825c3f27852290266b8" dependencies = [ "bytes", "futures", @@ -3977,7 +4010,7 @@ dependencies = [ "libsecp256k1", "log", "multiaddr 0.14.0", - "multihash", + "multihash 0.16.3", "multistream-select 0.11.0", "p256", "parking_lot 0.12.1", @@ -4011,7 +4044,7 @@ dependencies = [ "libsecp256k1", "log", "multiaddr 0.16.0", - "multihash", + "multihash 0.16.3", "multistream-select 0.12.1", "once_cell", "p256", @@ -4030,6 +4063,34 @@ dependencies = [ "zeroize", ] +[[package]] +name = "libp2p-core" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b7f8b7d65c070a5a1b5f8f0510648189da08f787b8963f8e21219e0710733af" +dependencies = [ + "either", + "fnv", + "futures", + "futures-timer", + "instant", + "libp2p-identity", + "log", + "multiaddr 0.17.1", + "multihash 0.17.0", + "multistream-select 0.12.1", + "once_cell", + "parking_lot 0.12.1", + "pin-project", + "quick-protobuf", + "rand 0.8.5", + "rw-stream-sink", + "smallvec", + "thiserror", + "unsigned-varint 0.7.1", + "void", +] + [[package]] name = "libp2p-dns" version = "0.38.0" @@ -4095,6 +4156,24 @@ dependencies = [ "void", ] +[[package]] +name = "libp2p-identity" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a8ea433ae0cea7e3315354305237b9897afe45278b2118a7a57ca744e70fd27" +dependencies = [ + "bs58", + "ed25519-dalek", + "log", + "multiaddr 0.17.1", + "multihash 0.17.0", + "prost", + "quick-protobuf", + "rand 0.8.5", + "thiserror", + "zeroize", +] + [[package]] name = "libp2p-mdns" version = "0.42.0" @@ -4237,7 +4316,7 @@ checksum = "9d527d5827582abd44a6d80c07ff8b50b4ee238a8979e05998474179e79dc400" dependencies = [ "heck", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -4258,13 +4337,14 @@ dependencies = [ [[package]] name = "libp2p-tls" -version = "0.1.0-alpha" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7905ce0d040576634e8a3229a7587cc8beab83f79db6023800f1792895defa8" +checksum = "ff08d13d0dc66e5e9ba6279c1de417b84fa0d0adc3b03e5732928c180ec02781" dependencies = [ "futures", "futures-rustls", - "libp2p-core 0.38.0", + "libp2p-core 0.39.1", + "libp2p-identity", "rcgen 0.10.0", "ring", "rustls 0.20.8", @@ -4290,7 +4370,7 @@ dependencies = [ "libp2p-core 0.38.0", "libp2p-noise", "log", - "multihash", + "multihash 0.16.3", "prost", "prost-build", "prost-codec", @@ -4538,6 +4618,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" +[[package]] +name = "linux-raw-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd550e73688e6d578f0ac2119e32b797a327631a42f9433e59d02e139c8df60d" + [[package]] name = "lmdb-rkv" version = "0.14.0" @@ -4722,9 +4808,9 @@ dependencies = [ [[package]] name = "memoffset" -version = "0.7.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" dependencies = [ "autocfg 1.1.0", ] @@ -4761,7 +4847,7 @@ dependencies = [ "proc-macro2", "quote", "smallvec", - "syn", + "syn 1.0.109", ] [[package]] @@ -4795,9 +4881,9 @@ dependencies = [ [[package]] name = "mime" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "mime_guess" @@ -4872,7 +4958,7 @@ dependencies = [ "bs58", "byteorder", "data-encoding", - "multihash", + "multihash 0.16.3", "percent-encoding", "serde", "static_assertions", @@ -4890,7 +4976,26 @@ dependencies = [ "byteorder", "data-encoding", "multibase", - "multihash", + "multihash 0.16.3", + "percent-encoding", + "serde", + "static_assertions", + "unsigned-varint 0.7.1", + "url", +] + +[[package]] +name = "multiaddr" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b36f567c7099511fa8612bbbb52dda2419ce0bdbacf31714e3a5ffdb766d3bd" +dependencies = [ + "arrayref", + "byteorder", + "data-encoding", + "log", + "multibase", + "multihash 0.17.0", "percent-encoding", "serde", "static_assertions", @@ -4922,6 +5027,19 @@ dependencies = [ "unsigned-varint 0.7.1", ] +[[package]] +name = "multihash" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "835d6ff01d610179fbce3de1694d007e500bf33a7f29689838941d6bf783ae40" +dependencies = [ + "core2", + "digest 0.10.6", + "multihash-derive", + "sha2 0.10.6", + "unsigned-varint 0.7.1", +] + [[package]] name = "multihash-derive" version = "0.8.1" @@ -4932,7 +5050,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "synstructure", ] @@ -5061,9 +5179,9 @@ dependencies = [ [[package]] name = "netlink-sys" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "260e21fbb6f3d253a14df90eb0000a6066780a15dd901a7519ce02d77a94985b" +checksum = "6471bf08e7ac0135876a9581bf3217ef0333c191c128d34878079f42ee150411" dependencies = [ "bytes", "futures", @@ -5114,7 +5232,7 @@ dependencies = [ "task_executor", "tokio", "tokio-stream", - "tokio-util 0.6.10", + "tokio-util 0.7.7", "types", ] @@ -5314,7 +5432,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bedf36ffb6ba96c2eb7144ef6270557b52e54b20c0a8e1eb2ff99a6c6959bff" dependencies = [ - "asn1-rs 0.5.1", + "asn1-rs 0.5.2", ] [[package]] @@ -5364,14 +5482,14 @@ dependencies = [ "bytes", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "openssl" -version = "0.10.45" +version = "0.10.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" +checksum = "518915b97df115dd36109bfa429a48b8f737bd05508cf9588977b599648926d2" dependencies = [ "bitflags", "cfg-if", @@ -5390,7 +5508,7 @@ checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -5401,18 +5519,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "111.25.1+1.1.1t" +version = "111.25.2+1.1.1t" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ef9a9cc6ea7d9d5e7c4a913dc4b48d0e359eddf01af1dfec96ba7064b4aba10" +checksum = "320708a054ad9b3bf314688b5db87cf4d6683d64cfc835e2337924ae62bf4431" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.80" +version = "0.9.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" +checksum = "666416d899cf077260dac8698d60a60b435a46d57e82acb1be3d0dad87284e5b" dependencies = [ "autocfg 1.1.0", "cc", @@ -5521,7 +5639,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -5533,7 +5651,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -5592,9 +5710,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba" +checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" [[package]] name = "pbkdf2" @@ -5644,16 +5762,6 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" -[[package]] -name = "pest" -version = "2.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "028accff104c4e513bad663bbcd2ad7cfd5304144404c31ed0a77ac103d00660" -dependencies = [ - "thiserror", - "ucd-trie", -] - [[package]] name = "petgraph" version = "0.6.3" @@ -5691,7 +5799,7 @@ checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -5770,16 +5878,18 @@ dependencies = [ [[package]] name = "polling" -version = "2.5.2" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22122d5ec4f9fe1b3916419b76be1e80bcb93f618d071d2edf841b137b2a2bd6" +checksum = "7e1f879b2998099c2d69ab9605d145d5b661195627eccc680002c4918a7fb6fa" dependencies = [ "autocfg 1.1.0", + "bitflags", "cfg-if", + "concurrent-queue", "libc", "log", - "wepoll-ffi", - "windows-sys 0.42.0", + "pin-project-lite 0.2.9", + "windows-sys 0.45.0", ] [[package]] @@ -5790,18 +5900,7 @@ checksum = "048aeb476be11a4b6ca432ca569e375810de9294ae78f4774e78ea98a9246ede" dependencies = [ "cpufeatures", "opaque-debug", - "universal-hash", -] - -[[package]] -name = "polyval" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eebcc4aa140b9abd2bc40d9c3f7ccec842679cd79045ac3a7ac698c1a064b7cd" -dependencies = [ - "cpuid-bool", - "opaque-debug", - "universal-hash", + "universal-hash 0.4.1", ] [[package]] @@ -5813,7 +5912,19 @@ dependencies = [ "cfg-if", "cpufeatures", "opaque-debug", - "universal-hash", + "universal-hash 0.4.1", +] + +[[package]] +name = "polyval" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef234e08c11dfcb2e56f79fd70f6f2eb7f025c0ce2333e82f4f0518ecad30c6" +dependencies = [ + "cfg-if", + "cpufeatures", + "opaque-debug", + "universal-hash 0.5.0", ] [[package]] @@ -5824,12 +5935,12 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "prettyplease" -version = "0.1.23" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e97e3215779627f01ee256d2fad52f3d95e8e1c11e9fc6fd08f7cd455d5d5c78" +checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" dependencies = [ "proc-macro2", - "syn", + "syn 1.0.109", ] [[package]] @@ -5878,7 +5989,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "version_check", ] @@ -5901,9 +6012,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.51" +version = "1.0.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" +checksum = "ba466839c78239c09faf015484e5cc04860f88242cff4d03eb038f04b4699b73" dependencies = [ "unicode-ident", ] @@ -5955,14 +6066,14 @@ checksum = "66a455fbcb954c1a7decf3c586e860fd7889cddf4b8e164be736dbac95a953cd" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "prost" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21dc42e00223fc37204bd4aa177e69420c604ca4a183209a8f9de30c6d934698" +checksum = "e48e50df39172a3e7eb17e14642445da64996989bc212b583015435d39a58537" dependencies = [ "bytes", "prost-derive", @@ -5970,9 +6081,9 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3f8ad728fb08fe212df3c05169e940fbb6d9d16a877ddde14644a983ba2012e" +checksum = "2c828f93f5ca4826f97fedcbd3f9a536c16b12cff3dbbb4a007f932bbad95b12" dependencies = [ "bytes", "heck", @@ -5985,7 +6096,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn", + "syn 1.0.109", "tempfile", "which", ] @@ -6005,24 +6116,23 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bda8c0881ea9f722eb9629376db3d0b903b462477c1aafcb0566610ac28ac5d" +checksum = "4ea9b0f8cbe5e15a8a042d030bd96668db28ecb567ec37d691971ff5731d2b1b" dependencies = [ "anyhow", "itertools", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "prost-types" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e0526209433e96d83d750dd81a99118edbc55739e7e61a46764fd2ad537788" +checksum = "379119666929a1afd7a043aa6cf96fa67a6dce9af60c88095a4686dbce4c9c88" dependencies = [ - "bytes", "prost", ] @@ -6070,6 +6180,15 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-protobuf" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d6da84cc204722a989e01ba2f6e1e276e190f22263d0cb6ce8526fcdb0d2e1f" +dependencies = [ + "byteorder", +] + [[package]] name = "quickcheck" version = "0.9.2" @@ -6090,7 +6209,7 @@ checksum = "608c156fd8e97febc07dc9c2e2c80bf74cfc6ef26893eae3daf8bc2bc94a4b7f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -6124,9 +6243,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.23" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" dependencies = [ "proc-macro2", ] @@ -6246,9 +6365,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" dependencies = [ "either", "rayon-core", @@ -6256,9 +6375,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.10.2" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -6274,7 +6393,7 @@ checksum = "6413f3de1edee53342e6138e75b56d32e7bc6e332b3bd62d497b1929d4cfbcdd" dependencies = [ "pem", "ring", - "time 0.3.17", + "time 0.3.20", "x509-parser 0.13.2", "yasna", ] @@ -6287,7 +6406,7 @@ checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b" dependencies = [ "pem", "ring", - "time 0.3.17", + "time 0.3.20", "yasna", ] @@ -6313,9 +6432,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" +checksum = "cce168fea28d3e05f158bda4576cf0c844d5045bc2cc3620fa0292ed5bb5814c" dependencies = [ "aho-corasick", "memchr", @@ -6333,15 +6452,15 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.28" +version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "reqwest" -version = "0.11.14" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9" +checksum = "0ba30cc2c0cd02af1222ed216ba659cdb2f879dfe3181852fe7c50b1d0005949" dependencies = [ "base64 0.21.0", "bytes", @@ -6441,7 +6560,7 @@ checksum = "e33d7b2abe0c340d8797fe2907d3f20d3b5ea5908683618bfe80df7f621f672a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -6510,9 +6629,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" +checksum = "d4a36c42d1873f9a77c53bde094f9664d9891bc604a45b4798fd2c389ed12e5b" [[package]] name = "rustc-hash" @@ -6535,22 +6654,13 @@ dependencies = [ "semver 0.9.0", ] -[[package]] -name = "rustc_version" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee" -dependencies = [ - "semver 0.11.0", -] - [[package]] name = "rustc_version" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.16", + "semver 1.0.17", ] [[package]] @@ -6564,15 +6674,29 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.9" +version = "0.36.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd5c6ff11fecd55b40746d1995a02f2eb375bf8c00d192d521ee09f42bef37bc" +checksum = "db4165c9963ab29e422d6c26fbc1d37f15bace6b2810221f9d925023480fcf0e" dependencies = [ "bitflags", - "errno", + "errno 0.2.8", "io-lifetimes", "libc", - "linux-raw-sys", + "linux-raw-sys 0.1.4", + "windows-sys 0.45.0", +] + +[[package]] +name = "rustix" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b24138615de35e32031d041a09032ef3487a616d901ca4db224e7d557efae2" +dependencies = [ + "bitflags", + "errno 0.3.0", + "io-lifetimes", + "libc", + "linux-raw-sys 0.3.0", "windows-sys 0.45.0", ] @@ -6612,9 +6736,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" +checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" [[package]] name = "rw-stream-sink" @@ -6629,9 +6753,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" [[package]] name = "safe_arith" @@ -6663,9 +6787,9 @@ dependencies = [ [[package]] name = "scale-info" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "001cf62ece89779fd16105b5f515ad0e5cedcd5440d3dd806bb067978e7c3608" +checksum = "61471dff9096de1d8b2319efed7162081e96793f5ebb147e50db10d50d648a4d" dependencies = [ "cfg-if", "derive_more", @@ -6675,14 +6799,14 @@ dependencies = [ [[package]] name = "scale-info-derive" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "303959cf613a6f6efd19ed4b4ad5bf79966a13352716299ad532cfb115f4205c" +checksum = "219580e803a66b3f05761fd06f1f879a872444e49ce23f73694d26e5a954c7e6" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -6696,9 +6820,9 @@ dependencies = [ [[package]] name = "scheduled-thread-pool" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "977a7519bff143a44f842fd07e80ad1329295bd71686457f18e496736f4bf9bf" +checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" dependencies = [ "parking_lot 0.12.1", ] @@ -6717,9 +6841,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "scratch" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" +checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" [[package]] name = "scrypt" @@ -6826,23 +6950,14 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" dependencies = [ - "semver-parser 0.7.0", + "semver-parser", ] [[package]] name = "semver" -version = "0.11.0" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" -dependencies = [ - "semver-parser 0.10.2", -] - -[[package]] -name = "semver" -version = "1.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" +checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" [[package]] name = "semver-parser" @@ -6850,15 +6965,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" -[[package]] -name = "semver-parser" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" -dependencies = [ - "pest", -] - [[package]] name = "send_wrapper" version = "0.6.0" @@ -6875,9 +6981,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.152" +version = "1.0.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "771d4d9c4163ee138805e12c710dd365e4f44be8be0503cb1bb9eb989425d9c9" dependencies = [ "serde_derive", ] @@ -6904,20 +7010,20 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "e801c1712f48475582b7696ac71e0ca34ebb30e09338425384269d9717c62cad" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.9", ] [[package]] name = "serde_json" -version = "1.0.93" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" +checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea" dependencies = [ "itoa", "ryu", @@ -6926,13 +7032,13 @@ dependencies = [ [[package]] name = "serde_repr" -version = "0.1.10" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a5ec9fa74a20ebbe5d9ac23dac1fc96ba0ecfe9f50f2843b52e537b10fbcb4e" +checksum = "bcec881020c684085e55a25f7fd888954d56609ef363479dc5a1305eb0d40cab" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.9", ] [[package]] @@ -6966,7 +7072,7 @@ dependencies = [ "darling 0.13.4", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -7105,7 +7211,7 @@ dependencies = [ "num-bigint", "num-traits", "thiserror", - "time 0.3.17", + "time 0.3.20", ] [[package]] @@ -7128,9 +7234,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" dependencies = [ "autocfg 1.1.0", ] @@ -7231,7 +7337,7 @@ dependencies = [ "serde", "serde_json", "slog", - "time 0.3.17", + "time 0.3.20", ] [[package]] @@ -7276,7 +7382,7 @@ dependencies = [ "slog", "term", "thread_local", - "time 0.3.17", + "time 0.3.20", ] [[package]] @@ -7327,14 +7433,14 @@ checksum = "5e9f0ab6ef7eb7353d9119c170a436d1bf248eea575ac42d19d12f4e34130831" [[package]] name = "snow" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12ba5f4d4ff12bdb6a169ed51b7c48c0e0ac4b0b4b31012b2571e97d78d3201d" +checksum = "5ccba027ba85743e09d15c03296797cad56395089b832b48b5a5217880f57733" dependencies = [ "aes-gcm 0.9.4", "blake2", "chacha20poly1305", - "curve25519-dalek 4.0.0-rc.0", + "curve25519-dalek 4.0.0-rc.1", "rand_core 0.6.4", "ring", "rustc_version 0.4.0", @@ -7344,9 +7450,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", @@ -7405,7 +7511,7 @@ source = "git+https://github.com/ralexstokes//ssz-rs?rev=adf1a0b14cef90b9536f28e dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -7509,7 +7615,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn", + "syn 1.0.109", ] [[package]] @@ -7557,7 +7663,7 @@ dependencies = [ "proc-macro2", "quote", "smallvec", - "syn", + "syn 1.0.109", ] [[package]] @@ -7571,7 +7677,7 @@ dependencies = [ "proc-macro2", "quote", "smallvec", - "syn", + "syn 1.0.109", ] [[package]] @@ -7585,9 +7691,20 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.107" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0da4a3c17e109f700685ec577c0f85efd9b19bcf15c913985f14dc1ac01775aa" dependencies = [ "proc-macro2", "quote", @@ -7608,7 +7725,7 @@ checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", "unicode-xid", ] @@ -7708,7 +7825,7 @@ dependencies = [ "cfg-if", "fastrand", "redox_syscall", - "rustix", + "rustix 0.36.11", "windows-sys 0.42.0", ] @@ -7745,7 +7862,7 @@ name = "test_random_derive" version = "0.2.0" dependencies = [ "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -7759,22 +7876,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.9", ] [[package]] @@ -7809,9 +7926,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.17" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" +checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" dependencies = [ "itoa", "libc", @@ -7829,9 +7946,9 @@ checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" [[package]] name = "time-macros" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" +checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" dependencies = [ "time-core", ] @@ -7938,7 +8055,7 @@ checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -7975,9 +8092,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" +checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" dependencies = [ "futures-core", "pin-project-lite 0.2.9", @@ -8123,7 +8240,7 @@ checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -8191,7 +8308,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebeb235c5847e2f82cfe0f07eb971d1e5f6804b18dac2ae16349cc604380f82f" dependencies = [ "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -8215,7 +8332,7 @@ version = "0.4.0" dependencies = [ "darling 0.13.4", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -8406,12 +8523,6 @@ dependencies = [ "tree_hash_derive", ] -[[package]] -name = "ucd-trie" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" - [[package]] name = "uint" version = "0.9.5" @@ -8442,15 +8553,15 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.10" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" [[package]] name = "unicode-normalization" @@ -8483,6 +8594,16 @@ dependencies = [ "subtle", ] +[[package]] +name = "universal-hash" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d3160b73c9a19f7e2939a2fdad446c57c1bbbbf4d919d3213ff1267a580d8b5" +dependencies = [ + "crypto-common", + "subtle", +] + [[package]] name = "unsigned-varint" version = "0.6.0" @@ -8668,12 +8789,11 @@ checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" [[package]] name = "walkdir" -version = "2.3.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" dependencies = [ "same-file", - "winapi", "winapi-util", ] @@ -8774,7 +8894,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-shared", ] @@ -8808,7 +8928,7 @@ checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -9008,7 +9128,7 @@ dependencies = [ "sha2 0.10.6", "stun", "thiserror", - "time 0.3.17", + "time 0.3.20", "tokio", "turn", "url", @@ -9040,22 +9160,22 @@ dependencies = [ [[package]] name = "webrtc-dtls" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7021987ae0a2ed6c8cd33f68e98e49bb6e74ffe9543310267b48a1bbe3900e5f" +checksum = "942be5bd85f072c3128396f6e5a9bfb93ca8c1939ded735d177b7bcba9a13d05" dependencies = [ "aes 0.6.0", - "aes-gcm 0.8.0", + "aes-gcm 0.10.1", "async-trait", "bincode", "block-modes", "byteorder", "ccm", "curve25519-dalek 3.2.0", - "der-parser 8.1.0", + "der-parser 8.2.0", "elliptic-curve", "hkdf", - "hmac 0.10.1", + "hmac 0.12.1", "log", "oid-registry 0.6.1", "p256", @@ -9067,8 +9187,8 @@ dependencies = [ "rustls 0.19.1", "sec1", "serde", - "sha-1 0.9.8", - "sha2 0.9.9", + "sha1", + "sha2 0.10.6", "signature", "subtle", "thiserror", @@ -9194,15 +9314,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "wepoll-ffi" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d743fdedc5c64377b5fc2bc036b01c7fd642205a0d96356034ae3404d49eb7fb" -dependencies = [ - "cc", -] - [[package]] name = "which" version = "4.4.0" @@ -9270,6 +9381,15 @@ dependencies = [ "windows_x86_64_msvc 0.34.0", ] +[[package]] +name = "windows" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-acl" version = "0.3.0" @@ -9289,12 +9409,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc 0.42.1", - "windows_i686_gnu 0.42.1", - "windows_i686_msvc 0.42.1", - "windows_x86_64_gnu 0.42.1", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", "windows_x86_64_gnullvm", - "windows_x86_64_msvc 0.42.1", + "windows_x86_64_msvc 0.42.2", ] [[package]] @@ -9308,24 +9428,24 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc 0.42.1", - "windows_i686_gnu 0.42.1", - "windows_i686_msvc 0.42.1", - "windows_x86_64_gnu 0.42.1", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", "windows_x86_64_gnullvm", - "windows_x86_64_msvc 0.42.1", + "windows_x86_64_msvc 0.42.2", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_msvc" @@ -9335,9 +9455,9 @@ checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d" [[package]] name = "windows_aarch64_msvc" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_i686_gnu" @@ -9347,9 +9467,9 @@ checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed" [[package]] name = "windows_i686_gnu" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_msvc" @@ -9359,9 +9479,9 @@ checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956" [[package]] name = "windows_i686_msvc" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_x86_64_gnu" @@ -9371,15 +9491,15 @@ checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4" [[package]] name = "windows_x86_64_gnu" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_msvc" @@ -9389,9 +9509,9 @@ checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9" [[package]] name = "windows_x86_64_msvc" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "winreg" @@ -9474,7 +9594,7 @@ dependencies = [ "ring", "rusticata-macros", "thiserror", - "time 0.3.17", + "time 0.3.20", ] [[package]] @@ -9483,16 +9603,16 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0ecbeb7b67ce215e40e3cc7f2ff902f94a223acf44995934763467e7b1febc8" dependencies = [ - "asn1-rs 0.5.1", + "asn1-rs 0.5.2", "base64 0.13.1", "data-encoding", - "der-parser 8.1.0", + "der-parser 8.2.0", "lazy_static", "nom 7.1.3", "oid-registry 0.6.1", "rusticata-macros", "thiserror", - "time 0.3.17", + "time 0.3.20", ] [[package]] @@ -9539,7 +9659,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aed2e7a52e3744ab4d0c05c20aa065258e84c49fd4226f5191b2ed29712710b4" dependencies = [ - "time 0.3.17", + "time 0.3.20", ] [[package]] @@ -9559,7 +9679,7 @@ checksum = "44bf07cb3e50ea2003396695d58bf46bc9887a1f362260446fad6bc4e79bd36c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", "synstructure", ] diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index a23061fff..1d4f1d17e 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -8,15 +8,19 @@ use crate::beacon_block_streamer::{BeaconBlockStreamer, CheckEarlyAttesterCache} use crate::beacon_proposer_cache::compute_proposer_duties_from_head; use crate::beacon_proposer_cache::BeaconProposerCache; use crate::blob_cache::BlobCache; -use crate::blob_verification::{AsBlock, AvailableBlock, BlockWrapper}; +use crate::blob_verification::{self, AsBlock, BlobError, BlockWrapper, GossipVerifiedBlob}; use crate::block_times_cache::BlockTimesCache; +use crate::block_verification::POS_PANDA_BANNER; use crate::block_verification::{ check_block_is_finalized_checkpoint_or_descendant, check_block_relevancy, get_block_root, - signature_verify_chain_segment, BlockError, ExecutionPendingBlock, GossipVerifiedBlock, - IntoExecutionPendingBlock, PayloadVerificationOutcome, POS_PANDA_BANNER, + signature_verify_chain_segment, AvailableExecutedBlock, BlockError, BlockImportData, + ExecutedBlock, ExecutionPendingBlock, GossipVerifiedBlock, IntoExecutionPendingBlock, }; pub use crate::canonical_head::{CanonicalHead, CanonicalHeadRwLock}; use crate::chain_config::ChainConfig; +use crate::data_availability_checker::{ + Availability, AvailabilityCheckError, AvailableBlock, DataAvailabilityChecker, +}; use crate::early_attester_cache::EarlyAttesterCache; use crate::errors::{BeaconChainError as Error, BlockProductionError}; use crate::eth1_chain::{Eth1Chain, Eth1ChainBackend}; @@ -109,8 +113,9 @@ use store::{ use task_executor::{ShutdownReason, TaskExecutor}; use tokio_stream::Stream; use tree_hash::TreeHash; +use types::beacon_block_body::KzgCommitments; use types::beacon_state::CloneConfig; -use types::blobs_sidecar::KzgCommitments; +use types::blob_sidecar::{BlobIdentifier, BlobSidecarList, Blobs}; use types::consts::eip4844::MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS; use types::consts::merge::INTERVALS_PER_SLOT; use types::*; @@ -183,6 +188,36 @@ pub enum WhenSlotSkipped { Prev, } +#[derive(Debug, PartialEq)] +pub enum AvailabilityProcessingStatus { + PendingBlobs(Vec), + PendingBlock(Hash256), + Imported(Hash256), +} + +//TODO(sean) using this in tests for now +impl TryInto for AvailabilityProcessingStatus { + type Error = (); + + fn try_into(self) -> Result { + match self { + AvailabilityProcessingStatus::Imported(hash) => Ok(hash.into()), + _ => Err(()), + } + } +} + +impl TryInto for AvailabilityProcessingStatus { + type Error = (); + + fn try_into(self) -> Result { + match self { + AvailabilityProcessingStatus::Imported(hash) => Ok(hash), + _ => Err(()), + } + } +} + /// The result of a chain segment processing. pub enum ChainSegmentResult { /// Processing this chain segment finished successfully. @@ -433,8 +468,9 @@ pub struct BeaconChain { pub slasher: Option>>, /// Provides monitoring of a set of explicitly defined validators. pub validator_monitor: RwLock>, - pub blob_cache: BlobCache, - pub kzg: Option>, + pub proposal_blob_cache: BlobCache, + pub data_availability_checker: DataAvailabilityChecker, + pub kzg: Option>, } type BeaconBlockAndState = (BeaconBlock, BeaconState); @@ -991,19 +1027,9 @@ impl BeaconChain { &self, block_root: &Hash256, ) -> Result>, Error> { - // If there is no data availability boundary, the Eip4844 fork is disabled. - if let Some(finalized_data_availability_boundary) = - self.finalized_data_availability_boundary() - { - self.early_attester_cache - .get_blobs(*block_root) - .map_or_else( - || self.get_blobs(block_root, finalized_data_availability_boundary), - |blobs| Ok(Some(blobs)), - ) - } else { - Ok(None) - } + self.early_attester_cache + .get_blobs(*block_root) + .map_or_else(|| self.get_blobs(block_root), |blobs| Ok(Some(blobs))) } /// Returns the block at the given root, if any. @@ -1086,35 +1112,8 @@ impl BeaconChain { pub fn get_blobs( &self, block_root: &Hash256, - data_availability_boundary: Epoch, ) -> Result>, Error> { - match self.store.get_blobs(block_root)? { - Some(blob_sidecar_list) => Ok(Some(blob_sidecar_list)), - None => { - // Check for the corresponding block to understand whether we *should* have blobs. - self.get_blinded_block(block_root)? - .map(|block| { - // If there are no KZG commitments in the block, we know the sidecar should - // be empty. - let expected_kzg_commitments = - match block.message().body().blob_kzg_commitments() { - Ok(kzg_commitments) => kzg_commitments, - Err(_) => return Err(Error::NoKzgCommitmentsFieldOnBlock), - }; - if expected_kzg_commitments.is_empty() { - // TODO (mark): verify this - Ok(BlobSidecarList::empty()) - } else if data_availability_boundary <= block.epoch() { - // We should have blobs for all blocks younger than the boundary. - Err(Error::BlobsUnavailable) - } else { - // We shouldn't have blobs for blocks older than the boundary. - Err(Error::BlobsOlderThanDataAvailabilityBoundary(block.epoch())) - } - }) - .transpose() - } - } + Ok(self.store.get_blobs(block_root)?) } pub fn get_blinded_block( @@ -1936,6 +1935,15 @@ impl BeaconChain { }) } + pub fn verify_blob_sidecar_for_gossip( + self: &Arc, + blob_sidecar: SignedBlobSidecar, + subnet_id: u64, + ) -> Result, BlobError> // TODO(pawan): make a GossipVerifedBlob type + { + blob_verification::validate_blob_sidecar_for_gossip(blob_sidecar, subnet_id, self) + } + /// Accepts some 'LightClientOptimisticUpdate' from the network and attempts to verify it pub fn verify_optimistic_update_for_gossip( self: &Arc, @@ -2686,13 +2694,29 @@ impl BeaconChain { .map_err(BeaconChainError::TokioJoin)? } + pub async fn process_blob( + self: &Arc, + blob: GossipVerifiedBlob, + count_unrealized: CountUnrealized, + ) -> Result> { + self.check_availability_and_maybe_import( + |chain| chain.data_availability_checker.put_gossip_blob(blob), + count_unrealized, + ) + .await + } + /// Returns `Ok(block_root)` if the given `unverified_block` was successfully verified and /// imported into the chain. /// + /// For post deneb blocks, this returns a `BlockError::AvailabilityPending` error + /// if the corresponding blobs are not in the required caches. + /// /// Items that implement `IntoExecutionPendingBlock` include: /// /// - `SignedBeaconBlock` /// - `GossipVerifiedBlock` + /// - `BlockWrapper` /// /// ## Errors /// @@ -2704,105 +2728,67 @@ impl BeaconChain { unverified_block: B, count_unrealized: CountUnrealized, notify_execution_layer: NotifyExecutionLayer, - ) -> Result> { + ) -> Result> { // Start the Prometheus timer. let _full_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_TIMES); // Increment the Prometheus counter for block processing requests. metrics::inc_counter(&metrics::BLOCK_PROCESSING_REQUESTS); - let slot = unverified_block.block().slot(); - - // A small closure to group the verification and import errors. let chain = self.clone(); - let import_block = async move { - let execution_pending = unverified_block.into_execution_pending_block( - block_root, - &chain, - notify_execution_layer, - )?; - chain - .import_execution_pending_block(execution_pending, count_unrealized) + + let execution_pending = unverified_block.into_execution_pending_block( + block_root, + &chain, + notify_execution_layer, + )?; + + let executed_block = self + .clone() + .into_executed_block(execution_pending) + .await + .map_err(|e| self.handle_block_error(e))?; + + match executed_block { + ExecutedBlock::Available(block) => { + self.import_available_block(Box::new(block), count_unrealized) + .await + } + ExecutedBlock::AvailabilityPending(block) => { + self.check_availability_and_maybe_import( + |chain| { + chain + .data_availability_checker + .put_pending_executed_block(block) + }, + count_unrealized, + ) .await - }; - - // Verify and import the block. - match import_block.await { - // The block was successfully verified and imported. Yay. - Ok(block_root) => { - trace!( - self.log, - "Beacon block imported"; - "block_root" => ?block_root, - "block_slot" => slot, - ); - - // Increment the Prometheus counter for block processing successes. - metrics::inc_counter(&metrics::BLOCK_PROCESSING_SUCCESSES); - - Ok(block_root) - } - Err(e @ BlockError::BeaconChainError(BeaconChainError::TokioJoin(_))) => { - debug!( - self.log, - "Beacon block processing cancelled"; - "error" => ?e, - ); - Err(e) - } - // There was an error whilst attempting to verify and import the block. The block might - // be partially verified or partially imported. - Err(BlockError::BeaconChainError(e)) => { - crit!( - self.log, - "Beacon block processing error"; - "error" => ?e, - ); - Err(BlockError::BeaconChainError(e)) - } - // The block failed verification. - Err(other) => { - trace!( - self.log, - "Beacon block rejected"; - "reason" => other.to_string(), - ); - Err(other) } } } - /// Accepts a fully-verified block and imports it into the chain without performing any - /// additional verification. + /// Accepts a fully-verified block and awaits on it's payload verification handle to + /// get a fully `ExecutedBlock` /// - /// An error is returned if the block was unable to be imported. It may be partially imported - /// (i.e., this function is not atomic). - async fn import_execution_pending_block( + /// An error is returned if the verification handle couldn't be awaited. + async fn into_executed_block( self: Arc, execution_pending_block: ExecutionPendingBlock, - count_unrealized: CountUnrealized, - ) -> Result> { + ) -> Result, BlockError> { let ExecutionPendingBlock { block, - block_root, - state, - parent_block, - confirmed_state_roots, + import_data, payload_verification_handle, - parent_eth1_finalization_data, - consensus_context, } = execution_pending_block; - let PayloadVerificationOutcome { - payload_verification_status, - is_valid_merge_transition_block, - } = payload_verification_handle + let payload_verification_outcome = payload_verification_handle .await .map_err(BeaconChainError::TokioJoin)? .ok_or(BeaconChainError::RuntimeShutdown)??; // Log the PoS pandas if a merge transition just occurred. - if is_valid_merge_transition_block { + if payload_verification_outcome.is_valid_merge_transition_block { info!(self.log, "{}", POS_PANDA_BANNER); info!( self.log, @@ -2830,9 +2816,91 @@ impl BeaconChain { .into_root() ); } + Ok(ExecutedBlock::new( + block, + import_data, + payload_verification_outcome, + )) + } + fn handle_block_error(&self, e: BlockError) -> BlockError { + match e { + e @ BlockError::BeaconChainError(BeaconChainError::TokioJoin(_)) => { + debug!( + self.log, + "Beacon block processing cancelled"; + "error" => ?e, + ); + e + } + BlockError::BeaconChainError(e) => { + crit!( + self.log, + "Beacon block processing error"; + "error" => ?e, + ); + BlockError::BeaconChainError(e) + } + other => { + trace!( + self.log, + "Beacon block rejected"; + "reason" => other.to_string(), + ); + other + } + } + } + + /// Accepts a fully-verified, available block and imports it into the chain without performing any + /// additional verification. + /// + /// An error is returned if the block was unable to be imported. It may be partially imported + /// (i.e., this function is not atomic). + pub async fn check_availability_and_maybe_import( + self: &Arc, + cache_fn: impl FnOnce(Arc) -> Result, AvailabilityCheckError>, + count_unrealized: CountUnrealized, + ) -> Result> { + let availability = cache_fn(self.clone())?; + match availability { + Availability::Available(block) => { + self.import_available_block(block, count_unrealized).await + } + Availability::PendingBlock(block_root) => { + Ok(AvailabilityProcessingStatus::PendingBlock(block_root)) + } + Availability::PendingBlobs(blob_ids) => { + Ok(AvailabilityProcessingStatus::PendingBlobs(blob_ids)) + } + } + } + + async fn import_available_block( + self: &Arc, + block: Box>, + count_unrealized: CountUnrealized, + ) -> Result> { + let AvailableExecutedBlock { + block, + import_data, + payload_verification_outcome, + } = *block; + + let BlockImportData { + block_root, + state, + parent_block, + parent_eth1_finalization_data, + confirmed_state_roots, + consensus_context, + } = import_data; + + let slot = block.slot(); + + // import let chain = self.clone(); - let block_hash = self + let result = self .spawn_blocking_handle( move || { chain.import_block( @@ -2840,7 +2908,7 @@ impl BeaconChain { block_root, state, confirmed_state_roots, - payload_verification_status, + payload_verification_outcome.payload_verification_status, count_unrealized, parent_block, parent_eth1_finalization_data, @@ -2849,12 +2917,32 @@ impl BeaconChain { }, "payload_verification_handle", ) - .await??; + .await + .map_err(|e| { + let b = BlockError::from(e); + self.handle_block_error(b) + })?; - Ok(block_hash) + match result { + // The block was successfully verified and imported. Yay. + Ok(block_root) => { + trace!( + self.log, + "Beacon block imported"; + "block_root" => ?block_root, + "block_slot" => slot, + ); + + // Increment the Prometheus counter for block processing successes. + metrics::inc_counter(&metrics::BLOCK_PROCESSING_SUCCESSES); + + Ok(AvailabilityProcessingStatus::Imported(block_root)) + } + Err(e) => Err(self.handle_block_error(e)), + } } - /// Accepts a fully-verified block and imports it into the chain without performing any + /// Accepts a fully-verified and available block and imports it into the chain without performing any /// additional verification. /// /// An error is returned if the block was unable to be imported. It may be partially imported @@ -3038,27 +3126,17 @@ impl BeaconChain { ops.push(StoreOp::PutBlock(block_root, signed_block.clone())); ops.push(StoreOp::PutState(block.state_root(), &state)); - // Only consider blobs if the eip4844 fork is enabled. - if let Some(data_availability_boundary) = self.data_availability_boundary() { - let block_epoch = block.slot().epoch(T::EthSpec::slots_per_epoch()); - let margin_epochs = self.store.get_config().blob_prune_margin_epochs; - let import_boundary = data_availability_boundary - margin_epochs; - - // Only store blobs at the data availability boundary, minus any configured epochs - // margin, or younger (of higher epoch number). - if block_epoch >= import_boundary { - if let Some(blobs) = blobs { - if !blobs.is_empty() { - //FIXME(sean) using this for debugging for now - info!( - self.log, "Writing blobs to store"; - "block_root" => ?block_root - ); - ops.push(StoreOp::PutBlobs(block_root, blobs)); - } - } + if let Some(blobs) = blobs { + if !blobs.is_empty() { + //FIXME(sean) using this for debugging for now + info!( + self.log, "Writing blobs to store"; + "block_root" => ?block_root + ); + ops.push(StoreOp::PutBlobs(block_root, blobs)); } } + let txn_lock = self.store.hot_db.begin_rw_transaction(); if let Err(e) = self.store.do_atomically_with_block_and_blobs_cache(ops) { @@ -4782,12 +4860,11 @@ impl BeaconChain { .as_ref() .ok_or(BlockProductionError::TrustedSetupNotInitialized)?; let beacon_block_root = block.canonical_root(); - let expected_kzg_commitments: &KzgCommitments = - block.body().blob_kzg_commitments().map_err(|_| { - BlockProductionError::InvalidBlockVariant( - "EIP4844 block does not contain kzg commitments".to_string(), - ) - })?; + let expected_kzg_commitments = block.body().blob_kzg_commitments().map_err(|_| { + BlockProductionError::InvalidBlockVariant( + "EIP4844 block does not contain kzg commitments".to_string(), + ) + })?; if expected_kzg_commitments.len() != blobs.len() { return Err(BlockProductionError::MissingKzgCommitment(format!( @@ -4836,7 +4913,8 @@ impl BeaconChain { .collect::, BlockProductionError>>()?, ); - self.blob_cache.put(beacon_block_root, blob_sidecars); + self.proposal_blob_cache + .put(beacon_block_root, blob_sidecars); } metrics::inc_counter(&metrics::BLOCK_PRODUCTION_SUCCESSES); @@ -4854,8 +4932,8 @@ impl BeaconChain { fn compute_blob_kzg_proofs( kzg: &Arc, - blobs: &Blobs<::EthSpec>, - expected_kzg_commitments: &KzgCommitments<::EthSpec>, + blobs: &Blobs, + expected_kzg_commitments: &KzgCommitments, slot: Slot, ) -> Result, BlockProductionError> { blobs @@ -6119,18 +6197,10 @@ impl BeaconChain { }) } - /// The epoch that is a data availability boundary, or the latest finalized epoch. - /// `None` if the `Eip4844` fork is disabled. - pub fn finalized_data_availability_boundary(&self) -> Option { - self.data_availability_boundary().map(|boundary| { - std::cmp::max( - boundary, - self.canonical_head - .cached_head() - .finalized_checkpoint() - .epoch, - ) - }) + /// Returns true if the given epoch lies within the da boundary and false otherwise. + pub fn block_needs_da_check(&self, block_epoch: Epoch) -> bool { + self.data_availability_boundary() + .map_or(false, |da_epoch| block_epoch >= da_epoch) } /// Returns `true` if we are at or past the `Eip4844` fork. This will always return `false` if diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index e2288dbb8..9d1a7c708 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -2,16 +2,20 @@ use derivative::Derivative; use slot_clock::SlotClock; use std::sync::Arc; -use crate::beacon_chain::{BeaconChain, BeaconChainTypes, MAXIMUM_GOSSIP_CLOCK_DISPARITY}; -use crate::BeaconChainError; -use state_processing::per_block_processing::eip4844::eip4844::verify_kzg_commitments_against_transactions; -use types::signed_beacon_block::BlobReconstructionError; -use types::{ - BeaconBlockRef, BeaconStateError, BlobsSidecar, EthSpec, Hash256, KzgCommitment, - SignedBeaconBlock, SignedBeaconBlockAndBlobsSidecar, SignedBeaconBlockHeader, Slot, - Transactions, +use crate::beacon_chain::{ + BeaconChain, BeaconChainTypes, MAXIMUM_GOSSIP_CLOCK_DISPARITY, + VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT, +}; +use crate::data_availability_checker::{ + AvailabilityCheckError, AvailabilityPendingBlock, AvailableBlock, +}; +use crate::kzg_utils::{validate_blob, validate_blobs}; +use crate::BeaconChainError; +use kzg::Kzg; +use types::{ + BeaconBlockRef, BeaconStateError, BlobSidecar, BlobSidecarList, Epoch, EthSpec, Hash256, + KzgCommitment, SignedBeaconBlock, SignedBeaconBlockHeader, SignedBlobSidecar, Slot, }; -use types::{Epoch, ExecPayload}; #[derive(Debug)] pub enum BlobError { @@ -62,15 +66,54 @@ pub enum BlobError { UnavailableBlobs, /// Blobs provided for a pre-Eip4844 fork. InconsistentFork, -} -impl From for BlobError { - fn from(e: BlobReconstructionError) -> Self { - match e { - BlobReconstructionError::UnavailableBlobs => BlobError::UnavailableBlobs, - BlobReconstructionError::InconsistentFork => BlobError::InconsistentFork, - } - } + /// The `blobs_sidecar.message.beacon_block_root` block is unknown. + /// + /// ## Peer scoring + /// + /// The blob points to a block we have not yet imported. The blob cannot be imported + /// into fork choice yet + UnknownHeadBlock { + beacon_block_root: Hash256, + }, + + /// The `BlobSidecar` was gossiped over an incorrect subnet. + InvalidSubnet { + expected: u64, + received: u64, + }, + + /// The sidecar corresponds to a slot older than the finalized head slot. + PastFinalizedSlot { + blob_slot: Slot, + finalized_slot: Slot, + }, + + /// The proposer index specified in the sidecar does not match the locally computed + /// proposer index. + ProposerIndexMismatch { + sidecar: usize, + local: usize, + }, + + ProposerSignatureInvalid, + + /// A sidecar with same slot, beacon_block_root and proposer_index but different blob is received for + /// the same blob index. + RepeatSidecar { + proposer: usize, + slot: Slot, + blob_index: usize, + }, + + /// The proposal_index corresponding to blob.beacon_block_root is not known. + /// + /// ## Peer scoring + /// + /// The block is invalid and the peer is faulty. + UnknownValidator(u64), + + BlobCacheError(AvailabilityCheckError), } impl From for BlobError { @@ -85,302 +128,220 @@ impl From for BlobError { } } -pub fn validate_blob_for_gossip( - block_wrapper: BlockWrapper, - block_root: Hash256, - chain: &BeaconChain, -) -> Result, BlobError> { - if let BlockWrapper::BlockAndBlob(ref block, ref blobs_sidecar) = block_wrapper { - let blob_slot = blobs_sidecar.beacon_block_slot; - // Do not gossip or process blobs from future or past slots. - let latest_permissible_slot = chain - .slot_clock - .now_with_future_tolerance(MAXIMUM_GOSSIP_CLOCK_DISPARITY) - .ok_or(BeaconChainError::UnableToReadSlot)?; - if blob_slot > latest_permissible_slot { - return Err(BlobError::FutureSlot { - message_slot: latest_permissible_slot, - latest_permissible_slot: blob_slot, - }); - } - - if blob_slot != block.slot() { - return Err(BlobError::SlotMismatch { - blob_slot, - block_slot: block.slot(), - }); - } - } - - block_wrapper.into_available_block(block_root, chain) +/// A wrapper around a `BlobSidecar` that indicates it has been approved for re-gossiping on +/// the p2p network. +#[derive(Debug)] +pub struct GossipVerifiedBlob { + blob: Arc>, } -fn verify_data_availability( - _blob_sidecar: &BlobsSidecar, - kzg_commitments: &[KzgCommitment], - transactions: &Transactions, - _block_slot: Slot, - _block_root: Hash256, +impl GossipVerifiedBlob { + pub fn block_root(&self) -> Hash256 { + self.blob.block_root + } +} + +pub fn validate_blob_sidecar_for_gossip( + signed_blob_sidecar: SignedBlobSidecar, + subnet: u64, chain: &BeaconChain, -) -> Result<(), BlobError> { - if verify_kzg_commitments_against_transactions::(transactions, kzg_commitments) - .is_err() +) -> Result, BlobError> { + let blob_slot = signed_blob_sidecar.message.slot; + let blob_index = signed_blob_sidecar.message.index; + let block_root = signed_blob_sidecar.message.block_root; + + // Verify that the blob_sidecar was received on the correct subnet. + if blob_index != subnet { + return Err(BlobError::InvalidSubnet { + expected: blob_index, + received: subnet, + }); + } + + // Verify that the sidecar is not from a future slot. + let latest_permissible_slot = chain + .slot_clock + .now_with_future_tolerance(MAXIMUM_GOSSIP_CLOCK_DISPARITY) + .ok_or(BeaconChainError::UnableToReadSlot)?; + if blob_slot > latest_permissible_slot { + return Err(BlobError::FutureSlot { + message_slot: blob_slot, + latest_permissible_slot, + }); + } + + // TODO(pawan): Verify not from a past slot? + + // Verify that the sidecar slot is greater than the latest finalized slot + let latest_finalized_slot = chain + .head() + .finalized_checkpoint() + .epoch + .start_slot(T::EthSpec::slots_per_epoch()); + if blob_slot <= latest_finalized_slot { + return Err(BlobError::PastFinalizedSlot { + blob_slot, + finalized_slot: latest_finalized_slot, + }); + } + + // TODO(pawan): should we verify locally that the parent root is correct + // or just use whatever the proposer gives us? + let proposer_shuffling_root = signed_blob_sidecar.message.block_parent_root; + + let (proposer_index, fork) = match chain + .beacon_proposer_cache + .lock() + .get_slot::(proposer_shuffling_root, blob_slot) { - return Err(BlobError::TransactionCommitmentMismatch); - } - - // Validatate that the kzg proof is valid against the commitments and blobs - let _kzg = chain - .kzg - .as_ref() - .ok_or(BlobError::TrustedSetupNotInitialized)?; - - todo!("use `kzg_utils::validate_blobs` once the function is updated") - // if !kzg_utils::validate_blobs_sidecar( - // kzg, - // block_slot, - // block_root, - // kzg_commitments, - // blob_sidecar, - // ) - // .map_err(BlobError::KzgError)? - // { - // return Err(BlobError::InvalidKzgProof); - // } - // Ok(()) -} - -/// A wrapper over a [`SignedBeaconBlock`] or a [`SignedBeaconBlockAndBlobsSidecar`]. This makes no -/// claims about data availability and should not be used in consensus. This struct is useful in -/// networking when we want to send blocks around without consensus checks. -#[derive(Clone, Debug, Derivative)] -#[derivative(PartialEq, Hash(bound = "E: EthSpec"))] -pub enum BlockWrapper { - Block(Arc>), - BlockAndBlob(Arc>, Arc>), -} - -impl BlockWrapper { - pub fn new( - block: Arc>, - blobs_sidecar: Option>>, - ) -> Self { - if let Some(blobs_sidecar) = blobs_sidecar { - BlockWrapper::BlockAndBlob(block, blobs_sidecar) - } else { - BlockWrapper::Block(block) + Some(proposer) => (proposer.index, proposer.fork), + None => { + let state = &chain.canonical_head.cached_head().snapshot.beacon_state; + ( + state.get_beacon_proposer_index(blob_slot, &chain.spec)?, + state.fork(), + ) } + }; + + let blob_proposer_index = signed_blob_sidecar.message.proposer_index; + if proposer_index != blob_proposer_index as usize { + return Err(BlobError::ProposerIndexMismatch { + sidecar: blob_proposer_index as usize, + local: proposer_index, + }); + } + + let signature_is_valid = { + let pubkey_cache = chain + .validator_pubkey_cache + .try_read_for(VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT) + .ok_or(BeaconChainError::ValidatorPubkeyCacheLockTimeout) + .map_err(BlobError::BeaconChainError)?; + + let pubkey = pubkey_cache + .get(proposer_index) + .ok_or_else(|| BlobError::UnknownValidator(proposer_index as u64))?; + + signed_blob_sidecar.verify_signature( + None, + pubkey, + &fork, + chain.genesis_validators_root, + &chain.spec, + ) + }; + + if !signature_is_valid { + return Err(BlobError::ProposerSignatureInvalid); + } + + // TODO(pawan): kzg validations. + + // TODO(pawan): Check if other blobs for the same proposer index and blob index have been + // received and drop if required. + + // Verify if the corresponding block for this blob has been received. + // Note: this should be the last gossip check so that we can forward the blob + // over the gossip network even if we haven't received the corresponding block yet + // as all other validations have passed. + let block_opt = chain + .canonical_head + .fork_choice_read_lock() + .get_block(&block_root) + .or_else(|| chain.early_attester_cache.get_proto_block(block_root)); // TODO(pawan): should we be checking this cache? + + // TODO(pawan): this may be redundant with the new `AvailabilityProcessingStatus::PendingBlock variant` + if block_opt.is_none() { + return Err(BlobError::UnknownHeadBlock { + beacon_block_root: block_root, + }); + } + + Ok(GossipVerifiedBlob { + blob: signed_blob_sidecar.message, + }) +} + +#[derive(Debug, Clone)] +pub struct KzgVerifiedBlob { + blob: Arc>, +} + +impl KzgVerifiedBlob { + pub fn to_blob(self) -> Arc> { + self.blob + } + pub fn as_blob(&self) -> &BlobSidecar { + &self.blob + } + pub fn clone_blob(&self) -> Arc> { + self.blob.clone() + } + pub fn kzg_commitment(&self) -> KzgCommitment { + self.blob.kzg_commitment + } + pub fn block_root(&self) -> Hash256 { + self.blob.block_root } } -impl From> for BlockWrapper { - fn from(block: SignedBeaconBlock) -> Self { - BlockWrapper::Block(Arc::new(block)) +pub fn verify_kzg_for_blob( + blob: GossipVerifiedBlob, + kzg: &Kzg, +) -> Result, AvailabilityCheckError> { + //TODO(sean) remove clone + if validate_blob::( + kzg, + blob.blob.blob.clone(), + blob.blob.kzg_commitment, + blob.blob.kzg_proof, + ) + .map_err(AvailabilityCheckError::Kzg)? + { + Ok(KzgVerifiedBlob { blob: blob.blob }) + } else { + Err(AvailabilityCheckError::KzgVerificationFailed) } } -impl From> for BlockWrapper { - fn from(block: SignedBeaconBlockAndBlobsSidecar) -> Self { - let SignedBeaconBlockAndBlobsSidecar { - beacon_block, - blobs_sidecar, - } = block; - BlockWrapper::BlockAndBlob(beacon_block, blobs_sidecar) +pub fn verify_kzg_for_blob_list( + blob_list: BlobSidecarList, + kzg: &Kzg, +) -> Result, AvailabilityCheckError> { + let (blobs, (commitments, proofs)): (Vec<_>, (Vec<_>, Vec<_>)) = blob_list + .clone() + .into_iter() + //TODO(sean) remove clone + .map(|blob| (blob.blob.clone(), (blob.kzg_commitment, blob.kzg_proof))) + .unzip(); + if validate_blobs::( + kzg, + commitments.as_slice(), + blobs.as_slice(), + proofs.as_slice(), + ) + .map_err(AvailabilityCheckError::Kzg)? + { + Ok(blob_list + .into_iter() + .map(|blob| KzgVerifiedBlob { blob }) + .collect()) + } else { + Err(AvailabilityCheckError::KzgVerificationFailed) } } -impl From>> for BlockWrapper { - fn from(block: Arc>) -> Self { - BlockWrapper::Block(block) - } -} +pub type KzgVerifiedBlobList = Vec>; -#[derive(Copy, Clone)] -pub enum DataAvailabilityCheckRequired { - Yes, - No, -} - -pub trait IntoAvailableBlock { - fn into_available_block( - self, - block_root: Hash256, - chain: &BeaconChain, - ) -> Result, BlobError>; -} - -impl IntoAvailableBlock for BlockWrapper { - fn into_available_block( - self, - block_root: Hash256, - chain: &BeaconChain, - ) -> Result, BlobError> { - let data_availability_boundary = chain.data_availability_boundary(); - let da_check_required = - data_availability_boundary.map_or(DataAvailabilityCheckRequired::No, |boundary| { - if self.slot().epoch(T::EthSpec::slots_per_epoch()) >= boundary { - DataAvailabilityCheckRequired::Yes - } else { - DataAvailabilityCheckRequired::No - } - }); - match self { - BlockWrapper::Block(block) => AvailableBlock::new(block, block_root, da_check_required), - BlockWrapper::BlockAndBlob(block, blobs_sidecar) => { - if matches!(da_check_required, DataAvailabilityCheckRequired::Yes) { - let kzg_commitments = block - .message() - .body() - .blob_kzg_commitments() - .map_err(|_| BlobError::KzgCommitmentMissing)?; - let transactions = block - .message() - .body() - .execution_payload_eip4844() - .map(|payload| payload.transactions()) - .map_err(|_| BlobError::TransactionsMissing)? - .ok_or(BlobError::TransactionsMissing)?; - verify_data_availability( - &blobs_sidecar, - kzg_commitments, - transactions, - block.slot(), - block_root, - chain, - )?; - } - - AvailableBlock::new_with_blobs(block, blobs_sidecar, da_check_required) - } - } - } -} - -/// A wrapper over a [`SignedBeaconBlock`] or a [`SignedBeaconBlockAndBlobsSidecar`]. An -/// `AvailableBlock` has passed any required data availability checks and should be used in -/// consensus. This newtype wraps `AvailableBlockInner` to ensure data availability checks -/// cannot be circumvented on construction. -#[derive(Clone, Debug, Derivative)] -#[derivative(PartialEq, Hash(bound = "E: EthSpec"))] -pub struct AvailableBlock(AvailableBlockInner); - -/// A wrapper over a [`SignedBeaconBlock`] or a [`SignedBeaconBlockAndBlobsSidecar`]. -#[derive(Clone, Debug, Derivative)] -#[derivative(PartialEq, Hash(bound = "E: EthSpec"))] -enum AvailableBlockInner { - Block(Arc>), - BlockAndBlob(SignedBeaconBlockAndBlobsSidecar), -} - -impl AvailableBlock { - pub fn new( - beacon_block: Arc>, - block_root: Hash256, - da_check_required: DataAvailabilityCheckRequired, - ) -> Result { - match beacon_block.as_ref() { - // No data availability check required prior to Eip4844. - SignedBeaconBlock::Base(_) - | SignedBeaconBlock::Altair(_) - | SignedBeaconBlock::Capella(_) - | SignedBeaconBlock::Merge(_) => { - Ok(AvailableBlock(AvailableBlockInner::Block(beacon_block))) - } - SignedBeaconBlock::Eip4844(_) => { - match da_check_required { - DataAvailabilityCheckRequired::Yes => { - // Attempt to reconstruct empty blobs here. - let blobs_sidecar = beacon_block - .reconstruct_empty_blobs(Some(block_root)) - .map(Arc::new)?; - Ok(AvailableBlock(AvailableBlockInner::BlockAndBlob( - SignedBeaconBlockAndBlobsSidecar { - beacon_block, - blobs_sidecar, - }, - ))) - } - DataAvailabilityCheckRequired::No => { - Ok(AvailableBlock(AvailableBlockInner::Block(beacon_block))) - } - } - } - } - } - - /// This function is private because an `AvailableBlock` should be - /// constructed via the `into_available_block` method. - fn new_with_blobs( - beacon_block: Arc>, - blobs_sidecar: Arc>, - da_check_required: DataAvailabilityCheckRequired, - ) -> Result { - match beacon_block.as_ref() { - // This method shouldn't be called with a pre-Eip4844 block. - SignedBeaconBlock::Base(_) - | SignedBeaconBlock::Altair(_) - | SignedBeaconBlock::Capella(_) - | SignedBeaconBlock::Merge(_) => Err(BlobError::InconsistentFork), - SignedBeaconBlock::Eip4844(_) => { - match da_check_required { - DataAvailabilityCheckRequired::Yes => Ok(AvailableBlock( - AvailableBlockInner::BlockAndBlob(SignedBeaconBlockAndBlobsSidecar { - beacon_block, - blobs_sidecar, - }), - )), - DataAvailabilityCheckRequired::No => { - // Blobs were not verified so we drop them, we'll instead just pass around - // an available `Eip4844` block without blobs. - Ok(AvailableBlock(AvailableBlockInner::Block(beacon_block))) - } - } - } - } - } - - pub fn blobs(&self) -> Option>> { - match &self.0 { - AvailableBlockInner::Block(_) => None, - AvailableBlockInner::BlockAndBlob(block_sidecar_pair) => { - Some(block_sidecar_pair.blobs_sidecar.clone()) - } - } - } - - pub fn deconstruct(self) -> (Arc>, Option>>) { - match self.0 { - AvailableBlockInner::Block(block) => (block, None), - AvailableBlockInner::BlockAndBlob(block_sidecar_pair) => { - let SignedBeaconBlockAndBlobsSidecar { - beacon_block, - blobs_sidecar, - } = block_sidecar_pair; - (beacon_block, Some(blobs_sidecar)) - } - } - } -} - -pub trait IntoBlockWrapper: AsBlock { - fn into_block_wrapper(self) -> BlockWrapper; -} - -impl IntoBlockWrapper for BlockWrapper { - fn into_block_wrapper(self) -> BlockWrapper { - self - } -} - -impl IntoBlockWrapper for AvailableBlock { - fn into_block_wrapper(self) -> BlockWrapper { - let (block, blobs) = self.deconstruct(); - if let Some(blobs) = blobs { - BlockWrapper::BlockAndBlob(block, blobs) - } else { - BlockWrapper::Block(block) - } - } +#[derive(Debug, Clone)] +pub enum MaybeAvailableBlock { + /// This variant is fully available. + /// i.e. for pre-eip4844 blocks, it contains a (`SignedBeaconBlock`, `Blobs::None`) and for + /// post-4844 blocks, it contains a `SignedBeaconBlock` and a Blobs variant other than `Blobs::None`. + Available(AvailableBlock), + /// This variant is not fully available and requires blobs to become fully available. + AvailabilityPending(AvailabilityPendingBlock), } pub trait AsBlock { @@ -393,193 +354,149 @@ pub trait AsBlock { fn as_block(&self) -> &SignedBeaconBlock; fn block_cloned(&self) -> Arc>; fn canonical_root(&self) -> Hash256; + fn into_block_wrapper(self) -> BlockWrapper; +} + +impl AsBlock for MaybeAvailableBlock { + fn slot(&self) -> Slot { + self.as_block().slot() + } + fn epoch(&self) -> Epoch { + self.as_block().epoch() + } + fn parent_root(&self) -> Hash256 { + self.as_block().parent_root() + } + fn state_root(&self) -> Hash256 { + self.as_block().state_root() + } + fn signed_block_header(&self) -> SignedBeaconBlockHeader { + self.as_block().signed_block_header() + } + fn message(&self) -> BeaconBlockRef { + self.as_block().message() + } + fn as_block(&self) -> &SignedBeaconBlock { + match &self { + MaybeAvailableBlock::Available(block) => block.as_block(), + MaybeAvailableBlock::AvailabilityPending(block) => block.as_block(), + } + } + fn block_cloned(&self) -> Arc> { + match &self { + MaybeAvailableBlock::Available(block) => block.block_cloned(), + MaybeAvailableBlock::AvailabilityPending(block) => block.block_cloned(), + } + } + fn canonical_root(&self) -> Hash256 { + self.as_block().canonical_root() + } + + fn into_block_wrapper(self) -> BlockWrapper { + match self { + MaybeAvailableBlock::Available(available_block) => available_block.into_block_wrapper(), + MaybeAvailableBlock::AvailabilityPending(pending_block) => { + BlockWrapper::Block(pending_block.to_block()) + } + } + } +} + +impl AsBlock for &MaybeAvailableBlock { + fn slot(&self) -> Slot { + self.as_block().slot() + } + fn epoch(&self) -> Epoch { + self.as_block().epoch() + } + fn parent_root(&self) -> Hash256 { + self.as_block().parent_root() + } + fn state_root(&self) -> Hash256 { + self.as_block().state_root() + } + fn signed_block_header(&self) -> SignedBeaconBlockHeader { + self.as_block().signed_block_header() + } + fn message(&self) -> BeaconBlockRef { + self.as_block().message() + } + fn as_block(&self) -> &SignedBeaconBlock { + match &self { + MaybeAvailableBlock::Available(block) => block.as_block(), + MaybeAvailableBlock::AvailabilityPending(block) => block.as_block(), + } + } + fn block_cloned(&self) -> Arc> { + match &self { + MaybeAvailableBlock::Available(block) => block.block_cloned(), + MaybeAvailableBlock::AvailabilityPending(block) => block.block_cloned(), + } + } + fn canonical_root(&self) -> Hash256 { + self.as_block().canonical_root() + } + + fn into_block_wrapper(self) -> BlockWrapper { + self.clone().into_block_wrapper() + } +} + +#[derive(Debug, Clone, Derivative)] +#[derivative(Hash(bound = "E: EthSpec"))] +pub enum BlockWrapper { + Block(Arc>), + BlockAndBlobs(Arc>, BlobSidecarList), } impl AsBlock for BlockWrapper { fn slot(&self) -> Slot { - match self { - BlockWrapper::Block(block) => block.slot(), - BlockWrapper::BlockAndBlob(block, _) => block.slot(), - } + self.as_block().slot() } fn epoch(&self) -> Epoch { - match self { - BlockWrapper::Block(block) => block.epoch(), - BlockWrapper::BlockAndBlob(block, _) => block.epoch(), - } + self.as_block().epoch() } fn parent_root(&self) -> Hash256 { - match self { - BlockWrapper::Block(block) => block.parent_root(), - BlockWrapper::BlockAndBlob(block, _) => block.parent_root(), - } + self.as_block().parent_root() } fn state_root(&self) -> Hash256 { - match self { - BlockWrapper::Block(block) => block.state_root(), - BlockWrapper::BlockAndBlob(block, _) => block.state_root(), - } + self.as_block().state_root() } fn signed_block_header(&self) -> SignedBeaconBlockHeader { - match &self { - BlockWrapper::Block(block) => block.signed_block_header(), - BlockWrapper::BlockAndBlob(block, _) => block.signed_block_header(), - } + self.as_block().signed_block_header() } fn message(&self) -> BeaconBlockRef { - match &self { - BlockWrapper::Block(block) => block.message(), - BlockWrapper::BlockAndBlob(block, _) => block.message(), - } + self.as_block().message() } fn as_block(&self) -> &SignedBeaconBlock { match &self { BlockWrapper::Block(block) => block, - BlockWrapper::BlockAndBlob(block, _) => block, + BlockWrapper::BlockAndBlobs(block, _) => block, } } fn block_cloned(&self) -> Arc> { match &self { BlockWrapper::Block(block) => block.clone(), - BlockWrapper::BlockAndBlob(block, _) => block.clone(), + BlockWrapper::BlockAndBlobs(block, _) => block.clone(), } } fn canonical_root(&self) -> Hash256 { - match &self { - BlockWrapper::Block(block) => block.canonical_root(), - BlockWrapper::BlockAndBlob(block, _) => block.canonical_root(), - } + self.as_block().canonical_root() + } + + fn into_block_wrapper(self) -> BlockWrapper { + self } } -impl AsBlock for &BlockWrapper { - fn slot(&self) -> Slot { - match self { - BlockWrapper::Block(block) => block.slot(), - BlockWrapper::BlockAndBlob(block, _) => block.slot(), - } - } - fn epoch(&self) -> Epoch { - match self { - BlockWrapper::Block(block) => block.epoch(), - BlockWrapper::BlockAndBlob(block, _) => block.epoch(), - } - } - fn parent_root(&self) -> Hash256 { - match self { - BlockWrapper::Block(block) => block.parent_root(), - BlockWrapper::BlockAndBlob(block, _) => block.parent_root(), - } - } - fn state_root(&self) -> Hash256 { - match self { - BlockWrapper::Block(block) => block.state_root(), - BlockWrapper::BlockAndBlob(block, _) => block.state_root(), - } - } - fn signed_block_header(&self) -> SignedBeaconBlockHeader { - match &self { - BlockWrapper::Block(block) => block.signed_block_header(), - BlockWrapper::BlockAndBlob(block, _) => block.signed_block_header(), - } - } - fn message(&self) -> BeaconBlockRef { - match &self { - BlockWrapper::Block(block) => block.message(), - BlockWrapper::BlockAndBlob(block, _) => block.message(), - } - } - fn as_block(&self) -> &SignedBeaconBlock { - match &self { - BlockWrapper::Block(block) => block, - BlockWrapper::BlockAndBlob(block, _) => block, - } - } - fn block_cloned(&self) -> Arc> { - match &self { - BlockWrapper::Block(block) => block.clone(), - BlockWrapper::BlockAndBlob(block, _) => block.clone(), - } - } - fn canonical_root(&self) -> Hash256 { - match &self { - BlockWrapper::Block(block) => block.canonical_root(), - BlockWrapper::BlockAndBlob(block, _) => block.canonical_root(), - } +impl From>> for BlockWrapper { + fn from(value: Arc>) -> Self { + Self::Block(value) } } -impl AsBlock for AvailableBlock { - fn slot(&self) -> Slot { - match &self.0 { - AvailableBlockInner::Block(block) => block.slot(), - AvailableBlockInner::BlockAndBlob(block_sidecar_pair) => { - block_sidecar_pair.beacon_block.slot() - } - } - } - fn epoch(&self) -> Epoch { - match &self.0 { - AvailableBlockInner::Block(block) => block.epoch(), - AvailableBlockInner::BlockAndBlob(block_sidecar_pair) => { - block_sidecar_pair.beacon_block.epoch() - } - } - } - fn parent_root(&self) -> Hash256 { - match &self.0 { - AvailableBlockInner::Block(block) => block.parent_root(), - AvailableBlockInner::BlockAndBlob(block_sidecar_pair) => { - block_sidecar_pair.beacon_block.parent_root() - } - } - } - fn state_root(&self) -> Hash256 { - match &self.0 { - AvailableBlockInner::Block(block) => block.state_root(), - AvailableBlockInner::BlockAndBlob(block_sidecar_pair) => { - block_sidecar_pair.beacon_block.state_root() - } - } - } - fn signed_block_header(&self) -> SignedBeaconBlockHeader { - match &self.0 { - AvailableBlockInner::Block(block) => block.signed_block_header(), - AvailableBlockInner::BlockAndBlob(block_sidecar_pair) => { - block_sidecar_pair.beacon_block.signed_block_header() - } - } - } - fn message(&self) -> BeaconBlockRef { - match &self.0 { - AvailableBlockInner::Block(block) => block.message(), - AvailableBlockInner::BlockAndBlob(block_sidecar_pair) => { - block_sidecar_pair.beacon_block.message() - } - } - } - fn as_block(&self) -> &SignedBeaconBlock { - match &self.0 { - AvailableBlockInner::Block(block) => block, - AvailableBlockInner::BlockAndBlob(block_sidecar_pair) => { - &block_sidecar_pair.beacon_block - } - } - } - fn block_cloned(&self) -> Arc> { - match &self.0 { - AvailableBlockInner::Block(block) => block.clone(), - AvailableBlockInner::BlockAndBlob(block_sidecar_pair) => { - block_sidecar_pair.beacon_block.clone() - } - } - } - fn canonical_root(&self) -> Hash256 { - match &self.0 { - AvailableBlockInner::Block(block) => block.canonical_root(), - AvailableBlockInner::BlockAndBlob(block_sidecar_pair) => { - block_sidecar_pair.beacon_block.canonical_root() - } - } +impl From> for BlockWrapper { + fn from(value: SignedBeaconBlock) -> Self { + Self::Block(Arc::new(value)) } } diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index a1f7abf06..4cadc86bd 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -24,9 +24,6 @@ //! â–¼ //! SignedBeaconBlock //! | -//! â–¼ -//! AvailableBlock -//! | //! |--------------- //! | | //! | â–¼ @@ -51,9 +48,9 @@ // returned alongside. #![allow(clippy::result_large_err)] -use crate::blob_verification::{ - validate_blob_for_gossip, AsBlock, AvailableBlock, BlobError, BlockWrapper, IntoAvailableBlock, - IntoBlockWrapper, +use crate::blob_verification::{AsBlock, BlobError, BlockWrapper, MaybeAvailableBlock}; +use crate::data_availability_checker::{ + AvailabilityCheckError, AvailabilityPendingBlock, AvailableBlock, }; use crate::eth1_finalization_cache::Eth1FinalizationData; use crate::execution_payload::{ @@ -96,7 +93,6 @@ use std::time::Duration; use store::{Error as DBError, HotStateSummary, KeyValueStore, StoreOp}; use task_executor::JoinHandle; use tree_hash::TreeHash; -use types::signed_beacon_block::BlobReconstructionError; use types::ExecPayload; use types::{ BeaconBlockRef, BeaconState, BeaconStateError, BlindedPayload, ChainSpec, CloneConfig, Epoch, @@ -314,6 +310,7 @@ pub enum BlockError { parent_root: Hash256, }, BlobValidation(BlobError), + AvailabilityCheck(AvailabilityCheckError), } impl From for BlockError { @@ -322,6 +319,12 @@ impl From for BlockError { } } +impl From for BlockError { + fn from(e: AvailabilityCheckError) -> Self { + Self::AvailabilityCheck(e) + } +} + /// Returned when block validation failed due to some issue verifying /// the execution payload. #[derive(Debug)] @@ -494,13 +497,8 @@ impl From for BlockError { } } -impl From for BlockError { - fn from(e: BlobReconstructionError) -> Self { - BlockError::BlobValidation(BlobError::from(e)) - } -} - /// Stores information about verifying a payload against an execution engine. +#[derive(Clone)] pub struct PayloadVerificationOutcome { pub payload_verification_status: PayloadVerificationStatus, pub is_valid_merge_transition_block: bool, @@ -605,14 +603,14 @@ pub fn signature_verify_chain_segment( signature_verifier.include_all_signatures(block.as_block(), &mut consensus_context)?; - //FIXME(sean) batch kzg verification - let available_block = block.clone().into_available_block(*block_root, chain)?; - consensus_context = consensus_context.set_kzg_commitments_consistent(true); + let maybe_available_block = chain + .data_availability_checker + .check_availability(block.clone())?; // Save the block and its consensus context. The context will have had its proposer index // and attesting indices filled in, which can be used to accelerate later block processing. signature_verified_blocks.push(SignatureVerifiedBlock { - block: available_block, + block: maybe_available_block, block_root: *block_root, parent: None, consensus_context, @@ -637,7 +635,7 @@ pub fn signature_verify_chain_segment( #[derive(Derivative)] #[derivative(Debug(bound = "T: BeaconChainTypes"))] pub struct GossipVerifiedBlock { - pub block: AvailableBlock, + pub block: MaybeAvailableBlock, pub block_root: Hash256, parent: Option>, consensus_context: ConsensusContext, @@ -646,7 +644,7 @@ pub struct GossipVerifiedBlock { /// A wrapper around a `SignedBeaconBlock` that indicates that all signatures (except the deposit /// signatures) have been verified. pub struct SignatureVerifiedBlock { - block: AvailableBlock, + block: MaybeAvailableBlock, block_root: Hash256, parent: Option>, consensus_context: ConsensusContext, @@ -669,14 +667,103 @@ type PayloadVerificationHandle = /// due to finality or some other event. A `ExecutionPendingBlock` should be imported into the /// `BeaconChain` immediately after it is instantiated. pub struct ExecutionPendingBlock { - pub block: AvailableBlock, + pub block: MaybeAvailableBlock, + pub import_data: BlockImportData, + pub payload_verification_handle: PayloadVerificationHandle, +} + +pub enum ExecutedBlock { + Available(AvailableExecutedBlock), + AvailabilityPending(AvailabilityPendingExecutedBlock), +} + +impl ExecutedBlock { + pub fn as_block(&self) -> &SignedBeaconBlock { + match self { + Self::Available(available) => available.block.block(), + Self::AvailabilityPending(pending) => pending.block.as_block(), + } + } +} + +impl std::fmt::Debug for ExecutedBlock { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self.as_block()) + } +} + +impl ExecutedBlock { + pub fn new( + block: MaybeAvailableBlock, + import_data: BlockImportData, + payload_verification_outcome: PayloadVerificationOutcome, + ) -> Self { + match block { + MaybeAvailableBlock::Available(available_block) => { + Self::Available(AvailableExecutedBlock::new( + available_block, + import_data, + payload_verification_outcome, + )) + } + MaybeAvailableBlock::AvailabilityPending(pending_block) => { + Self::AvailabilityPending(AvailabilityPendingExecutedBlock::new( + pending_block, + import_data, + payload_verification_outcome, + )) + } + } + } +} + +pub struct AvailableExecutedBlock { + pub block: AvailableBlock, + pub import_data: BlockImportData, + pub payload_verification_outcome: PayloadVerificationOutcome, +} + +impl AvailableExecutedBlock { + pub fn new( + block: AvailableBlock, + import_data: BlockImportData, + payload_verification_outcome: PayloadVerificationOutcome, + ) -> Self { + Self { + block, + import_data, + payload_verification_outcome, + } + } +} + +pub struct AvailabilityPendingExecutedBlock { + pub block: AvailabilityPendingBlock, + pub import_data: BlockImportData, + pub payload_verification_outcome: PayloadVerificationOutcome, +} + +impl AvailabilityPendingExecutedBlock { + pub fn new( + block: AvailabilityPendingBlock, + import_data: BlockImportData, + payload_verification_outcome: PayloadVerificationOutcome, + ) -> Self { + Self { + block, + import_data, + payload_verification_outcome, + } + } +} + +pub struct BlockImportData { pub block_root: Hash256, - pub state: BeaconState, - pub parent_block: SignedBeaconBlock>, + pub state: BeaconState, + pub parent_block: SignedBeaconBlock>, pub parent_eth1_finalization_data: Eth1FinalizationData, pub confirmed_state_roots: Vec, - pub consensus_context: ConsensusContext, - pub payload_verification_handle: PayloadVerificationHandle, + pub consensus_context: ConsensusContext, } /// Implemented on types that can be converted into a `ExecutionPendingBlock`. @@ -720,19 +807,20 @@ impl GossipVerifiedBlock { block: BlockWrapper, chain: &BeaconChain, ) -> Result> { + let maybe_available = chain.data_availability_checker.check_availability(block)?; // If the block is valid for gossip we don't supply it to the slasher here because // we assume it will be transformed into a fully verified block. We *do* need to supply // it to the slasher if an error occurs, because that's the end of this block's journey, // and it could be a repeat proposal (a likely cause for slashing!). - let header = block.signed_block_header(); - Self::new_without_slasher_checks(block, chain).map_err(|e| { + let header = maybe_available.signed_block_header(); + Self::new_without_slasher_checks(maybe_available, chain).map_err(|e| { process_block_slash_info(chain, BlockSlashInfo::from_early_error(header, e)) }) } /// As for new, but doesn't pass the block to the slasher. fn new_without_slasher_checks( - block: BlockWrapper, + block: MaybeAvailableBlock, chain: &BeaconChain, ) -> Result> { // Ensure the block is the correct structure for the fork at `block.slot()`. @@ -929,16 +1017,14 @@ impl GossipVerifiedBlock { // Validate the block's execution_payload (if any). validate_execution_payload_for_gossip(&parent_block, block.message(), chain)?; - let available_block = validate_blob_for_gossip(block, block_root, chain)?; - // Having checked the proposer index and the block root we can cache them. - let consensus_context = ConsensusContext::new(available_block.slot()) + let consensus_context = ConsensusContext::new(block.slot()) .set_current_block_root(block_root) - .set_proposer_index(available_block.as_block().message().proposer_index()) + .set_proposer_index(block.as_block().message().proposer_index()) .set_kzg_commitments_consistent(true); Ok(Self { - block: available_block, + block, block_root, parent, consensus_context, @@ -978,10 +1064,11 @@ impl SignatureVerifiedBlock { /// /// Returns an error if the block is invalid, or if the block was unable to be verified. pub fn new( - block: AvailableBlock, + block: BlockWrapper, block_root: Hash256, chain: &BeaconChain, ) -> Result> { + let block = chain.data_availability_checker.check_availability(block)?; // Ensure the block is the correct structure for the fork at `block.slot()`. block .as_block() @@ -1028,7 +1115,7 @@ impl SignatureVerifiedBlock { /// As for `new` above but producing `BlockSlashInfo`. pub fn check_slashable( - block: AvailableBlock, + block: BlockWrapper, block_root: Hash256, chain: &BeaconChain, ) -> Result>> { @@ -1137,12 +1224,7 @@ impl IntoExecutionPendingBlock for Arc IntoExecutionPendingBlock for Arc IntoExecutionPendingBlock for AvailableBlock { +impl IntoExecutionPendingBlock for BlockWrapper { /// Verifies the `SignedBeaconBlock` by first transforming it into a `SignatureVerifiedBlock` /// and then using that implementation of `IntoExecutionPendingBlock` to complete verification. fn into_execution_pending_block_slashable( @@ -1182,7 +1264,7 @@ impl ExecutionPendingBlock { /// /// Returns an error if the block is invalid, or if the block was unable to be verified. pub fn from_signature_verified_components( - block: AvailableBlock, + block: MaybeAvailableBlock, block_root: Hash256, parent: PreProcessingSnapshot, mut consensus_context: ConsensusContext, @@ -1562,12 +1644,14 @@ impl ExecutionPendingBlock { Ok(Self { block, - block_root, - state, - parent_block: parent.beacon_block, - parent_eth1_finalization_data, - confirmed_state_roots, - consensus_context, + import_data: BlockImportData { + block_root, + state, + parent_block: parent.beacon_block, + parent_eth1_finalization_data, + confirmed_state_roots, + consensus_context, + }, payload_verification_handle, }) } @@ -1648,7 +1732,7 @@ fn check_block_against_finalized_slot( /// Taking a lock on the `chain.canonical_head.fork_choice` might cause a deadlock here. pub fn check_block_is_finalized_checkpoint_or_descendant< T: BeaconChainTypes, - B: IntoBlockWrapper, + B: AsBlock, >( chain: &BeaconChain, fork_choice: &BeaconForkChoice, @@ -1746,8 +1830,8 @@ pub fn get_block_root(block: &SignedBeaconBlock) -> Hash256 { #[allow(clippy::type_complexity)] fn verify_parent_block_is_known( chain: &BeaconChain, - block: BlockWrapper, -) -> Result<(ProtoBlock, BlockWrapper), BlockError> { + block: MaybeAvailableBlock, +) -> Result<(ProtoBlock, MaybeAvailableBlock), BlockError> { if let Some(proto_block) = chain .canonical_head .fork_choice_read_lock() @@ -1755,7 +1839,7 @@ fn verify_parent_block_is_known( { Ok((proto_block, block)) } else { - Err(BlockError::ParentUnknown(block)) + Err(BlockError::ParentUnknown(block.into_block_wrapper())) } } @@ -1764,7 +1848,7 @@ fn verify_parent_block_is_known( /// Returns `Err(BlockError::ParentUnknown)` if the parent is not found, or if an error occurs /// whilst attempting the operation. #[allow(clippy::type_complexity)] -fn load_parent>( +fn load_parent>( block_root: Hash256, block: B, chain: &BeaconChain, diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index 3698549ed..7620a588d 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -1,5 +1,6 @@ use crate::beacon_chain::{CanonicalHead, BEACON_CHAIN_DB_KEY, ETH1_CACHE_DB_KEY, OP_POOL_DB_KEY}; use crate::blob_cache::BlobCache; +use crate::data_availability_checker::DataAvailabilityChecker; use crate::eth1_chain::{CachingEth1Backend, SszEth1}; use crate::eth1_finalization_cache::Eth1FinalizationCache; use crate::fork_choice_signal::ForkChoiceSignalTx; @@ -641,7 +642,8 @@ where let kzg = if let Some(trusted_setup) = self.trusted_setup { let kzg = Kzg::new_from_trusted_setup(trusted_setup) .map_err(|e| format!("Failed to load trusted setup: {:?}", e))?; - Some(Arc::new(kzg)) + let kzg_arc = Arc::new(kzg); + Some(kzg_arc) } else { None }; @@ -781,14 +783,14 @@ where let shuffling_cache_size = self.chain_config.shuffling_cache_size; let beacon_chain = BeaconChain { - spec: self.spec, + spec: self.spec.clone(), config: self.chain_config, store, task_executor: self .task_executor .ok_or("Cannot build without task executor")?, store_migrator, - slot_clock, + slot_clock: slot_clock.clone(), op_pool: self.op_pool.ok_or("Cannot build without op pool")?, // TODO: allow for persisting and loading the pool from disk. naive_aggregation_pool: <_>::default(), @@ -847,7 +849,13 @@ where graffiti: self.graffiti, slasher: self.slasher.clone(), validator_monitor: RwLock::new(validator_monitor), - blob_cache: BlobCache::default(), + //TODO(sean) should we move kzg solely to the da checker? + data_availability_checker: DataAvailabilityChecker::new( + slot_clock, + kzg.clone(), + self.spec, + ), + proposal_blob_cache: BlobCache::default(), kzg, }; diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs new file mode 100644 index 000000000..42e97a974 --- /dev/null +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -0,0 +1,516 @@ +use crate::blob_verification::{ + verify_kzg_for_blob, verify_kzg_for_blob_list, AsBlock, BlockWrapper, GossipVerifiedBlob, + KzgVerifiedBlob, KzgVerifiedBlobList, MaybeAvailableBlock, +}; +use crate::block_verification::{AvailabilityPendingExecutedBlock, AvailableExecutedBlock}; + +use kzg::Error as KzgError; +use kzg::Kzg; +use parking_lot::{Mutex, RwLock}; +use slot_clock::SlotClock; +use ssz_types::{Error, VariableList}; +use state_processing::per_block_processing::eip4844::eip4844::verify_kzg_commitments_against_transactions; +use std::collections::hash_map::Entry; +use std::collections::HashMap; +use std::sync::Arc; +use types::beacon_block_body::KzgCommitments; +use types::blob_sidecar::{BlobIdentifier, BlobSidecar}; +use types::consts::eip4844::MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS; +use types::{ + BeaconBlockRef, BlobSidecarList, ChainSpec, Epoch, EthSpec, ExecPayload, FullPayload, Hash256, + SignedBeaconBlock, SignedBeaconBlockHeader, Slot, +}; + +#[derive(Debug)] +pub enum AvailabilityCheckError { + DuplicateBlob(Hash256), + Kzg(KzgError), + KzgVerificationFailed, + KzgNotInitialized, + SszTypes(ssz_types::Error), + MissingBlobs, + NumBlobsMismatch { + num_kzg_commitments: usize, + num_blobs: usize, + }, + TxKzgCommitmentMismatch, + KzgCommitmentMismatch { + blob_index: u64, + }, + Pending, + IncorrectFork, +} + +impl From for AvailabilityCheckError { + fn from(value: Error) -> Self { + Self::SszTypes(value) + } +} + +/// This cache contains +/// - blobs that have been gossip verified +/// - commitments for blocks that have been gossip verified, but the commitments themselves +/// have not been verified against blobs +/// - blocks that have been fully verified and only require a data availability check +pub struct DataAvailabilityChecker { + rpc_blob_cache: RwLock>>>, + gossip_blob_cache: Mutex>>, + slot_clock: S, + kzg: Option>, + spec: ChainSpec, +} + +struct GossipBlobCache { + verified_blobs: Vec>, + executed_block: Option>, +} + +pub enum Availability { + PendingBlobs(Vec), + PendingBlock(Hash256), + Available(Box>), +} + +impl DataAvailabilityChecker { + pub fn new(slot_clock: S, kzg: Option>, spec: ChainSpec) -> Self { + Self { + rpc_blob_cache: <_>::default(), + gossip_blob_cache: <_>::default(), + slot_clock, + kzg, + spec, + } + } + + /// This first validate the KZG commitments included in the blob sidecar. + /// Check if we've cached other blobs for this block. If it completes a set and we also + /// have a block cached, return the Availability variant triggering block import. + /// Otherwise cache the blob sidecar. + /// + /// This should only accept gossip verified blobs, so we should not have to worry about dupes. + pub fn put_gossip_blob( + &self, + verified_blob: GossipVerifiedBlob, + ) -> Result, AvailabilityCheckError> { + let block_root = verified_blob.block_root(); + + let kzg_verified_blob = if let Some(kzg) = self.kzg.as_ref() { + verify_kzg_for_blob(verified_blob, kzg)? + } else { + return Err(AvailabilityCheckError::KzgNotInitialized); + }; + + //TODO(sean) can we just use a referece to the blob here? + let blob = kzg_verified_blob.clone_blob(); + + // check if we have a block + // check if the complete set matches the block + // verify, otherwise cache + + let mut blob_cache = self.gossip_blob_cache.lock(); + + // Gossip cache. + let availability = match blob_cache.entry(blob.block_root) { + Entry::Occupied(mut occupied_entry) => { + // All blobs reaching this cache should be gossip verified and gossip verification + // should filter duplicates, as well as validate indices. + let cache = occupied_entry.get_mut(); + + cache + .verified_blobs + .insert(blob.index as usize, kzg_verified_blob); + + if let Some(executed_block) = cache.executed_block.take() { + self.check_block_availability_or_cache(cache, executed_block)? + } else { + Availability::PendingBlock(block_root) + } + } + Entry::Vacant(vacant_entry) => { + let block_root = kzg_verified_blob.block_root(); + vacant_entry.insert(GossipBlobCache { + verified_blobs: vec![kzg_verified_blob], + executed_block: None, + }); + Availability::PendingBlock(block_root) + } + }; + + drop(blob_cache); + + // RPC cache. + self.rpc_blob_cache.write().insert(blob.id(), blob.clone()); + + Ok(availability) + } + + /// Check if we have all the blobs for a block. If we do, return the Availability variant that + /// triggers import of the block. + pub fn put_pending_executed_block( + &self, + executed_block: AvailabilityPendingExecutedBlock, + ) -> Result, AvailabilityCheckError> { + let mut guard = self.gossip_blob_cache.lock(); + let entry = guard.entry(executed_block.import_data.block_root); + + let availability = match entry { + Entry::Occupied(mut occupied_entry) => { + let cache: &mut GossipBlobCache = occupied_entry.get_mut(); + + self.check_block_availability_or_cache(cache, executed_block)? + } + Entry::Vacant(vacant_entry) => { + let kzg_commitments_len = executed_block.block.kzg_commitments()?.len(); + let mut blob_ids = Vec::with_capacity(kzg_commitments_len); + for i in 0..kzg_commitments_len { + blob_ids.push(BlobIdentifier { + block_root: executed_block.import_data.block_root, + index: i as u64, + }); + } + + vacant_entry.insert(GossipBlobCache { + verified_blobs: vec![], + executed_block: Some(executed_block), + }); + + Availability::PendingBlobs(blob_ids) + } + }; + + Ok(availability) + } + + fn check_block_availability_or_cache( + &self, + cache: &mut GossipBlobCache, + executed_block: AvailabilityPendingExecutedBlock, + ) -> Result, AvailabilityCheckError> { + let AvailabilityPendingExecutedBlock { + block, + import_data, + payload_verification_outcome, + } = executed_block; + let kzg_commitments_len = block.kzg_commitments()?.len(); + let verified_commitments_len = cache.verified_blobs.len(); + if kzg_commitments_len == verified_commitments_len { + //TODO(sean) can we remove this clone + let blobs = cache.verified_blobs.clone(); + let available_block = self.make_available(block, blobs)?; + Ok(Availability::Available(Box::new( + AvailableExecutedBlock::new( + available_block, + import_data, + payload_verification_outcome, + ), + ))) + } else { + let mut missing_blobs = Vec::with_capacity(kzg_commitments_len); + for i in 0..kzg_commitments_len { + if cache.verified_blobs.get(i).is_none() { + missing_blobs.push(BlobIdentifier { + block_root: import_data.block_root, + index: i as u64, + }) + } + } + + let _ = cache + .executed_block + .insert(AvailabilityPendingExecutedBlock::new( + block, + import_data, + payload_verification_outcome, + )); + + Ok(Availability::PendingBlobs(missing_blobs)) + } + } + + /// Checks if a block is available, returns a MaybeAvailableBlock enum that may include the fully + /// available block. + pub fn check_availability( + &self, + block: BlockWrapper, + ) -> Result, AvailabilityCheckError> { + match block { + BlockWrapper::Block(block) => self.check_availability_without_blobs(block), + BlockWrapper::BlockAndBlobs(block, blob_list) => { + let kzg = self + .kzg + .as_ref() + .ok_or(AvailabilityCheckError::KzgNotInitialized)?; + let verified_blobs = verify_kzg_for_blob_list(blob_list, kzg)?; + + Ok(MaybeAvailableBlock::Available( + self.check_availability_with_blobs(block, verified_blobs)?, + )) + } + } + } + + /// Checks if a block is available, returning an error if the block is not immediately available. + /// Does not access the gossip cache. + pub fn try_check_availability( + &self, + block: BlockWrapper, + ) -> Result, AvailabilityCheckError> { + match block { + BlockWrapper::Block(block) => { + let blob_requirements = self.get_blob_requirements(&block)?; + let blobs = match blob_requirements { + BlobRequirements::EmptyBlobs => VerifiedBlobs::EmptyBlobs, + BlobRequirements::NotRequired => VerifiedBlobs::NotRequired, + BlobRequirements::PreEip4844 => VerifiedBlobs::PreEip4844, + BlobRequirements::Required => return Err(AvailabilityCheckError::MissingBlobs), + }; + Ok(AvailableBlock { block, blobs }) + } + BlockWrapper::BlockAndBlobs(_, _) => Err(AvailabilityCheckError::Pending), + } + } + + /// Verifies a block against a set of KZG verified blobs. Returns an AvailableBlock if block's + /// commitments are consistent with the provided verified blob commitments. + pub fn check_availability_with_blobs( + &self, + block: Arc>, + blobs: KzgVerifiedBlobList, + ) -> Result, AvailabilityCheckError> { + match self.check_availability_without_blobs(block)? { + MaybeAvailableBlock::Available(block) => Ok(block), + MaybeAvailableBlock::AvailabilityPending(pending_block) => { + self.make_available(pending_block, blobs) + } + } + } + + /// Verifies a block as much as possible, returning a MaybeAvailableBlock enum that may include + /// an AvailableBlock if no blobs are required. Otherwise this will return an AvailabilityPendingBlock. + pub fn check_availability_without_blobs( + &self, + block: Arc>, + ) -> Result, AvailabilityCheckError> { + let blob_requirements = self.get_blob_requirements(&block)?; + let blobs = match blob_requirements { + BlobRequirements::EmptyBlobs => VerifiedBlobs::EmptyBlobs, + BlobRequirements::NotRequired => VerifiedBlobs::NotRequired, + BlobRequirements::PreEip4844 => VerifiedBlobs::PreEip4844, + BlobRequirements::Required => { + return Ok(MaybeAvailableBlock::AvailabilityPending( + AvailabilityPendingBlock { block }, + )) + } + }; + Ok(MaybeAvailableBlock::Available(AvailableBlock { + block, + blobs, + })) + } + + /// Verifies an AvailabilityPendingBlock against a set of KZG verified blobs. + /// This does not check whether a block *should* have blobs, these checks should must have been + /// completed when producing the AvailabilityPendingBlock. + pub fn make_available( + &self, + block: AvailabilityPendingBlock, + blobs: KzgVerifiedBlobList, + ) -> Result, AvailabilityCheckError> { + let block_kzg_commitments = block.kzg_commitments()?; + if blobs.len() != block_kzg_commitments.len() { + return Err(AvailabilityCheckError::NumBlobsMismatch { + num_kzg_commitments: block_kzg_commitments.len(), + num_blobs: blobs.len(), + }); + } + + for (block_commitment, blob) in block_kzg_commitments.iter().zip(blobs.iter()) { + if *block_commitment != blob.kzg_commitment() { + return Err(AvailabilityCheckError::KzgCommitmentMismatch { + blob_index: blob.as_blob().index, + }); + } + } + + let blobs = VariableList::new(blobs.into_iter().map(|blob| blob.to_blob()).collect())?; + + Ok(AvailableBlock { + block: block.block, + blobs: VerifiedBlobs::Available(blobs), + }) + } + + /// Determines the blob requirements for a block. Answers the question: "Does this block require + /// blobs?". + fn get_blob_requirements( + &self, + block: &Arc>>, + ) -> Result { + let verified_blobs = if let (Ok(block_kzg_commitments), Ok(payload)) = ( + block.message().body().blob_kzg_commitments(), + block.message().body().execution_payload(), + ) { + if let Some(transactions) = payload.transactions() { + let verified = verify_kzg_commitments_against_transactions::( + transactions, + block_kzg_commitments, + ) + .map_err(|_| AvailabilityCheckError::TxKzgCommitmentMismatch)?; + if !verified { + return Err(AvailabilityCheckError::TxKzgCommitmentMismatch); + } + } + + if self.da_check_required(block.epoch()) { + if block_kzg_commitments.is_empty() { + BlobRequirements::EmptyBlobs + } else { + BlobRequirements::Required + } + } else { + BlobRequirements::NotRequired + } + } else { + BlobRequirements::PreEip4844 + }; + Ok(verified_blobs) + } + + /// The epoch at which we require a data availability check in block processing. + /// `None` if the `Eip4844` fork is disabled. + pub fn data_availability_boundary(&self) -> Option { + self.spec.eip4844_fork_epoch.and_then(|fork_epoch| { + self.slot_clock + .now() + .map(|slot| slot.epoch(T::slots_per_epoch())) + .map(|current_epoch| { + std::cmp::max( + fork_epoch, + current_epoch.saturating_sub(*MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS), + ) + }) + }) + } + + /// Returns true if the given epoch lies within the da boundary and false otherwise. + pub fn da_check_required(&self, block_epoch: Epoch) -> bool { + self.data_availability_boundary() + .map_or(false, |da_epoch| block_epoch >= da_epoch) + } +} + +pub enum BlobRequirements { + Required, + /// This block is from outside the data availability boundary so doesn't require + /// a data availability check. + NotRequired, + /// The block's `kzg_commitments` field is empty so it does not contain any blobs. + EmptyBlobs, + /// This is a block prior to the 4844 fork, so doesn't require any blobs + PreEip4844, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct AvailabilityPendingBlock { + block: Arc>, +} + +impl AvailabilityPendingBlock { + pub fn to_block(self) -> Arc> { + self.block + } + pub fn as_block(&self) -> &SignedBeaconBlock { + &self.block + } + pub fn block_cloned(&self) -> Arc> { + self.block.clone() + } + pub fn kzg_commitments(&self) -> Result<&KzgCommitments, AvailabilityCheckError> { + self.block + .message() + .body() + .blob_kzg_commitments() + .map_err(|_| AvailabilityCheckError::IncorrectFork) + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct AvailableBlock { + block: Arc>, + blobs: VerifiedBlobs, +} + +impl AvailableBlock { + pub fn block(&self) -> &SignedBeaconBlock { + &self.block + } + + pub fn deconstruct(self) -> (Arc>, Option>) { + match self.blobs { + VerifiedBlobs::EmptyBlobs | VerifiedBlobs::NotRequired | VerifiedBlobs::PreEip4844 => { + (self.block, None) + } + VerifiedBlobs::Available(blobs) => (self.block, Some(blobs)), + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub enum VerifiedBlobs { + /// These blobs are available. + Available(BlobSidecarList), + /// This block is from outside the data availability boundary so doesn't require + /// a data availability check. + NotRequired, + /// The block's `kzg_commitments` field is empty so it does not contain any blobs. + EmptyBlobs, + /// This is a block prior to the 4844 fork, so doesn't require any blobs + PreEip4844, +} + +impl AsBlock for AvailableBlock { + fn slot(&self) -> Slot { + self.block.slot() + } + + fn epoch(&self) -> Epoch { + self.block.epoch() + } + + fn parent_root(&self) -> Hash256 { + self.block.parent_root() + } + + fn state_root(&self) -> Hash256 { + self.block.state_root() + } + + fn signed_block_header(&self) -> SignedBeaconBlockHeader { + self.block.signed_block_header() + } + + fn message(&self) -> BeaconBlockRef { + self.block.message() + } + + fn as_block(&self) -> &SignedBeaconBlock { + &self.block + } + + fn block_cloned(&self) -> Arc> { + self.block.clone() + } + + fn canonical_root(&self) -> Hash256 { + self.block.canonical_root() + } + + fn into_block_wrapper(self) -> BlockWrapper { + let (block, blobs_opt) = self.deconstruct(); + if let Some(blobs) = blobs_opt { + BlockWrapper::BlockAndBlobs(block, blobs) + } else { + BlockWrapper::Block(block) + } + } +} diff --git a/beacon_node/beacon_chain/src/early_attester_cache.rs b/beacon_node/beacon_chain/src/early_attester_cache.rs index 5fe14c7e2..082c2242c 100644 --- a/beacon_node/beacon_chain/src/early_attester_cache.rs +++ b/beacon_node/beacon_chain/src/early_attester_cache.rs @@ -1,4 +1,4 @@ -use crate::blob_verification::AvailableBlock; +use crate::data_availability_checker::AvailableBlock; use crate::{ attester_cache::{CommitteeLengths, Error}, metrics, @@ -6,6 +6,7 @@ use crate::{ use parking_lot::RwLock; use proto_array::Block as ProtoBlock; use std::sync::Arc; +use types::blob_sidecar::BlobSidecarList; use types::*; pub struct CacheItem { @@ -21,6 +22,7 @@ pub struct CacheItem { * Values used to make the block available. */ block: Arc>, + //TODO(sean) remove this and just use the da checker?' blobs: Option>, proto_block: ProtoBlock, } diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index feafa7fd8..c76ba752a 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -16,6 +16,7 @@ pub mod builder; pub mod canonical_head; pub mod capella_readiness; pub mod chain_config; +pub mod data_availability_checker; mod early_attester_cache; mod errors; pub mod eth1_chain; @@ -54,9 +55,10 @@ pub mod validator_monitor; pub mod validator_pubkey_cache; pub use self::beacon_chain::{ - AttestationProcessingOutcome, BeaconChain, BeaconChainTypes, BeaconStore, ChainSegmentResult, - CountUnrealized, ForkChoiceError, OverrideForkchoiceUpdate, ProduceBlockVerification, - StateSkipConfig, WhenSlotSkipped, INVALID_FINALIZED_MERGE_TRANSITION_BLOCK_SHUTDOWN_REASON, + AttestationProcessingOutcome, AvailabilityProcessingStatus, BeaconChain, BeaconChainTypes, + BeaconStore, ChainSegmentResult, CountUnrealized, ForkChoiceError, OverrideForkchoiceUpdate, + ProduceBlockVerification, StateSkipConfig, WhenSlotSkipped, + INVALID_FINALIZED_MERGE_TRANSITION_BLOCK_SHUTDOWN_REASON, INVALID_JUSTIFIED_PAYLOAD_SHUTDOWN_REASON, MAXIMUM_GOSSIP_CLOCK_DISPARITY, }; pub use self::beacon_snapshot::BeaconSnapshot; @@ -66,7 +68,7 @@ pub use self::historical_blocks::HistoricalBlockError; pub use attestation_verification::Error as AttestationError; pub use beacon_fork_choice_store::{BeaconForkChoiceStore, Error as ForkChoiceStoreError}; pub use block_verification::{ - get_block_root, BlockError, ExecutionPayloadError, GossipVerifiedBlock, + get_block_root, BlockError, ExecutedBlock, ExecutionPayloadError, GossipVerifiedBlock, }; pub use canonical_head::{CachedHead, CanonicalHead, CanonicalHeadRwLock}; pub use eth1_chain::{Eth1Chain, Eth1ChainBackend}; diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index a784c6741..df2545adf 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1680,7 +1680,8 @@ where NotifyExecutionLayer::Yes, ) .await? - .into(); + .try_into() + .unwrap(); self.chain.recompute_head_at_current_slot().await; Ok(block_hash) } @@ -1698,7 +1699,8 @@ where NotifyExecutionLayer::Yes, ) .await? - .into(); + .try_into() + .unwrap(); self.chain.recompute_head_at_current_slot().await; Ok(block_hash) } diff --git a/beacon_node/beacon_chain/tests/attestation_production.rs b/beacon_node/beacon_chain/tests/attestation_production.rs index 9cfff81c3..feea5c721 100644 --- a/beacon_node/beacon_chain/tests/attestation_production.rs +++ b/beacon_node/beacon_chain/tests/attestation_production.rs @@ -1,9 +1,7 @@ #![cfg(not(debug_assertions))] -use beacon_chain::{ - blob_verification::{BlockWrapper, IntoAvailableBlock}, - test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy}, -}; +use beacon_chain::blob_verification::BlockWrapper; +use beacon_chain::test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy}; use beacon_chain::{StateSkipConfig, WhenSlotSkipped}; use lazy_static::lazy_static; use std::sync::Arc; @@ -135,6 +133,10 @@ async fn produces_attestations() { assert_eq!(data.target.root, target_root, "bad target root"); let block_wrapper: BlockWrapper = Arc::new(block.clone()).into(); + let available_block = chain + .data_availability_checker + .try_check_availability(block_wrapper) + .unwrap(); let early_attestation = { let proto_block = chain @@ -146,9 +148,7 @@ async fn produces_attestations() { .early_attester_cache .add_head_block( block_root, - block_wrapper - .into_available_block(block_root, chain) - .expect("should wrap into available block"), + available_block, proto_block, &state, &chain.spec, @@ -199,18 +199,19 @@ async fn early_attester_cache_old_request() { .get_block(&head.beacon_block_root) .unwrap(); - let block: BlockWrapper = head.beacon_block.clone().into(); + let block_wrapper: BlockWrapper = head.beacon_block.clone().into(); + let available_block = harness + .chain + .data_availability_checker + .try_check_availability(block_wrapper) + .unwrap(); - let chain = &harness.chain; harness .chain .early_attester_cache .add_head_block( head.beacon_block_root, - block - .clone() - .into_available_block(head.beacon_block_root, &chain) - .expect("should wrap into available block"), + available_block, head_proto_block, &head.beacon_state, &harness.chain.spec, diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index 8de906afd..88364d6ff 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -1,7 +1,8 @@ #![cfg(not(debug_assertions))] +use beacon_chain::blob_verification::BlockWrapper; use beacon_chain::{ - blob_verification::{AsBlock, BlockWrapper}, + blob_verification::AsBlock, test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType}, }; use beacon_chain::{BeaconSnapshot, BlockError, ChainSegmentResult, NotifyExecutionLayer}; diff --git a/beacon_node/beacon_chain/tests/payload_invalidation.rs b/beacon_node/beacon_chain/tests/payload_invalidation.rs index d0f81652b..dbb2ebfae 100644 --- a/beacon_node/beacon_chain/tests/payload_invalidation.rs +++ b/beacon_node/beacon_chain/tests/payload_invalidation.rs @@ -702,6 +702,8 @@ async fn invalidates_all_descendants() { NotifyExecutionLayer::Yes, ) .await + .unwrap() + .try_into() .unwrap(); rig.recompute_head().await; @@ -799,6 +801,8 @@ async fn switches_heads() { NotifyExecutionLayer::Yes, ) .await + .unwrap() + .try_into() .unwrap(); rig.recompute_head().await; diff --git a/beacon_node/beacon_chain/tests/tests.rs b/beacon_node/beacon_chain/tests/tests.rs index b4eabc809..f73cd0aca 100644 --- a/beacon_node/beacon_chain/tests/tests.rs +++ b/beacon_node/beacon_chain/tests/tests.rs @@ -681,19 +681,20 @@ async fn run_skip_slot_test(skip_slots: u64) { Slot::new(0) ); - assert_eq!( - harness_b - .chain - .process_block( - harness_a.chain.head_snapshot().beacon_block_root, - harness_a.chain.head_snapshot().beacon_block.clone(), - CountUnrealized::True, - NotifyExecutionLayer::Yes, - ) - .await - .unwrap(), - harness_a.chain.head_snapshot().beacon_block_root - ); + let status = harness_b + .chain + .process_block( + harness_a.chain.head_snapshot().beacon_block_root, + harness_a.chain.head_snapshot().beacon_block.clone(), + CountUnrealized::True, + NotifyExecutionLayer::Yes, + ) + .await + .unwrap(); + + let root: Hash256 = status.try_into().unwrap(); + + assert_eq!(root, harness_a.chain.head_snapshot().beacon_block_root); harness_b.chain.recompute_head_at_current_slot().await; diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index e84e8dd81..45ec48626 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -2,9 +2,10 @@ use super::*; use serde::{Deserialize, Serialize}; use strum::EnumString; use superstruct::superstruct; -use types::blobs_sidecar::KzgCommitments; +use types::beacon_block_body::KzgCommitments; +use types::blob_sidecar::Blobs; use types::{ - Blobs, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, + EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadEip4844, ExecutionPayloadMerge, FixedVector, Transactions, Unsigned, VariableList, Withdrawal, }; diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 2160c06e4..9704801d7 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -41,15 +41,16 @@ use tokio::{ }; use tokio_stream::wrappers::WatchStream; use tree_hash::TreeHash; +use types::beacon_block_body::KzgCommitments; +use types::blob_sidecar::Blobs; use types::consts::eip4844::BLOB_TX_TYPE; use types::transaction::{AccessTuple, BlobTransaction, EcdsaSignature, SignedBlobTransaction}; use types::Withdrawals; +use types::{AbstractExecPayload, BeaconStateError, ExecPayload, VersionedHash}; use types::{ - blobs_sidecar::{Blobs, KzgCommitments}, BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadEip4844, ExecutionPayloadMerge, ForkName, }; -use types::{AbstractExecPayload, BeaconStateError, ExecPayload, VersionedHash}; use types::{ ProposerPreparationData, PublicKeyBytes, Signature, SignedBeaconBlock, Slot, Transaction, Uint256, diff --git a/beacon_node/http_api/src/block_id.rs b/beacon_node/http_api/src/block_id.rs index ef7affeb7..36675f74b 100644 --- a/beacon_node/http_api/src/block_id.rs +++ b/beacon_node/http_api/src/block_id.rs @@ -218,10 +218,7 @@ impl BlockId { chain: &BeaconChain, ) -> Result, warp::Rejection> { let root = self.root(chain)?.0; - let Some(data_availability_boundary) = chain.data_availability_boundary() else { - return Err(warp_utils::reject::custom_not_found("Deneb fork disabled".into())); - }; - match chain.get_blobs(&root, data_availability_boundary) { + match chain.get_blobs(&root) { Ok(Some(blob_sidecar_list)) => Ok(blob_sidecar_list), Ok(None) => Err(warp_utils::reject::custom_not_found(format!( "No blobs with block root {} found in the store", diff --git a/beacon_node/http_api/src/build_block_contents.rs b/beacon_node/http_api/src/build_block_contents.rs index 9fbde0ce0..d40fef1d9 100644 --- a/beacon_node/http_api/src/build_block_contents.rs +++ b/beacon_node/http_api/src/build_block_contents.rs @@ -1,7 +1,7 @@ use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProductionError}; -use eth2::types::BlockContents; +use eth2::types::{BeaconBlockAndBlobSidecars, BlockContents}; use std::sync::Arc; -use types::{AbstractExecPayload, BeaconBlock, BeaconBlockAndBlobSidecars, ForkName}; +use types::{AbstractExecPayload, BeaconBlock, ForkName}; type Error = warp::reject::Rejection; @@ -16,7 +16,7 @@ pub fn build_block_contents { let block_root = &block.canonical_root(); - if let Some(blob_sidecars) = chain.blob_cache.pop(block_root) { + if let Some(blob_sidecars) = chain.proposal_blob_cache.pop(block_root) { let block_and_blobs = BeaconBlockAndBlobSidecars { block, blob_sidecars, diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index b2d423633..c470686ad 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -1,10 +1,10 @@ use crate::metrics; -use beacon_chain::blob_verification::{AsBlock, BlockWrapper, IntoAvailableBlock}; + +use beacon_chain::blob_verification::{AsBlock, BlockWrapper}; use beacon_chain::validator_monitor::{get_block_delay_ms, timestamp_now}; -use beacon_chain::{ - BeaconChain, BeaconChainTypes, BlockError, CountUnrealized, NotifyExecutionLayer, -}; -use eth2::types::SignedBlockContents; +use beacon_chain::{AvailabilityProcessingStatus, NotifyExecutionLayer}; +use beacon_chain::{BeaconChain, BeaconChainTypes, BlockError, CountUnrealized}; +use eth2::types::{SignedBlockContents, VariableList}; use execution_layer::ProvenancedPayload; use lighthouse_network::PubsubMessage; use network::NetworkMessage; @@ -70,54 +70,51 @@ pub async fn publish_block( } SignedBeaconBlock::Eip4844(_) => { crate::publish_pubsub_message(network_tx, PubsubMessage::BeaconBlock(block.clone()))?; - if let Some(blobs) = maybe_blobs { - for (blob_index, blob) in blobs.into_iter().enumerate() { + if let Some(signed_blobs) = maybe_blobs { + for (blob_index, blob) in signed_blobs.clone().into_iter().enumerate() { crate::publish_pubsub_message( network_tx, PubsubMessage::BlobSidecar(Box::new((blob_index as u64, blob))), )?; } + let blobs_vec = signed_blobs.into_iter().map(|blob| blob.message).collect(); + let blobs = VariableList::new(blobs_vec).map_err(|e| { + warp_utils::reject::custom_server_error(format!("Invalid blobs length: {e:?}")) + })?; + BlockWrapper::BlockAndBlobs(block, blobs) + } else { + block.into() } - block.into() - } - }; - - let available_block = match wrapped_block.into_available_block(block_root, &chain) { - Ok(available_block) => available_block, - Err(e) => { - let msg = format!("{:?}", e); - error!( - log, - "Invalid block provided to HTTP API"; - "reason" => &msg - ); - return Err(warp_utils::reject::broadcast_without_import(msg)); } }; + // Determine the delay after the start of the slot, register it with metrics. + let block_clone = wrapped_block.block_cloned(); + let slot = block_clone.message().slot(); + let proposer_index = block_clone.message().proposer_index(); match chain .process_block( block_root, - available_block.clone(), + wrapped_block, CountUnrealized::True, NotifyExecutionLayer::Yes, ) .await { - Ok(root) => { + Ok(AvailabilityProcessingStatus::Imported(root)) => { info!( log, "Valid block from HTTP API"; "block_delay" => ?delay, "root" => format!("{}", root), - "proposer_index" => available_block.message().proposer_index(), - "slot" => available_block.slot(), + "proposer_index" => proposer_index, + "slot" =>slot, ); // Notify the validator monitor. chain.validator_monitor.read().register_api_block( seen_timestamp, - available_block.message(), + block_clone.message(), root, &chain.slot_clock, ); @@ -133,7 +130,7 @@ pub async fn publish_block( late_block_logging( &chain, seen_timestamp, - available_block.message(), + block_clone.message(), root, "local", &log, @@ -142,12 +139,30 @@ pub async fn publish_block( Ok(()) } + Ok(AvailabilityProcessingStatus::PendingBlock(block_root)) => { + let msg = format!("Missing block with root {:?}", block_root); + error!( + log, + "Invalid block provided to HTTP API"; + "reason" => &msg + ); + Err(warp_utils::reject::broadcast_without_import(msg)) + } + Ok(AvailabilityProcessingStatus::PendingBlobs(blob_ids)) => { + let msg = format!("Missing blobs {:?}", blob_ids); + error!( + log, + "Invalid block provided to HTTP API"; + "reason" => &msg + ); + Err(warp_utils::reject::broadcast_without_import(msg)) + } Err(BlockError::BlockIsAlreadyKnown) => { info!( log, "Block from HTTP API already known"; "block" => ?block_root, - "slot" => available_block.slot(), + "slot" => slot, ); Ok(()) } diff --git a/beacon_node/lighthouse_network/src/rpc/protocol.rs b/beacon_node/lighthouse_network/src/rpc/protocol.rs index 6264d6933..bf67c9435 100644 --- a/beacon_node/lighthouse_network/src/rpc/protocol.rs +++ b/beacon_node/lighthouse_network/src/rpc/protocol.rs @@ -20,10 +20,9 @@ use tokio_util::{ codec::Framed, compat::{Compat, FuturesAsyncReadCompatExt}, }; -use types::BlobsSidecar; use types::{ BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockCapella, BeaconBlockMerge, - EmptyBlock, EthSpec, ForkContext, ForkName, Hash256, MainnetEthSpec, Signature, + BlobSidecar, EmptyBlock, EthSpec, ForkContext, ForkName, Hash256, MainnetEthSpec, Signature, SignedBeaconBlock, }; @@ -115,12 +114,12 @@ lazy_static! { .as_ssz_bytes() .len(); - pub static ref BLOBS_SIDECAR_MIN: usize = BlobsSidecar::::empty().as_ssz_bytes().len(); - pub static ref BLOBS_SIDECAR_MAX: usize = BlobsSidecar::::max_size(); + pub static ref BLOB_SIDECAR_MIN: usize = BlobSidecar::::empty().as_ssz_bytes().len(); + pub static ref BLOB_SIDECAR_MAX: usize = BlobSidecar::::max_size(); //FIXME(sean) these are underestimates - pub static ref SIGNED_BLOCK_AND_BLOBS_MIN: usize = *BLOBS_SIDECAR_MIN + *SIGNED_BEACON_BLOCK_BASE_MIN; - pub static ref SIGNED_BLOCK_AND_BLOBS_MAX: usize =*BLOBS_SIDECAR_MAX + *SIGNED_BEACON_BLOCK_EIP4844_MAX; + pub static ref SIGNED_BLOCK_AND_BLOBS_MIN: usize = *BLOB_SIDECAR_MIN + *SIGNED_BEACON_BLOCK_BASE_MIN; + pub static ref SIGNED_BLOCK_AND_BLOBS_MAX: usize =*BLOB_SIDECAR_MAX + *SIGNED_BEACON_BLOCK_EIP4844_MAX; } /// The maximum bytes that can be sent across the RPC pre-merge. @@ -385,7 +384,7 @@ impl ProtocolId { Protocol::Goodbye => RpcLimits::new(0, 0), // Goodbye request has no response Protocol::BlocksByRange => rpc_block_limits_by_fork(fork_context.current_fork()), Protocol::BlocksByRoot => rpc_block_limits_by_fork(fork_context.current_fork()), - Protocol::BlobsByRange => RpcLimits::new(*BLOBS_SIDECAR_MIN, *BLOBS_SIDECAR_MAX), + Protocol::BlobsByRange => RpcLimits::new(*BLOB_SIDECAR_MIN, *BLOB_SIDECAR_MAX), Protocol::BlobsByRoot => { // TODO: wrong too RpcLimits::new(*SIGNED_BLOCK_AND_BLOBS_MIN, *SIGNED_BLOCK_AND_BLOBS_MAX) diff --git a/beacon_node/network/Cargo.toml b/beacon_node/network/Cargo.toml index d068a2007..ea415c005 100644 --- a/beacon_node/network/Cargo.toml +++ b/beacon_node/network/Cargo.toml @@ -41,7 +41,7 @@ num_cpus = "1.13.0" lru_cache = { path = "../../common/lru_cache" } if-addrs = "0.6.4" strum = "0.24.0" -tokio-util = { version = "0.6.3", features = ["time"] } +tokio-util = { version = "0.7.7", features = ["time"] } derivative = "2.2.0" delay_map = "0.3.0" ethereum-types = { version = "0.14.1", optional = true } diff --git a/beacon_node/network/src/beacon_processor/mod.rs b/beacon_node/network/src/beacon_processor/mod.rs index 410389b11..f58f28135 100644 --- a/beacon_node/network/src/beacon_processor/mod.rs +++ b/beacon_node/network/src/beacon_processor/mod.rs @@ -449,7 +449,7 @@ impl WorkEvent { peer_id: PeerId, peer_client: Client, blob_index: u64, - signed_blob: Arc>, + signed_blob: SignedBlobSidecar, seen_timestamp: Duration, ) -> Self { Self { @@ -459,7 +459,7 @@ impl WorkEvent { peer_id, peer_client, blob_index, - signed_blob, + signed_blob: Box::new(signed_blob), seen_timestamp, }, } @@ -729,7 +729,7 @@ impl WorkEvent { impl std::convert::From> for WorkEvent { fn from(ready_work: ReadyWork) -> Self { match ready_work { - ReadyWork::Block(QueuedGossipBlock { + ReadyWork::GossipBlock(QueuedGossipBlock { peer_id, block, seen_timestamp, @@ -864,7 +864,7 @@ pub enum Work { peer_id: PeerId, peer_client: Client, blob_index: u64, - signed_blob: Arc>, + signed_blob: Box>, seen_timestamp: Duration, }, DelayedImportBlock { @@ -1759,7 +1759,7 @@ impl BeaconProcessor { peer_id, peer_client, blob_index, - signed_blob, + *signed_blob, seen_timestamp, ) .await diff --git a/beacon_node/network/src/beacon_processor/work_reprocessing_queue.rs b/beacon_node/network/src/beacon_processor/work_reprocessing_queue.rs index 0e4a08f5d..4a5653079 100644 --- a/beacon_node/network/src/beacon_processor/work_reprocessing_queue.rs +++ b/beacon_node/network/src/beacon_processor/work_reprocessing_queue.rs @@ -13,14 +13,15 @@ use super::MAX_SCHEDULED_WORK_QUEUE_LEN; use crate::metrics; use crate::sync::manager::BlockProcessType; -use beacon_chain::blob_verification::{AsBlock, BlockWrapper}; +use beacon_chain::blob_verification::AsBlock; +use beacon_chain::blob_verification::BlockWrapper; use beacon_chain::{BeaconChainTypes, GossipVerifiedBlock, MAXIMUM_GOSSIP_CLOCK_DISPARITY}; use fnv::FnvHashMap; use futures::task::Poll; use futures::{Stream, StreamExt}; use lighthouse_network::{MessageId, PeerId}; use logging::TimeLatch; -use slog::{crit, debug, error, trace, warn, Logger}; +use slog::{debug, error, trace, warn, Logger}; use slot_clock::SlotClock; use std::collections::{HashMap, HashSet}; use std::pin::Pin; @@ -28,7 +29,6 @@ use std::task::Context; use std::time::Duration; use task_executor::TaskExecutor; use tokio::sync::mpsc::{self, Receiver, Sender}; -use tokio::time::error::Error as TimeError; use tokio_util::time::delay_queue::{DelayQueue, Key as DelayKey}; use types::{ Attestation, EthSpec, Hash256, LightClientOptimisticUpdate, SignedAggregateAndProof, SubnetId, @@ -87,7 +87,7 @@ pub enum ReprocessQueueMessage { /// Events sent by the scheduler once they are ready for re-processing. pub enum ReadyWork { - Block(QueuedGossipBlock), + GossipBlock(QueuedGossipBlock), RpcBlock(QueuedRpcBlock), Unaggregate(QueuedUnaggregate), Aggregate(QueuedAggregate), @@ -154,8 +154,6 @@ enum InboundEvent { ReadyAttestation(QueuedAttestationId), /// A light client update that is ready for re-processing. ReadyLightClientUpdate(QueuedLightClientUpdateId), - /// A `DelayQueue` returned an error. - DelayQueueError(TimeError, &'static str), /// A message sent to the `ReprocessQueue` Msg(ReprocessQueueMessage), } @@ -233,54 +231,42 @@ impl Stream for ReprocessQueue { // The sequential nature of blockchains means it is generally better to try and import all // existing blocks before new ones. match self.gossip_block_delay_queue.poll_expired(cx) { - Poll::Ready(Some(Ok(queued_block))) => { + Poll::Ready(Some(queued_block)) => { return Poll::Ready(Some(InboundEvent::ReadyGossipBlock( queued_block.into_inner(), ))); } - Poll::Ready(Some(Err(e))) => { - return Poll::Ready(Some(InboundEvent::DelayQueueError(e, "gossip_block_queue"))); - } // `Poll::Ready(None)` means that there are no more entries in the delay queue and we // will continue to get this result until something else is added into the queue. Poll::Ready(None) | Poll::Pending => (), } match self.rpc_block_delay_queue.poll_expired(cx) { - Poll::Ready(Some(Ok(queued_block))) => { + Poll::Ready(Some(queued_block)) => { return Poll::Ready(Some(InboundEvent::ReadyRpcBlock(queued_block.into_inner()))); } - Poll::Ready(Some(Err(e))) => { - return Poll::Ready(Some(InboundEvent::DelayQueueError(e, "rpc_block_queue"))); - } // `Poll::Ready(None)` means that there are no more entries in the delay queue and we // will continue to get this result until something else is added into the queue. Poll::Ready(None) | Poll::Pending => (), } match self.attestations_delay_queue.poll_expired(cx) { - Poll::Ready(Some(Ok(attestation_id))) => { + Poll::Ready(Some(attestation_id)) => { return Poll::Ready(Some(InboundEvent::ReadyAttestation( attestation_id.into_inner(), ))); } - Poll::Ready(Some(Err(e))) => { - return Poll::Ready(Some(InboundEvent::DelayQueueError(e, "attestations_queue"))); - } // `Poll::Ready(None)` means that there are no more entries in the delay queue and we // will continue to get this result until something else is added into the queue. Poll::Ready(None) | Poll::Pending => (), } match self.lc_updates_delay_queue.poll_expired(cx) { - Poll::Ready(Some(Ok(lc_id))) => { + Poll::Ready(Some(lc_id)) => { return Poll::Ready(Some(InboundEvent::ReadyLightClientUpdate( lc_id.into_inner(), ))); } - Poll::Ready(Some(Err(e))) => { - return Poll::Ready(Some(InboundEvent::DelayQueueError(e, "lc_updates_queue"))); - } // `Poll::Ready(None)` means that there are no more entries in the delay queue and we // will continue to get this result until something else is added into the queue. Poll::Ready(None) | Poll::Pending => (), @@ -400,7 +386,7 @@ impl ReprocessQueue { if block_slot <= now && self .ready_work_tx - .try_send(ReadyWork::Block(early_block)) + .try_send(ReadyWork::GossipBlock(early_block)) .is_err() { error!( @@ -694,7 +680,7 @@ impl ReprocessQueue { if self .ready_work_tx - .try_send(ReadyWork::Block(ready_block)) + .try_send(ReadyWork::GossipBlock(ready_block)) .is_err() { error!( @@ -703,14 +689,7 @@ impl ReprocessQueue { ); } } - InboundEvent::DelayQueueError(e, queue_name) => { - crit!( - log, - "Failed to poll queue"; - "queue" => queue_name, - "e" => ?e - ) - } + InboundEvent::ReadyAttestation(queued_id) => { metrics::inc_counter( &metrics::BEACON_PROCESSOR_REPROCESSING_QUEUE_EXPIRED_ATTESTATIONS, diff --git a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs b/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs index e5b4bdf86..69e167b4d 100644 --- a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs @@ -1,6 +1,6 @@ use crate::{metrics, service::NetworkMessage, sync::SyncMessage}; -use beacon_chain::blob_verification::{AsBlock, BlockWrapper}; +use beacon_chain::blob_verification::{AsBlock, BlockWrapper, GossipVerifiedBlob}; use beacon_chain::store::Error; use beacon_chain::{ attestation_verification::{self, Error as AttnError, VerifiedAttestation}, @@ -9,15 +9,14 @@ use beacon_chain::{ observed_operations::ObservationOutcome, sync_committee_verification::{self, Error as SyncCommitteeError}, validator_monitor::get_block_delay_ms, - BeaconChainError, BeaconChainTypes, BlockError, CountUnrealized, ForkChoiceError, - GossipVerifiedBlock, NotifyExecutionLayer, + AvailabilityProcessingStatus, BeaconChainError, BeaconChainTypes, BlockError, CountUnrealized, + ForkChoiceError, GossipVerifiedBlock, NotifyExecutionLayer, }; use lighthouse_network::{Client, MessageAcceptance, MessageId, PeerAction, PeerId, ReportSource}; use operation_pool::ReceivedPreCapella; use slog::{crit, debug, error, info, trace, warn}; use slot_clock::SlotClock; use ssz::Encode; -use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use store::hot_cold_store::HotColdDBError; use tokio::sync::mpsc; @@ -654,19 +653,57 @@ impl Worker { self, _message_id: MessageId, peer_id: PeerId, - peer_client: Client, + _peer_client: Client, blob_index: u64, - signed_blob: Arc>, + signed_blob: SignedBlobSidecar, _seen_duration: Duration, ) { - // TODO: gossip verification - crit!(self.log, "UNIMPLEMENTED gossip blob verification"; - "peer_id" => %peer_id, - "client" => %peer_client, - "blob_topic" => blob_index, - "blob_index" => signed_blob.message.index, - "blob_slot" => signed_blob.message.slot - ); + match self + .chain + .verify_blob_sidecar_for_gossip(signed_blob, blob_index) + { + Ok(gossip_verified_blob) => { + self.process_gossip_verified_blob(peer_id, gossip_verified_blob, _seen_duration) + .await + } + Err(_) => { + // TODO(pawan): handle all blob errors for peer scoring + todo!() + } + } + } + + pub async fn process_gossip_verified_blob( + self, + peer_id: PeerId, + verified_blob: GossipVerifiedBlob, + // This value is not used presently, but it might come in handy for debugging. + _seen_duration: Duration, + ) { + // TODO + match self + .chain + .process_blob(verified_blob, CountUnrealized::True) + .await + { + Ok(AvailabilityProcessingStatus::Imported(_hash)) => { + todo!() + // add to metrics + // logging + } + Ok(AvailabilityProcessingStatus::PendingBlobs(pending_blobs)) => self + .send_sync_message(SyncMessage::UnknownBlobHash { + peer_id, + pending_blobs, + }), + Ok(AvailabilityProcessingStatus::PendingBlock(block_hash)) => { + self.send_sync_message(SyncMessage::UnknownBlockHash(peer_id, block_hash)); + } + Err(_err) => { + // handle errors + todo!() + } + } } /// Process the beacon block received from the gossip network and: @@ -802,6 +839,9 @@ impl Worker { verified_block } + Err(BlockError::AvailabilityCheck(_err)) => { + todo!() + } Err(BlockError::ParentUnknown(block)) => { debug!( self.log, @@ -984,7 +1024,7 @@ impl Worker { ) .await { - Ok(block_root) => { + Ok(AvailabilityProcessingStatus::Imported(block_root)) => { metrics::inc_counter(&metrics::BEACON_PROCESSOR_GOSSIP_BLOCK_IMPORTED_TOTAL); if reprocess_tx @@ -1011,6 +1051,24 @@ impl Worker { self.chain.recompute_head_at_current_slot().await; } + Ok(AvailabilityProcessingStatus::PendingBlock(block_root)) => { + // This error variant doesn't make any sense in this context + crit!( + self.log, + "Internal error. Cannot get AvailabilityProcessingStatus::PendingBlock on processing block"; + "block_root" => %block_root + ); + } + Ok(AvailabilityProcessingStatus::PendingBlobs(pending_blobs)) => { + // make rpc request for blob + self.send_sync_message(SyncMessage::UnknownBlobHash { + peer_id, + pending_blobs, + }); + } + Err(BlockError::AvailabilityCheck(_)) => { + todo!() + } Err(BlockError::ParentUnknown(block)) => { // Inform the sync manager to find parents for this block // This should not occur. It should be checked by `should_forward_block` diff --git a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs index 45c31b77e..ecb2eb424 100644 --- a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - use crate::beacon_processor::{worker::FUTURE_SLOT_TOLERANCE, SendOnDrop}; use crate::service::NetworkMessage; use crate::status::ToStatusMessage; @@ -15,7 +13,6 @@ use lighthouse_network::{PeerId, PeerRequestId, ReportSource, Response, SyncInfo use slog::{debug, error, warn}; use slot_clock::SlotClock; use std::collections::{hash_map::Entry, HashMap}; -use std::sync::Arc; use task_executor::TaskExecutor; use tokio_stream::StreamExt; use types::blob_sidecar::BlobIdentifier; @@ -840,7 +837,7 @@ impl Worker { let mut send_response = true; for root in block_roots { - match self.chain.get_blobs(&root, data_availability_boundary) { + match self.chain.get_blobs(&root) { Ok(Some(blob_sidecar_list)) => { for blob_sidecar in blob_sidecar_list.iter() { blobs_sent += 1; diff --git a/beacon_node/network/src/beacon_processor/worker/sync_methods.rs b/beacon_node/network/src/beacon_processor/worker/sync_methods.rs index 8b7e0f2cb..09e98cb18 100644 --- a/beacon_node/network/src/beacon_processor/worker/sync_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/sync_methods.rs @@ -7,8 +7,9 @@ use crate::beacon_processor::DuplicateCache; use crate::metrics; use crate::sync::manager::{BlockProcessType, SyncMessage}; use crate::sync::{BatchProcessResult, ChainId}; -use beacon_chain::blob_verification::{AsBlock, BlockWrapper, IntoAvailableBlock}; -use beacon_chain::CountUnrealized; +use beacon_chain::blob_verification::AsBlock; +use beacon_chain::blob_verification::BlockWrapper; +use beacon_chain::{AvailabilityProcessingStatus, CountUnrealized}; use beacon_chain::{ BeaconChainError, BeaconChainTypes, BlockError, ChainSegmentResult, HistoricalBlockError, NotifyExecutionLayer, @@ -86,28 +87,22 @@ impl Worker { }; let slot = block.slot(); let parent_root = block.message().parent_root(); - let available_block = block - .into_available_block(block_root, &self.chain) - .map_err(BlockError::BlobValidation); - let result = match available_block { - Ok(block) => { - self.chain - .process_block( - block_root, - block, - CountUnrealized::True, - NotifyExecutionLayer::Yes, - ) - .await - } - Err(e) => Err(e), - }; + let result = self + .chain + .process_block( + block_root, + block, + CountUnrealized::True, + NotifyExecutionLayer::Yes, + ) + .await; metrics::inc_counter(&metrics::BEACON_PROCESSOR_RPC_BLOCK_IMPORTED_TOTAL); // RPC block imported, regardless of process type - if let &Ok(hash) = &result { + //TODO(sean) handle pending availability variants + if let &Ok(AvailabilityProcessingStatus::Imported(hash)) = &result { info!(self.log, "New RPC block received"; "slot" => slot, "hash" => %hash); // Trigger processing for work referencing this block. diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index 5e2cd6181..fed799988 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -280,7 +280,6 @@ impl Router { PubsubMessage::BlobSidecar(data) => { let (blob_index, signed_blob) = *data; let peer_client = self.network_globals.client(&peer_id); - let signed_blob = Arc::new(signed_blob); self.send_beacon_processor_work(BeaconWorkEvent::gossip_signed_blob_sidecar( message_id, peer_id, diff --git a/beacon_node/network/src/sync/block_lookups/mod.rs b/beacon_node/network/src/sync/block_lookups/mod.rs index 690d56644..77e659a26 100644 --- a/beacon_node/network/src/sync/block_lookups/mod.rs +++ b/beacon_node/network/src/sync/block_lookups/mod.rs @@ -2,7 +2,8 @@ use std::collections::hash_map::Entry; use std::collections::HashMap; use std::time::Duration; -use beacon_chain::blob_verification::{AsBlock, BlockWrapper}; +use beacon_chain::blob_verification::AsBlock; +use beacon_chain::blob_verification::BlockWrapper; use beacon_chain::{BeaconChainTypes, BlockError}; use fnv::FnvHashMap; use lighthouse_network::rpc::{RPCError, RPCResponseErrorCode}; diff --git a/beacon_node/network/src/sync/block_lookups/parent_lookup.rs b/beacon_node/network/src/sync/block_lookups/parent_lookup.rs index b6de52d70..f066191c0 100644 --- a/beacon_node/network/src/sync/block_lookups/parent_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/parent_lookup.rs @@ -1,5 +1,6 @@ use super::RootBlockTuple; -use beacon_chain::blob_verification::{AsBlock, BlockWrapper}; +use beacon_chain::blob_verification::AsBlock; +use beacon_chain::blob_verification::BlockWrapper; use beacon_chain::BeaconChainTypes; use lighthouse_network::PeerId; use store::Hash256; diff --git a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs index 02482d619..60911dbb3 100644 --- a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs @@ -1,5 +1,6 @@ use super::RootBlockTuple; -use beacon_chain::blob_verification::{AsBlock, BlockWrapper}; +use beacon_chain::blob_verification::AsBlock; +use beacon_chain::blob_verification::BlockWrapper; use beacon_chain::get_block_root; use lighthouse_network::{rpc::BlocksByRootRequest, PeerId}; use rand::seq::IteratorRandom; diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 43921b585..482bfab70 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -42,7 +42,8 @@ use crate::beacon_processor::{ChainSegmentProcessId, WorkEvent as BeaconWorkEven use crate::service::NetworkMessage; use crate::status::ToStatusMessage; use crate::sync::range_sync::ByRangeRequestType; -use beacon_chain::blob_verification::{AsBlock, BlockWrapper}; +use beacon_chain::blob_verification::AsBlock; +use beacon_chain::blob_verification::BlockWrapper; use beacon_chain::{BeaconChain, BeaconChainTypes, BlockError, EngineState}; use futures::StreamExt; use lighthouse_network::rpc::methods::MAX_REQUEST_BLOCKS; @@ -56,6 +57,7 @@ use std::ops::Sub; use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc; +use types::blob_sidecar::BlobIdentifier; use types::{BlobSidecar, EthSpec, Hash256, SignedBeaconBlock, Slot}; /// The number of slots ahead of us that is allowed before requesting a long-range (batch) Sync @@ -119,6 +121,13 @@ pub enum SyncMessage { /// manager to attempt to find the block matching the unknown hash. UnknownBlockHash(PeerId, Hash256), + /// A peer has sent us a block that we haven't received all the blobs for. This triggers + /// the manager to attempt to find the pending blobs for the given block root. + UnknownBlobHash { + peer_id: PeerId, + pending_blobs: Vec, + }, + /// A peer has disconnected. Disconnect(PeerId), @@ -598,6 +607,9 @@ impl SyncManager { .search_block(block_hash, peer_id, &mut self.network); } } + SyncMessage::UnknownBlobHash { .. } => { + unimplemented!() + } SyncMessage::Disconnect(peer_id) => { self.peer_disconnect(&peer_id); } diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index 2c80c2c1a..fc902d866 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -38,6 +38,7 @@ use std::marker::PhantomData; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; +use types::blob_sidecar::BlobSidecarList; use types::consts::eip4844::MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS; use types::*; @@ -547,7 +548,7 @@ impl, Cold: ItemStore> HotColdDB /// Check if the blobs sidecar for a block exists on disk. pub fn blobs_sidecar_exists(&self, block_root: &Hash256) -> Result { - self.get_item::>(block_root) + self.get_item::>(block_root) .map(|blobs| blobs.is_some()) } diff --git a/beacon_node/store/src/impls/execution_payload.rs b/beacon_node/store/src/impls/execution_payload.rs index 01a2dba0b..f4e6c1ea6 100644 --- a/beacon_node/store/src/impls/execution_payload.rs +++ b/beacon_node/store/src/impls/execution_payload.rs @@ -1,7 +1,7 @@ use crate::{DBColumn, Error, StoreItem}; use ssz::{Decode, Encode}; use types::{ - BlobsSidecar, EthSpec, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadEip4844, + BlobSidecarList, EthSpec, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadEip4844, ExecutionPayloadMerge, }; @@ -25,7 +25,7 @@ macro_rules! impl_store_item { impl_store_item!(ExecutionPayloadMerge); impl_store_item!(ExecutionPayloadCapella); impl_store_item!(ExecutionPayloadEip4844); -impl_store_item!(BlobsSidecar); +impl_store_item!(BlobSidecarList); /// This fork-agnostic implementation should be only used for writing. /// diff --git a/beacon_node/store/src/lib.rs b/beacon_node/store/src/lib.rs index 29fded5fa..179b099de 100644 --- a/beacon_node/store/src/lib.rs +++ b/beacon_node/store/src/lib.rs @@ -43,6 +43,7 @@ pub use metrics::scrape_for_metrics; use parking_lot::MutexGuard; use std::sync::Arc; use strum::{EnumString, IntoStaticStr}; +use types::blob_sidecar::BlobSidecarList; pub use types::*; pub type ColumnIter<'a> = Box), Error>> + 'a>; diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index bc27ddb47..543b3fda6 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -1396,3 +1396,32 @@ pub struct SignedBeaconBlockAndBlobSidecars, pub signed_blob_sidecars: SignedBlobSidecarList, } + +#[derive(Debug, Clone, Serialize, Deserialize, Encode)] +#[serde(bound = "T: EthSpec, Payload: AbstractExecPayload")] +pub struct BeaconBlockAndBlobSidecars> { + pub block: BeaconBlock, + pub blob_sidecars: BlobSidecarList, +} + +impl> ForkVersionDeserialize + for BeaconBlockAndBlobSidecars +{ + fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( + value: serde_json::value::Value, + fork_name: ForkName, + ) -> Result { + #[derive(Deserialize)] + #[serde(bound = "T: EthSpec")] + struct Helper { + block: serde_json::Value, + blob_sidecars: BlobSidecarList, + } + let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; + + Ok(Self { + block: BeaconBlock::deserialize_by_fork::<'de, D>(helper.block, fork_name)?, + blob_sidecars: helper.blob_sidecars, + }) + } +} diff --git a/consensus/state_processing/src/consensus_context.rs b/consensus/state_processing/src/consensus_context.rs index 4c8966f92..37bd5fe44 100644 --- a/consensus/state_processing/src/consensus_context.rs +++ b/consensus/state_processing/src/consensus_context.rs @@ -7,7 +7,7 @@ use types::{ ChainSpec, Epoch, EthSpec, Hash256, IndexedAttestation, SignedBeaconBlock, Slot, }; -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ConsensusContext { /// Slot to act as an identifier/safeguard slot: Slot, diff --git a/consensus/types/src/beacon_block_and_blob_sidecars.rs b/consensus/types/src/beacon_block_and_blob_sidecars.rs deleted file mode 100644 index 78e704196..000000000 --- a/consensus/types/src/beacon_block_and_blob_sidecars.rs +++ /dev/null @@ -1,37 +0,0 @@ -use crate::{ - AbstractExecPayload, BeaconBlock, BlobSidecarList, EthSpec, ForkName, ForkVersionDeserialize, -}; -use derivative::Derivative; -use serde_derive::{Deserialize, Serialize}; -use ssz_derive::Encode; -use tree_hash_derive::TreeHash; - -#[derive(Debug, Clone, Serialize, Deserialize, Encode, TreeHash, Derivative)] -#[derivative(PartialEq, Hash(bound = "T: EthSpec"))] -#[serde(bound = "T: EthSpec, Payload: AbstractExecPayload")] -pub struct BeaconBlockAndBlobSidecars> { - pub block: BeaconBlock, - pub blob_sidecars: BlobSidecarList, -} - -impl> ForkVersionDeserialize - for BeaconBlockAndBlobSidecars -{ - fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( - value: serde_json::value::Value, - fork_name: ForkName, - ) -> Result { - #[derive(Deserialize)] - #[serde(bound = "T: EthSpec")] - struct Helper { - block: serde_json::Value, - blob_sidecars: BlobSidecarList, - } - let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; - - Ok(Self { - block: BeaconBlock::deserialize_by_fork::<'de, D>(helper.block, fork_name)?, - blob_sidecars: helper.blob_sidecars, - }) - } -} diff --git a/consensus/types/src/beacon_block_body.rs b/consensus/types/src/beacon_block_body.rs index c71739652..e49f63345 100644 --- a/consensus/types/src/beacon_block_body.rs +++ b/consensus/types/src/beacon_block_body.rs @@ -1,5 +1,5 @@ +use crate::test_utils::TestRandom; use crate::*; -use crate::{blobs_sidecar::KzgCommitments, test_utils::TestRandom}; use derivative::Derivative; use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; @@ -9,6 +9,8 @@ use superstruct::superstruct; use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; +pub type KzgCommitments = VariableList::MaxBlobsPerBlock>; + /// The body of a `BeaconChain` block, containing operations. /// /// This *superstruct* abstracts over the hard-fork. diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index 29eaadc58..12f8afd9c 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -11,7 +11,7 @@ use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; /// Container of the data that identifies an individual blob. -#[derive(Encode, Decode, Clone, Debug, PartialEq)] +#[derive(Encode, Decode, Clone, Debug, PartialEq, Eq, Hash)] pub struct BlobIdentifier { pub block_root: Hash256, pub index: u64, @@ -35,7 +35,6 @@ pub struct BlobIdentifier { #[derivative(PartialEq, Hash(bound = "T: EthSpec"))] pub struct BlobSidecar { pub block_root: Hash256, - // TODO: fix the type, should fit in u8 as well #[serde(with = "eth2_serde_utils::quoted_u64")] pub index: u64, pub slot: Slot, @@ -49,10 +48,18 @@ pub struct BlobSidecar { } pub type BlobSidecarList = VariableList>, ::MaxBlobsPerBlock>; +pub type Blobs = VariableList, ::MaxExtraDataBytes>; impl SignedRoot for BlobSidecar {} impl BlobSidecar { + pub fn id(&self) -> BlobIdentifier { + BlobIdentifier { + block_root: self.block_root, + index: self.index, + } + } + pub fn empty() -> Self { Self::default() } diff --git a/consensus/types/src/blobs_sidecar.rs b/consensus/types/src/blobs_sidecar.rs deleted file mode 100644 index e2560fb30..000000000 --- a/consensus/types/src/blobs_sidecar.rs +++ /dev/null @@ -1,62 +0,0 @@ -use crate::test_utils::TestRandom; -use crate::{Blob, EthSpec, Hash256, KzgCommitment, SignedRoot, Slot}; -use derivative::Derivative; -use kzg::KzgProof; -use serde_derive::{Deserialize, Serialize}; -use ssz::Encode; -use ssz_derive::{Decode, Encode}; -use ssz_types::VariableList; -use test_random_derive::TestRandom; -use tree_hash_derive::TreeHash; - -pub type KzgCommitments = VariableList::MaxBlobsPerBlock>; -pub type Blobs = VariableList, ::MaxBlobsPerBlock>; - -#[derive( - Debug, - Clone, - Serialize, - Deserialize, - Encode, - Decode, - TreeHash, - Default, - TestRandom, - Derivative, - arbitrary::Arbitrary, -)] -#[serde(bound = "T: EthSpec")] -#[arbitrary(bound = "T: EthSpec")] -#[derivative(PartialEq, Hash(bound = "T: EthSpec"))] -pub struct BlobsSidecar { - pub beacon_block_root: Hash256, - pub beacon_block_slot: Slot, - #[serde(with = "ssz_types::serde_utils::list_of_hex_fixed_vec")] - pub blobs: Blobs, - pub kzg_aggregated_proof: KzgProof, -} - -impl SignedRoot for BlobsSidecar {} - -impl BlobsSidecar { - pub fn empty() -> Self { - Self::default() - } - - pub fn empty_from_parts(beacon_block_root: Hash256, beacon_block_slot: Slot) -> Self { - Self { - beacon_block_root, - beacon_block_slot, - blobs: VariableList::empty(), - kzg_aggregated_proof: KzgProof::empty(), - } - } - - #[allow(clippy::integer_arithmetic)] - pub fn max_size() -> usize { - // Fixed part - Self::empty().as_ssz_bytes().len() - // Max size of variable length `blobs` field - + (T::max_blobs_per_block() * as Encode>::ssz_fixed_len()) - } -} diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 6a86e773a..14f3ff356 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -97,11 +97,8 @@ pub mod slot_data; #[cfg(feature = "sqlite")] pub mod sqlite; -pub mod beacon_block_and_blob_sidecars; pub mod blob_sidecar; -pub mod blobs_sidecar; pub mod signed_blob; -pub mod signed_block_and_blobs; pub mod transaction; use ethereum_types::{H160, H256}; @@ -115,7 +112,6 @@ pub use crate::beacon_block::{ BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockCapella, BeaconBlockEip4844, BeaconBlockMerge, BeaconBlockRef, BeaconBlockRefMut, BlindedBeaconBlock, EmptyBlock, }; -pub use crate::beacon_block_and_blob_sidecars::BeaconBlockAndBlobSidecars; pub use crate::beacon_block_body::{ BeaconBlockBody, BeaconBlockBodyAltair, BeaconBlockBodyBase, BeaconBlockBodyCapella, BeaconBlockBodyEip4844, BeaconBlockBodyMerge, BeaconBlockBodyRef, BeaconBlockBodyRefMut, @@ -124,7 +120,6 @@ pub use crate::beacon_block_header::BeaconBlockHeader; pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee}; pub use crate::beacon_state::{BeaconTreeHashCache, Error as BeaconStateError, *}; pub use crate::blob_sidecar::{BlobSidecar, BlobSidecarList}; -pub use crate::blobs_sidecar::{Blobs, BlobsSidecar}; pub use crate::bls_to_execution_change::BlsToExecutionChange; pub use crate::chain_spec::{ChainSpec, Config, Domain}; pub use crate::checkpoint::Checkpoint; @@ -183,9 +178,6 @@ pub use crate::signed_beacon_block::{ }; pub use crate::signed_beacon_block_header::SignedBeaconBlockHeader; pub use crate::signed_blob::*; -pub use crate::signed_block_and_blobs::{ - SignedBeaconBlockAndBlobsSidecar, SignedBeaconBlockAndBlobsSidecarDecode, -}; pub use crate::signed_bls_to_execution_change::SignedBlsToExecutionChange; pub use crate::signed_contribution_and_proof::SignedContributionAndProof; pub use crate::signed_voluntary_exit::SignedVoluntaryExit; diff --git a/consensus/types/src/signed_beacon_block.rs b/consensus/types/src/signed_beacon_block.rs index ae59690bf..301cfd5f8 100644 --- a/consensus/types/src/signed_beacon_block.rs +++ b/consensus/types/src/signed_beacon_block.rs @@ -35,14 +35,6 @@ impl From for Hash256 { } } -#[derive(Debug)] -pub enum BlobReconstructionError { - /// No blobs for the specified block where we would expect blobs. - UnavailableBlobs, - /// Blobs provided for a pre-Eip4844 fork. - InconsistentFork, -} - /// A `BeaconBlock` and a signature from its proposer. #[superstruct( variants(Base, Altair, Merge, Capella, Eip4844), @@ -250,28 +242,6 @@ impl> SignedBeaconBlock pub fn canonical_root(&self) -> Hash256 { self.message().tree_hash_root() } - - /// Reconstructs an empty `BlobsSidecar`, using the given block root if provided, else calculates it. - /// If this block has kzg commitments, an error will be returned. If this block is from prior to the - /// Eip4844 fork, this will error. - pub fn reconstruct_empty_blobs( - &self, - block_root_opt: Option, - ) -> Result, BlobReconstructionError> { - let kzg_commitments = self - .message() - .body() - .blob_kzg_commitments() - .map_err(|_| BlobReconstructionError::InconsistentFork)?; - if kzg_commitments.is_empty() { - Ok(BlobsSidecar::empty_from_parts( - block_root_opt.unwrap_or_else(|| self.canonical_root()), - self.slot(), - )) - } else { - Err(BlobReconstructionError::UnavailableBlobs) - } - } } // We can convert pre-Bellatrix blocks without payloads into blocks with payloads. diff --git a/consensus/types/src/signed_blob.rs b/consensus/types/src/signed_blob.rs index 4eb28794e..aaab02ca7 100644 --- a/consensus/types/src/signed_blob.rs +++ b/consensus/types/src/signed_blob.rs @@ -1,10 +1,15 @@ -use crate::{test_utils::TestRandom, BlobSidecar, EthSpec, Signature}; +use crate::{ + test_utils::TestRandom, BlobSidecar, ChainSpec, Domain, EthSpec, Fork, Hash256, Signature, + SignedRoot, SigningData, +}; +use bls::PublicKey; use derivative::Derivative; use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::VariableList; use std::sync::Arc; use test_random_derive::TestRandom; +use tree_hash::TreeHash; use tree_hash_derive::TreeHash; #[derive( @@ -30,3 +35,37 @@ pub struct SignedBlobSidecar { pub type SignedBlobSidecarList = VariableList, ::MaxBlobsPerBlock>; + +impl SignedBlobSidecar { + /// Verify `self.signature`. + /// + /// If the root of `block.message` is already known it can be passed in via `object_root_opt`. + /// Otherwise, it will be computed locally. + pub fn verify_signature( + &self, + object_root_opt: Option, + pubkey: &PublicKey, + fork: &Fork, + genesis_validators_root: Hash256, + spec: &ChainSpec, + ) -> bool { + let domain = spec.get_domain( + self.message.slot.epoch(T::slots_per_epoch()), + Domain::BlobSidecar, + fork, + genesis_validators_root, + ); + + let message = if let Some(object_root) = object_root_opt { + SigningData { + object_root, + domain, + } + .tree_hash_root() + } else { + self.message.signing_root(domain) + }; + + self.signature.verify(pubkey, message) + } +} diff --git a/consensus/types/src/signed_block_and_blobs.rs b/consensus/types/src/signed_block_and_blobs.rs deleted file mode 100644 index a3bd34475..000000000 --- a/consensus/types/src/signed_block_and_blobs.rs +++ /dev/null @@ -1,35 +0,0 @@ -use crate::{BlobsSidecar, EthSpec, SignedBeaconBlock, SignedBeaconBlockEip4844}; -use derivative::Derivative; -use serde_derive::{Deserialize, Serialize}; -use ssz::{Decode, DecodeError}; -use ssz_derive::{Decode, Encode}; -use std::sync::Arc; -use tree_hash_derive::TreeHash; - -#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, PartialEq)] -#[serde(bound = "T: EthSpec")] -pub struct SignedBeaconBlockAndBlobsSidecarDecode { - pub beacon_block: SignedBeaconBlockEip4844, - pub blobs_sidecar: BlobsSidecar, -} - -// TODO: will be removed once we decouple blobs in Gossip -#[derive(Debug, Clone, Serialize, Deserialize, Encode, TreeHash, Derivative)] -#[derivative(PartialEq, Hash(bound = "T: EthSpec"))] -pub struct SignedBeaconBlockAndBlobsSidecar { - pub beacon_block: Arc>, - pub blobs_sidecar: Arc>, -} - -impl SignedBeaconBlockAndBlobsSidecar { - pub fn from_ssz_bytes(bytes: &[u8]) -> Result { - let SignedBeaconBlockAndBlobsSidecarDecode { - beacon_block, - blobs_sidecar, - } = SignedBeaconBlockAndBlobsSidecarDecode::from_ssz_bytes(bytes)?; - Ok(SignedBeaconBlockAndBlobsSidecar { - beacon_block: Arc::new(SignedBeaconBlock::Eip4844(beacon_block)), - blobs_sidecar: Arc::new(blobs_sidecar), - }) - } -} diff --git a/lcli/src/parse_ssz.rs b/lcli/src/parse_ssz.rs index 0e87e330b..70f34e09e 100644 --- a/lcli/src/parse_ssz.rs +++ b/lcli/src/parse_ssz.rs @@ -63,7 +63,7 @@ pub fn run_parse_ssz(matches: &ArgMatches) -> Result<(), String> { "state_merge" => decode_and_print::>(&bytes, format)?, "state_capella" => decode_and_print::>(&bytes, format)?, "state_eip4844" => decode_and_print::>(&bytes, format)?, - "blobs_sidecar" => decode_and_print::>(&bytes, format)?, + "blob_sidecar" => decode_and_print::>(&bytes, format)?, other => return Err(format!("Unknown type: {}", other)), }; diff --git a/testing/ef_tests/src/type_name.rs b/testing/ef_tests/src/type_name.rs index 19b3535fb..d94dfef48 100644 --- a/testing/ef_tests/src/type_name.rs +++ b/testing/ef_tests/src/type_name.rs @@ -50,7 +50,7 @@ type_name_generic!(BeaconBlockBodyCapella, "BeaconBlockBody"); type_name_generic!(BeaconBlockBodyEip4844, "BeaconBlockBody"); type_name!(BeaconBlockHeader); type_name_generic!(BeaconState); -type_name_generic!(BlobsSidecar); +type_name_generic!(BlobSidecar); type_name!(Checkpoint); type_name_generic!(ContributionAndProof); type_name!(Deposit); @@ -88,9 +88,5 @@ type_name!(Validator); type_name!(VoluntaryExit); type_name!(Withdrawal); type_name!(BlsToExecutionChange, "BLSToExecutionChange"); -type_name_generic!( - SignedBeaconBlockAndBlobsSidecarDecode, - "SignedBeaconBlockAndBlobsSidecar" -); type_name!(SignedBlsToExecutionChange, "SignedBLSToExecutionChange"); type_name!(HistoricalSummary); diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index fe2c5f796..d5b8dc80a 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -216,7 +216,6 @@ macro_rules! ssz_static_test_no_run { mod ssz_static { use ef_tests::{Handler, SszStaticHandler, SszStaticTHCHandler, SszStaticWithSpecHandler}; use types::historical_summary::HistoricalSummary; - use types::signed_block_and_blobs::SignedBeaconBlockAndBlobsSidecarDecode; use types::*; ssz_static_test!(aggregate_and_proof, AggregateAndProof<_>); @@ -378,12 +377,6 @@ mod ssz_static { SszStaticHandler::, MainnetEthSpec>::eip4844_only().run(); } - #[test] - fn signed_blobs_sidecar() { - SszStaticHandler::, MinimalEthSpec>::eip4844_only().run(); - SszStaticHandler::, MainnetEthSpec>::eip4844_only().run(); - } - #[test] fn historical_summary() { SszStaticHandler::::capella_and_later().run();