Merge pull request #38 from realbigsean/merge-unstable-deneb-jul-14

Merge unstable deneb jul 14

.github/workflows/test-suite.yml

@@ -58,8 +58,8 @@ jobs:
       uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-    - name: Install anvil
+    - name: Install Foundry (anvil)
-      run: cargo install --git https://github.com/foundry-rs/foundry --locked anvil
+      uses: foundry-rs/foundry-toolchain@v1
     - name: Run tests in release
       run: make test-release
   release-tests-windows:
@@ -78,9 +78,8 @@ jobs:
       run: |
         choco install python protoc visualstudio2019-workload-vctools -y
         npm config set msvs_version 2019
-    - name: Install anvil
+    - name: Install Foundry (anvil)
-      # Extra feature to work around https://github.com/foundry-rs/foundry/issues/5115
+      uses: foundry-rs/foundry-toolchain@v1
-      run: cargo install --git https://github.com/foundry-rs/foundry --locked anvil --features ethers/ipc
     - name: Install make
       run: choco install -y make
     - uses: KyleMayes/install-llvm-action@v1
@@ -155,8 +154,8 @@ jobs:
       uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-    - name: Install anvil
+    - name: Install Foundry (anvil)
-      run: cargo install --git https://github.com/foundry-rs/foundry --locked anvil
+      uses: foundry-rs/foundry-toolchain@v1
     - name: Run tests in debug
       run: make test-debug
   state-transition-vectors-ubuntu:
@@ -211,8 +210,8 @@ jobs:
       uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-    - name: Install anvil
+    - name: Install Foundry (anvil)
-      run: cargo install --git https://github.com/foundry-rs/foundry --locked anvil
+      uses: foundry-rs/foundry-toolchain@v1
     - name: Run the beacon chain sim that starts from an eth1 contract
       run: cargo run --release --bin simulator eth1-sim
   merge-transition-ubuntu:
@@ -227,8 +226,8 @@ jobs:
       uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-    - name: Install anvil
+    - name: Install Foundry (anvil)
-      run: cargo install --git https://github.com/foundry-rs/foundry --locked anvil
+      uses: foundry-rs/foundry-toolchain@v1
     - name: Run the beacon chain sim and go through the merge transition
       run: cargo run --release --bin simulator eth1-sim --post-merge
   no-eth1-simulator-ubuntu:
@@ -257,8 +256,8 @@ jobs:
       uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-    - name: Install anvil
+    - name: Install Foundry (anvil)
-      run: cargo install --git https://github.com/foundry-rs/foundry --locked anvil
+      uses: foundry-rs/foundry-toolchain@v1
     - name: Run the syncing simulator
       run: cargo run --release --bin simulator syncing-sim
   doppelganger-protection-test:

Cargo.lock

@@ -257,6 +257,15 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "anvil-rpc"
+version = "0.1.0"
+source = "git+https://github.com/foundry-rs/foundry?rev=b45456717ffae1af65acdc71099f8cb95e6683a0#b45456717ffae1af65acdc71099f8cb95e6683a0"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "anyhow"
 version = "1.0.71"
@@ -506,9 +515,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
 [[package]]
 name = "axum"
-version = "0.5.17"
+version = "0.6.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "acee9fd5073ab6b045a275b3e709c163dd36c90685219cb21804a147b58dba43"
+checksum = "f8175979259124331c1d7bf6586ee7e0da434155e4b2d48ec2c8386281d8df39"
 dependencies = [
  "async-trait",
  "axum-core",
@@ -524,22 +533,23 @@ dependencies = [
  "mime",
  "percent-encoding",
  "pin-project-lite 0.2.9",
+ "rustversion",
  "serde",
  "serde_json",
+ "serde_path_to_error",
  "serde_urlencoded",
  "sync_wrapper",
  "tokio",
  "tower",
- "tower-http",
  "tower-layer",
  "tower-service",
 ]
 
 [[package]]
 name = "axum-core"
-version = "0.2.9"
+version = "0.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37e5939e02c56fecd5c017c37df4238c0a839fa76b7f97acdd7efb804fd181cc"
+checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c"
 dependencies = [
  "async-trait",
  "bytes",
@@ -547,6 +557,7 @@ dependencies = [
  "http",
  "http-body",
  "mime",
+ "rustversion",
  "tower-layer",
  "tower-service",
 ]
@@ -605,7 +616,7 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b"
 [[package]]
 name = "beacon-api-client"
 version = "0.1.0"
-source = "git+https://github.com/ralexstokes/beacon-api-client#30679e9e25d61731cde54e14cd8a3688a39d8e5b"
+source = "git+https://github.com/ralexstokes/beacon-api-client?rev=93d7e8c#93d7e8c38fe9782c4862909663e7b57c44f805a9"
 dependencies = [
  "ethereum-consensus",
  "http",
@@ -717,6 +728,30 @@ dependencies = [
  "unused_port",
 ]
 
+[[package]]
+name = "beacon_processor"
+version = "0.1.0"
+dependencies = [
+ "derivative",
+ "ethereum_ssz",
+ "fnv",
+ "futures",
+ "hex",
+ "itertools",
+ "lazy_static",
+ "lighthouse_metrics",
+ "lighthouse_network",
+ "logging",
+ "parking_lot 0.12.1",
+ "slog",
+ "slot_clock",
+ "strum",
+ "task_executor",
+ "tokio",
+ "tokio-util 0.6.10",
+ "types",
+]
+
 [[package]]
 name = "bincode"
 version = "1.3.3"
@@ -1218,6 +1253,7 @@ name = "client"
 version = "0.2.0"
 dependencies = [
  "beacon_chain",
+ "beacon_processor",
  "directory",
  "dirs",
  "environment",
@@ -1235,6 +1271,7 @@ dependencies = [
  "logging",
  "monitoring_api",
  "network",
+ "num_cpus",
  "operation_pool",
  "parking_lot 0.12.1",
  "sensitive_url",
@@ -2579,7 +2616,7 @@ dependencies = [
 [[package]]
 name = "ethereum-consensus"
 version = "0.1.1"
-source = "git+https://github.com/ralexstokes//ethereum-consensus?rev=9b0ee0a8a45b968c8df5e7e64ea1c094e16f053d#9b0ee0a8a45b968c8df5e7e64ea1c094e16f053d"
+source = "git+https://github.com/ralexstokes/ethereum-consensus?rev=e380108#e380108d15fcc40349927fdf3d11c71f9edb67c2"
 dependencies = [
  "async-stream",
  "blst",
@@ -2592,8 +2629,9 @@ dependencies = [
  "rand 0.8.5",
  "serde",
  "serde_json",
+ "serde_yaml",
  "sha2 0.9.9",
- "ssz-rs",
+ "ssz_rs",
  "thiserror",
  "tokio",
  "tokio-stream",
@@ -2867,7 +2905,7 @@ dependencies = [
  "serde_json",
  "slog",
  "slot_clock",
- "ssz-rs",
+ "ssz_rs",
  "ssz_types",
  "state_processing",
  "strum",
@@ -3592,12 +3630,6 @@ dependencies = [
  "pin-project-lite 0.2.9",
 ]
 
-[[package]]
-name = "http-range-header"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29"
-
 [[package]]
 name = "http_api"
 version = "0.1.0"
@@ -5078,9 +5110,9 @@ checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5"
 
 [[package]]
 name = "matchit"
-version = "0.5.0"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb"
+checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40"
 
 [[package]]
 name = "md-5"
@@ -5169,16 +5201,20 @@ dependencies = [
 
 [[package]]
 name = "mev-rs"
-version = "0.2.1"
+version = "0.3.0"
-source = "git+https://github.com/ralexstokes//mev-rs?rev=7813d4a4a564e0754e9aaab2d95520ba437c3889#7813d4a4a564e0754e9aaab2d95520ba437c3889"
+source = "git+https://github.com/ralexstokes/mev-rs?rev=216657016d5c0889b505857c89ae42c7aa2764af#216657016d5c0889b505857c89ae42c7aa2764af"
 dependencies = [
+ "anvil-rpc",
  "async-trait",
  "axum",
  "beacon-api-client",
  "ethereum-consensus",
  "hyper",
+ "parking_lot 0.12.1",
+ "reqwest",
  "serde",
- "ssz-rs",
+ "serde_json",
+ "ssz_rs",
  "thiserror",
  "tokio",
  "tracing",
@@ -5539,6 +5575,7 @@ name = "network"
 version = "0.2.0"
 dependencies = [
  "beacon_chain",
+ "beacon_processor",
  "delay_map",
  "derivative",
  "environment",
@@ -5562,6 +5599,7 @@ dependencies = [
  "matches",
  "num_cpus",
  "operation_pool",
+ "parking_lot 0.12.1",
  "rand 0.8.5",
  "rlp",
  "slog",
@@ -5627,6 +5665,7 @@ dependencies = [
  "execution_layer",
  "sensitive_url",
  "tempfile",
+ "tokio",
  "types",
  "validator_client",
  "validator_dir",
@@ -7464,6 +7503,16 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serde_path_to_error"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b1b6471d7496b051e03f1958802a73f88b947866f5146f329e47e36554f4e55"
+dependencies = [
+ "itoa",
+ "serde",
+]
+
 [[package]]
 name = "serde_repr"
 version = "0.1.12"
@@ -7973,23 +8022,24 @@ dependencies = [
 ]
 
 [[package]]
-name = "ssz-rs"
+name = "ssz_rs"
-version = "0.8.0"
+version = "0.9.0"
-source = "git+https://github.com/ralexstokes//ssz-rs?rev=adf1a0b14cef90b9536f28ef89da1fab316465e1#adf1a0b14cef90b9536f28ef89da1fab316465e1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "057291e5631f280978fa9c8009390663ca4613359fc1318e36a8c24c392f6d1f"
 dependencies = [
  "bitvec 1.0.1",
  "hex",
  "num-bigint",
  "serde",
  "sha2 0.9.9",
- "ssz-rs-derive",
+ "ssz_rs_derive",
- "thiserror",
 ]
 
 [[package]]
-name = "ssz-rs-derive"
+name = "ssz_rs_derive"
-version = "0.8.0"
+version = "0.9.0"
-source = "git+https://github.com/ralexstokes//ssz-rs?rev=adf1a0b14cef90b9536f28ef89da1fab316465e1#adf1a0b14cef90b9536f28ef89da1fab316465e1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f07d54c4d01a1713eb363b55ba51595da15f6f1211435b71466460da022aa140"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -8784,25 +8834,6 @@ dependencies = [
  "tracing",
 ]
 
-[[package]]
-name = "tower-http"
-version = "0.3.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858"
-dependencies = [
- "bitflags 1.3.2",
- "bytes",
- "futures-core",
- "futures-util",
- "http",
- "http-body",
- "http-range-header",
- "pin-project-lite 0.2.9",
- "tower",
- "tower-layer",
- "tower-service",
-]
-
 [[package]]
 name = "tower-layer"
 version = "0.3.2"

Cargo.toml

@@ -4,6 +4,7 @@ members = [
 
     "beacon_node",
     "beacon_node/beacon_chain",
+    "beacon_node/beacon_processor",
     "beacon_node/builder_client",
     "beacon_node/client",
     "beacon_node/eth1",
@@ -92,13 +93,6 @@ resolver = "2"
 warp = { git = "https://github.com/macladson/warp", rev="7e75acc368229a46a236a8c991bf251fe7fe50ef" }
 arbitrary = { git = "https://github.com/michaelsproul/arbitrary", rev="f002b99989b561ddce62e4cf2887b0f8860ae991" }
 
-[patch."https://github.com/ralexstokes/mev-rs"]
-mev-rs = { git = "https://github.com/ralexstokes//mev-rs", rev = "7813d4a4a564e0754e9aaab2d95520ba437c3889" }
-[patch."https://github.com/ralexstokes/ethereum-consensus"]
-ethereum-consensus = { git = "https://github.com/ralexstokes//ethereum-consensus", rev = "9b0ee0a8a45b968c8df5e7e64ea1c094e16f053d" }
-[patch."https://github.com/ralexstokes/ssz-rs"]
-ssz-rs = { git = "https://github.com/ralexstokes//ssz-rs", rev = "adf1a0b14cef90b9536f28ef89da1fab316465e1" }
-
 [profile.maxperf]
 inherits = "release"
 lto = "fat"

beacon_node/Cargo.toml

@@ -43,4 +43,4 @@ monitoring_api = { path = "../common/monitoring_api" }
 sensitive_url = { path = "../common/sensitive_url" }
 http_api = { path = "http_api" }
 unused_port = { path = "../common/unused_port" }
-strum = "0.24.1"
+strum = "0.24.1"

beacon_node/beacon_chain/src/attestation_rewards.rs

@@ -86,7 +86,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                 let ideal_reward = reward_numerator
                     .safe_div(active_increments)?
                     .safe_div(WEIGHT_DENOMINATOR)?;
-                if !state.is_in_inactivity_leak(previous_epoch, spec) {
+                if !state.is_in_inactivity_leak(previous_epoch, spec)? {
                     ideal_rewards_hashmap
                         .insert((flag_index, effective_balance), (ideal_reward, penalty));
                 } else {

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -236,6 +236,17 @@ pub struct PrePayloadAttributes {
     pub parent_block_number: u64,
 }
 
+/// Information about a state/block at a specific slot.
+#[derive(Debug, Clone, Copy)]
+pub struct FinalizationAndCanonicity {
+    /// True if the slot of the state or block is finalized.
+    ///
+    /// This alone DOES NOT imply that the state/block is finalized, use `self.is_finalized()`.
+    pub slot_is_finalized: bool,
+    /// True if the state or block is canonical at its slot.
+    pub canonical: bool,
+}
+
 /// Define whether a forkchoiceUpdate needs to be checked for an override (`Yes`) or has already
 /// been checked (`AlreadyApplied`). It is safe to specify `Yes` even if re-orgs are disabled.
 #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
@@ -470,6 +481,12 @@ pub struct BeaconChain<T: BeaconChainTypes> {
 
 type BeaconBlockAndState<T, Payload> = (BeaconBlock<T, Payload>, BeaconState<T>);
 
+impl FinalizationAndCanonicity {
+    pub fn is_finalized(self) -> bool {
+        self.slot_is_finalized && self.canonical
+    }
+}
+
 impl<T: BeaconChainTypes> BeaconChain<T> {
     /// Checks if a block is finalized.
     /// The finalization check is done with the block slot. The block root is used to verify that
@@ -499,16 +516,30 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         state_root: &Hash256,
         state_slot: Slot,
     ) -> Result<bool, Error> {
+        self.state_finalization_and_canonicity(state_root, state_slot)
+            .map(FinalizationAndCanonicity::is_finalized)
+    }
+
+    /// Fetch the finalization and canonicity status of the state with `state_root`.
+    pub fn state_finalization_and_canonicity(
+        &self,
+        state_root: &Hash256,
+        state_slot: Slot,
+    ) -> Result<FinalizationAndCanonicity, Error> {
         let finalized_slot = self
             .canonical_head
             .cached_head()
             .finalized_checkpoint()
             .epoch
             .start_slot(T::EthSpec::slots_per_epoch());
-        let is_canonical = self
+        let slot_is_finalized = state_slot <= finalized_slot;
+        let canonical = self
             .state_root_at_slot(state_slot)?
             .map_or(false, |canonical_root| state_root == &canonical_root);
-        Ok(state_slot <= finalized_slot && is_canonical)
+        Ok(FinalizationAndCanonicity {
+            slot_is_finalized,
+            canonical,
+        })
     }
 
     /// Persists the head tracker and fork choice.

beacon_node/beacon_chain/src/chain_config.rs

@@ -83,6 +83,8 @@ pub struct ChainConfig {
     pub enable_backfill_rate_limiting: bool,
     /// Whether to use `ProgressiveBalancesCache` in unrealized FFG progression calculation.
     pub progressive_balances_mode: ProgressiveBalancesMode,
+    /// Number of epochs between each migration of data from the hot database to the freezer.
+    pub epochs_per_migration: u64,
 }
 
 impl Default for ChainConfig {
@@ -114,6 +116,7 @@ impl Default for ChainConfig {
             always_prepare_payload: false,
             enable_backfill_rate_limiting: true,
             progressive_balances_mode: ProgressiveBalancesMode::Checked,
+            epochs_per_migration: crate::migrate::DEFAULT_EPOCHS_PER_MIGRATION,
         }
     }
 }

beacon_node/beacon_chain/src/lib.rs

@@ -81,6 +81,7 @@ pub use execution_payload::NotifyExecutionLayer;
 pub use fork_choice::{ExecutionStatus, ForkchoiceUpdateParameters};
 pub use kzg::TrustedSetup;
 pub use metrics::scrape_for_metrics;
+pub use migrate::MigratorConfig;
 pub use parking_lot;
 pub use slot_clock;
 pub use state_processing::per_block_processing::errors::{

beacon_node/beacon_chain/src/migrate.rs

@@ -25,10 +25,15 @@ const MIN_COMPACTION_PERIOD_SECONDS: u64 = 7200;
 /// Compact after a large finality gap, if we respect `MIN_COMPACTION_PERIOD_SECONDS`.
 const COMPACTION_FINALITY_DISTANCE: u64 = 1024;
 
+/// Default number of epochs to wait between finalization migrations.
+pub const DEFAULT_EPOCHS_PER_MIGRATION: u64 = 1;
+
 /// The background migrator runs a thread to perform pruning and migrate state from the hot
 /// to the cold database.
 pub struct BackgroundMigrator<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
     db: Arc<HotColdDB<E, Hot, Cold>>,
+    /// Record of when the last migration ran, for enforcing `epochs_per_migration`.
+    prev_migration: Arc<Mutex<PrevMigration>>,
     #[allow(clippy::type_complexity)]
     tx_thread: Option<Mutex<(mpsc::Sender<Notification>, thread::JoinHandle<()>)>>,
     /// Genesis block root, for persisting the `PersistedBeaconChain`.
@@ -36,9 +41,22 @@ pub struct BackgroundMigrator<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>
     log: Logger,
 }
 
-#[derive(Debug, Default, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 pub struct MigratorConfig {
     pub blocking: bool,
+    /// Run migrations at most once per `epochs_per_migration`.
+    ///
+    /// If set to 0 or 1, then run every finalization.
+    pub epochs_per_migration: u64,
+}
+
+impl Default for MigratorConfig {
+    fn default() -> Self {
+        Self {
+            blocking: false,
+            epochs_per_migration: DEFAULT_EPOCHS_PER_MIGRATION,
+        }
+    }
 }
 
 impl MigratorConfig {
@@ -46,6 +64,19 @@ impl MigratorConfig {
         self.blocking = true;
         self
     }
+
+    pub fn epochs_per_migration(mut self, epochs_per_migration: u64) -> Self {
+        self.epochs_per_migration = epochs_per_migration;
+        self
+    }
+}
+
+/// Record of when the last migration ran.
+pub struct PrevMigration {
+    /// The epoch at which the last finalization migration ran.
+    epoch: Epoch,
+    /// The number of epochs to wait between runs.
+    epochs_per_migration: u64,
 }
 
 /// Pruning can be successful, or in rare cases deferred to a later point.
@@ -93,6 +124,7 @@ pub struct FinalizationNotification {
     finalized_state_root: BeaconStateHash,
     finalized_checkpoint: Checkpoint,
     head_tracker: Arc<HeadTracker>,
+    prev_migration: Arc<Mutex<PrevMigration>>,
     genesis_block_root: Hash256,
 }
 
@@ -104,6 +136,11 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
         genesis_block_root: Hash256,
         log: Logger,
     ) -> Self {
+        // Estimate last migration run from DB split slot.
+        let prev_migration = Arc::new(Mutex::new(PrevMigration {
+            epoch: db.get_split_slot().epoch(E::slots_per_epoch()),
+            epochs_per_migration: config.epochs_per_migration,
+        }));
         let tx_thread = if config.blocking {
             None
         } else {
@@ -112,6 +149,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
         Self {
             db,
             tx_thread,
+            prev_migration,
             genesis_block_root,
             log,
         }
@@ -132,6 +170,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
             finalized_state_root,
             finalized_checkpoint,
             head_tracker,
+            prev_migration: self.prev_migration.clone(),
             genesis_block_root: self.genesis_block_root,
         };
 
@@ -227,6 +266,26 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
         notif: FinalizationNotification,
         log: &Logger,
     ) {
+        // Do not run too frequently.
+        let epoch = notif.finalized_checkpoint.epoch;
+        let mut prev_migration = notif.prev_migration.lock();
+        if epoch < prev_migration.epoch + prev_migration.epochs_per_migration {
+            debug!(
+                log,
+                "Database consolidation deferred";
+                "last_finalized_epoch" => prev_migration.epoch,
+                "new_finalized_epoch" => epoch,
+                "epochs_per_migration" => prev_migration.epochs_per_migration,
+            );
+            return;
+        }
+
+        // Update the previous migration epoch immediately to avoid holding the lock. If the
+        // migration doesn't succeed then the next migration will be retried at the next scheduled
+        // run.
+        prev_migration.epoch = epoch;
+        drop(prev_migration);
+
         debug!(log, "Database consolidation started");
 
         let finalized_state_root = notif.finalized_state_root;

beacon_node/beacon_chain/src/test_utils.rs

@@ -519,6 +519,7 @@ where
         let validator_keypairs = self
             .validator_keypairs
             .expect("cannot build without validator keypairs");
+        let chain_config = self.chain_config.unwrap_or_default();
         let trusted_setup: TrustedSetup =
             serde_json::from_reader(eth2_network_config::get_trusted_setup::<E::Kzg>())
                 .map_err(|e| format!("Unable to read trusted setup file: {}", e))
@@ -528,13 +529,17 @@ where
             .logger(log.clone())
             .custom_spec(spec)
             .store(self.store.expect("cannot build without store"))
-            .store_migrator_config(MigratorConfig::default().blocking())
+            .store_migrator_config(
+                MigratorConfig::default()
+                    .blocking()
+                    .epochs_per_migration(chain_config.epochs_per_migration),
+            )
             .task_executor(self.runtime.task_executor.clone())
             .execution_layer(self.execution_layer)
             .dummy_eth1_backend()
             .expect("should build dummy backend")
             .shutdown_sender(shutdown_tx)
-            .chain_config(self.chain_config.unwrap_or_default())
+            .chain_config(chain_config)
             .event_handler(Some(ServerSentEventHandler::new_with_capacity(
                 log.clone(),
                 5,

beacon_node/beacon_processor/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "beacon_processor"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+slog = { version = "2.5.2", features = ["max_level_trace"] }
+itertools = "0.10.0"
+logging = { path = "../../common/logging" }
+tokio = { version = "1.14.0", features = ["full"] }
+tokio-util = { version = "0.6.3", features = ["time"] }
+futures = "0.3.7"
+fnv = "1.0.7"
+strum = "0.24.0"
+task_executor = { path = "../../common/task_executor" }
+slot_clock = { path = "../../common/slot_clock" }
+lighthouse_network =  { path = "../lighthouse_network" }
+hex = "0.4.2"
+derivative = "2.2.0"
+types = { path = "../../consensus/types" }
+ethereum_ssz = "0.5.0"
+lazy_static = "1.4.0"
+lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
+parking_lot = "0.12.0"

beacon_node/beacon_processor/src/metrics.rs

@@ -0,0 +1,151 @@
+pub use lighthouse_metrics::*;
+
+lazy_static::lazy_static! {
+
+    /*
+     * Gossip processor
+     */
+    pub static ref BEACON_PROCESSOR_WORK_EVENTS_RX_COUNT: Result<IntCounterVec> = try_create_int_counter_vec(
+        "beacon_processor_work_events_rx_count",
+        "Count of work events received (but not necessarily processed)",
+        &["type"]
+    );
+    pub static ref BEACON_PROCESSOR_WORK_EVENTS_IGNORED_COUNT: Result<IntCounterVec> = try_create_int_counter_vec(
+        "beacon_processor_work_events_ignored_count",
+        "Count of work events purposefully ignored",
+        &["type"]
+    );
+    pub static ref BEACON_PROCESSOR_WORK_EVENTS_STARTED_COUNT: Result<IntCounterVec> = try_create_int_counter_vec(
+        "beacon_processor_work_events_started_count",
+        "Count of work events which have been started by a worker",
+        &["type"]
+    );
+    pub static ref BEACON_PROCESSOR_WORKER_TIME: Result<HistogramVec> = try_create_histogram_vec(
+        "beacon_processor_worker_time",
+        "Time taken for a worker to fully process some parcel of work.",
+        &["type"]
+    );
+    pub static ref BEACON_PROCESSOR_WORKERS_SPAWNED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_workers_spawned_total",
+        "The number of workers ever spawned by the gossip processing pool."
+    );
+    pub static ref BEACON_PROCESSOR_WORKERS_ACTIVE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_workers_active_total",
+        "Count of active workers in the gossip processing pool."
+    );
+    pub static ref BEACON_PROCESSOR_IDLE_EVENTS_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_idle_events_total",
+        "Count of idle events processed by the gossip processor manager."
+    );
+    pub static ref BEACON_PROCESSOR_EVENT_HANDLING_SECONDS: Result<Histogram> = try_create_histogram(
+        "beacon_processor_event_handling_seconds",
+        "Time spent handling a new message and allocating it to a queue or worker."
+    );
+    // Gossip blocks.
+    pub static ref BEACON_PROCESSOR_GOSSIP_BLOCK_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_gossip_block_queue_total",
+        "Count of blocks from gossip waiting to be verified."
+    );
+    // Gossip blobs.
+    pub static ref BEACON_PROCESSOR_GOSSIP_BLOB_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_gossip_blob_queue_total",
+        "Count of blocks from gossip waiting to be verified."
+    );
+    // Gossip Exits.
+    pub static ref BEACON_PROCESSOR_EXIT_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_exit_queue_total",
+        "Count of exits from gossip waiting to be verified."
+    );
+    // Gossip proposer slashings.
+    pub static ref BEACON_PROCESSOR_PROPOSER_SLASHING_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_proposer_slashing_queue_total",
+        "Count of proposer slashings from gossip waiting to be verified."
+    );
+    // Gossip attester slashings.
+    pub static ref BEACON_PROCESSOR_ATTESTER_SLASHING_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_attester_slashing_queue_total",
+        "Count of attester slashings from gossip waiting to be verified."
+    );
+    // Gossip BLS to execution changes.
+    pub static ref BEACON_PROCESSOR_BLS_TO_EXECUTION_CHANGE_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_bls_to_execution_change_queue_total",
+        "Count of address changes from gossip waiting to be verified."
+    );
+    // Rpc blocks.
+    pub static ref BEACON_PROCESSOR_RPC_BLOCK_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_rpc_block_queue_total",
+        "Count of blocks from the rpc waiting to be verified."
+    );
+    // Rpc blobs.
+    pub static ref BEACON_PROCESSOR_RPC_BLOB_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_rpc_blob_queue_total",
+        "Count of blobs from the rpc waiting to be verified."
+    );
+    // Chain segments.
+    pub static ref BEACON_PROCESSOR_CHAIN_SEGMENT_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_chain_segment_queue_total",
+        "Count of chain segments from the rpc waiting to be verified."
+    );
+    pub static ref BEACON_PROCESSOR_BACKFILL_CHAIN_SEGMENT_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_backfill_chain_segment_queue_total",
+        "Count of backfill chain segments from the rpc waiting to be verified."
+    );
+    // Unaggregated attestations.
+    pub static ref BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_unaggregated_attestation_queue_total",
+        "Count of unagg. attestations waiting to be processed."
+    );
+    // Aggregated attestations.
+    pub static ref BEACON_PROCESSOR_AGGREGATED_ATTESTATION_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_aggregated_attestation_queue_total",
+        "Count of agg. attestations waiting to be processed."
+    );
+    // Sync committee messages.
+    pub static ref BEACON_PROCESSOR_SYNC_MESSAGE_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_sync_message_queue_total",
+        "Count of sync committee messages waiting to be processed."
+    );
+    // Sync contribution.
+    pub static ref BEACON_PROCESSOR_SYNC_CONTRIBUTION_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "beacon_processor_sync_contribution_queue_total",
+        "Count of sync committee contributions waiting to be processed."
+    );
+
+    /*
+     * Attestation reprocessing queue metrics.
+     */
+    pub static ref BEACON_PROCESSOR_REPROCESSING_QUEUE_TOTAL: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+        "beacon_processor_reprocessing_queue_total",
+        "Count of items in a reprocessing queue.",
+        &["type"]
+    );
+    pub static ref BEACON_PROCESSOR_REPROCESSING_QUEUE_EXPIRED_ATTESTATIONS: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_reprocessing_queue_expired_attestations",
+        "Number of queued attestations which have expired before a matching block has been found."
+    );
+    pub static ref BEACON_PROCESSOR_REPROCESSING_QUEUE_MATCHED_ATTESTATIONS: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_reprocessing_queue_matched_attestations",
+        "Number of queued attestations where as matching block has been imported."
+    );
+
+    /*
+     * Light client update reprocessing queue metrics.
+     */
+    pub static ref BEACON_PROCESSOR_REPROCESSING_QUEUE_EXPIRED_OPTIMISTIC_UPDATES: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_reprocessing_queue_expired_optimistic_updates",
+        "Number of queued light client optimistic updates which have expired before a matching block has been found."
+    );
+    pub static ref BEACON_PROCESSOR_REPROCESSING_QUEUE_MATCHED_OPTIMISTIC_UPDATES: Result<IntCounter> = try_create_int_counter(
+        "beacon_processor_reprocessing_queue_matched_optimistic_updates",
+        "Number of queued light client optimistic updates where as matching block has been imported."
+    );
+
+    /// Errors and Debugging Stats
+    pub static ref BEACON_PROCESSOR_SEND_ERROR_PER_WORK_TYPE: Result<IntCounterVec> =
+        try_create_int_counter_vec(
+            "beacon_processor_send_error_per_work_type",
+            "Total number of beacon processor send error per work type",
+            &["type"]
+        );
+}

beacon_node/beacon_processor/src/work_reprocessing_queue.rs

@@ -10,17 +10,12 @@
 //!
 //! Aggregated and unaggregated attestations that failed verification due to referencing an unknown
 //! block will be re-queued until their block is imported, or until they expire.
-use super::MAX_SCHEDULED_WORK_QUEUE_LEN;
-use crate::beacon_processor::{ChainSegmentProcessId, Work, WorkEvent};
 use crate::metrics;
-use crate::sync::manager::BlockProcessType;
+use crate::{AsyncFn, BlockingFn, Work, WorkEvent};
-use beacon_chain::blob_verification::{AsBlock, BlockWrapper};
-use beacon_chain::{BeaconChainTypes, GossipVerifiedBlock, MAXIMUM_GOSSIP_CLOCK_DISPARITY};
 use fnv::FnvHashMap;
 use futures::task::Poll;
 use futures::{Stream, StreamExt};
 use itertools::Itertools;
-use lighthouse_network::{MessageId, PeerId};
 use logging::TimeLatch;
 use slog::{crit, debug, error, trace, warn, Logger};
 use slot_clock::SlotClock;
@@ -34,9 +29,7 @@ use task_executor::TaskExecutor;
 use tokio::sync::mpsc::{self, Receiver, Sender};
 use tokio::time::error::Error as TimeError;
 use tokio_util::time::delay_queue::{DelayQueue, Key as DelayKey};
-use types::{
+use types::{EthSpec, Hash256, Slot};
-    Attestation, EthSpec, Hash256, LightClientOptimisticUpdate, SignedAggregateAndProof, SubnetId,
-};
 
 const TASK_NAME: &str = "beacon_processor_reprocess_queue";
 const GOSSIP_BLOCKS: &str = "gossip_blocks";
@@ -46,7 +39,7 @@ const LIGHT_CLIENT_UPDATES: &str = "lc_updates";
 
 /// Queue blocks for re-processing with an `ADDITIONAL_QUEUED_BLOCK_DELAY` after the slot starts.
 /// This is to account for any slight drift in the system clock.
-const ADDITIONAL_QUEUED_BLOCK_DELAY: Duration = Duration::from_millis(5);
+pub const ADDITIONAL_QUEUED_BLOCK_DELAY: Duration = Duration::from_millis(5);
 
 /// For how long to queue aggregated and unaggregated attestations for re-processing.
 pub const QUEUED_ATTESTATION_DELAY: Duration = Duration::from_secs(12);
@@ -83,12 +76,12 @@ pub const BACKFILL_SCHEDULE_IN_SLOT: [(u32, u32); 3] = [
 
 /// Messages that the scheduler can receive.
 #[derive(AsRefStr)]
-pub enum ReprocessQueueMessage<T: BeaconChainTypes> {
+pub enum ReprocessQueueMessage {
     /// A block that has been received early and we should queue for later processing.
-    EarlyBlock(QueuedGossipBlock<T>),
+    EarlyBlock(QueuedGossipBlock),
     /// A gossip block for hash `X` is being imported, we should queue the rpc block for the same
     /// hash until the gossip block is imported.
-    RpcBlock(QueuedRpcBlock<T::EthSpec>),
+    RpcBlock(QueuedRpcBlock),
     /// A block that was successfully processed. We use this to handle attestations and light client updates
     /// for unknown blocks.
     BlockImported {
@@ -96,139 +89,127 @@ pub enum ReprocessQueueMessage<T: BeaconChainTypes> {
         parent_root: Hash256,
     },
     /// An unaggregated attestation that references an unknown block.
-    UnknownBlockUnaggregate(QueuedUnaggregate<T::EthSpec>),
+    UnknownBlockUnaggregate(QueuedUnaggregate),
     /// An aggregated attestation that references an unknown block.
-    UnknownBlockAggregate(QueuedAggregate<T::EthSpec>),
+    UnknownBlockAggregate(QueuedAggregate),
     /// A light client optimistic update that references a parent root that has not been seen as a parent.
-    UnknownLightClientOptimisticUpdate(QueuedLightClientUpdate<T::EthSpec>),
+    UnknownLightClientOptimisticUpdate(QueuedLightClientUpdate),
     /// A new backfill batch that needs to be scheduled for processing.
-    BackfillSync(QueuedBackfillBatch<T::EthSpec>),
+    BackfillSync(QueuedBackfillBatch),
 }
 
 /// Events sent by the scheduler once they are ready for re-processing.
-pub enum ReadyWork<T: BeaconChainTypes> {
+pub enum ReadyWork {
-    GossipBlock(QueuedGossipBlock<T>),
+    Block(QueuedGossipBlock),
-    RpcBlock(QueuedRpcBlock<T::EthSpec>),
+    RpcBlock(QueuedRpcBlock),
-    Unaggregate(QueuedUnaggregate<T::EthSpec>),
+    IgnoredRpcBlock(IgnoredRpcBlock),
-    Aggregate(QueuedAggregate<T::EthSpec>),
+    Unaggregate(QueuedUnaggregate),
-    LightClientUpdate(QueuedLightClientUpdate<T::EthSpec>),
+    Aggregate(QueuedAggregate),
-    BackfillSync(QueuedBackfillBatch<T::EthSpec>),
+    LightClientUpdate(QueuedLightClientUpdate),
+    BackfillSync(QueuedBackfillBatch),
 }
 
 /// An Attestation for which the corresponding block was not seen while processing, queued for
 /// later.
-pub struct QueuedUnaggregate<T: EthSpec> {
+pub struct QueuedUnaggregate {
-    pub peer_id: PeerId,
+    pub beacon_block_root: Hash256,
-    pub message_id: MessageId,
+    pub process_fn: BlockingFn,
-    pub attestation: Box<Attestation<T>>,
-    pub subnet_id: SubnetId,
-    pub should_import: bool,
-    pub seen_timestamp: Duration,
 }
 
 /// An aggregated attestation for which the corresponding block was not seen while processing, queued for
 /// later.
-pub struct QueuedAggregate<T: EthSpec> {
+pub struct QueuedAggregate {
-    pub peer_id: PeerId,
+    pub beacon_block_root: Hash256,
-    pub message_id: MessageId,
+    pub process_fn: BlockingFn,
-    pub attestation: Box<SignedAggregateAndProof<T>>,
-    pub seen_timestamp: Duration,
 }
 
 /// A light client update for which the corresponding parent block was not seen while processing,
 /// queued for later.
-pub struct QueuedLightClientUpdate<T: EthSpec> {
+pub struct QueuedLightClientUpdate {
-    pub peer_id: PeerId,
-    pub message_id: MessageId,
-    pub light_client_optimistic_update: Box<LightClientOptimisticUpdate<T>>,
     pub parent_root: Hash256,
-    pub seen_timestamp: Duration,
+    pub process_fn: BlockingFn,
 }
 
 /// A block that arrived early and has been queued for later import.
-pub struct QueuedGossipBlock<T: BeaconChainTypes> {
+pub struct QueuedGossipBlock {
-    pub peer_id: PeerId,
+    pub beacon_block_slot: Slot,
-    pub block: Box<GossipVerifiedBlock<T>>,
+    pub beacon_block_root: Hash256,
-    pub seen_timestamp: Duration,
+    pub process_fn: AsyncFn,
 }
 
 /// A block that arrived for processing when the same block was being imported over gossip.
 /// It is queued for later import.
-pub struct QueuedRpcBlock<T: EthSpec> {
+pub struct QueuedRpcBlock {
-    pub block_root: Hash256,
+    pub beacon_block_root: Hash256,
-    pub block: BlockWrapper<T>,
+    /// Processes/imports the block.
-    pub process_type: BlockProcessType,
+    pub process_fn: AsyncFn,
-    pub seen_timestamp: Duration,
+    /// Ignores the block.
-    /// Indicates if the beacon chain should process this block or not.
+    pub ignore_fn: BlockingFn,
-    /// We use this to ignore block processing when rpc block queues are full.
+}
-    pub should_process: bool,
+
+/// A block that arrived for processing when the same block was being imported over gossip.
+/// It is queued for later import.
+pub struct IgnoredRpcBlock {
+    pub process_fn: BlockingFn,
 }
 
 /// A backfill batch work that has been queued for processing later.
-#[derive(Clone)]
+pub struct QueuedBackfillBatch(pub AsyncFn);
-pub struct QueuedBackfillBatch<E: EthSpec> {
-    pub process_id: ChainSegmentProcessId,
-    pub blocks: Vec<BlockWrapper<E>>,
-}
 
-impl<T: BeaconChainTypes> TryFrom<WorkEvent<T>> for QueuedBackfillBatch<T::EthSpec> {
+impl<T: EthSpec> TryFrom<WorkEvent<T>> for QueuedBackfillBatch {
     type Error = WorkEvent<T>;
 
     fn try_from(event: WorkEvent<T>) -> Result<Self, WorkEvent<T>> {
         match event {
             WorkEvent {
-                work:
+                work: Work::ChainSegmentBackfill(process_fn),
-                    Work::ChainSegment {
-                        process_id: process_id @ ChainSegmentProcessId::BackSyncBatchId(_),
-                        blocks,
-                    },
                 ..
-            } => Ok(QueuedBackfillBatch { process_id, blocks }),
+            } => Ok(QueuedBackfillBatch(process_fn)),
             _ => Err(event),
         }
     }
 }
 
-impl<T: BeaconChainTypes> From<QueuedBackfillBatch<T::EthSpec>> for WorkEvent<T> {
+impl<T: EthSpec> From<QueuedBackfillBatch> for WorkEvent<T> {
-    fn from(queued_backfill_batch: QueuedBackfillBatch<T::EthSpec>) -> WorkEvent<T> {
+    fn from(queued_backfill_batch: QueuedBackfillBatch) -> WorkEvent<T> {
-        WorkEvent::chain_segment(
+        WorkEvent {
-            queued_backfill_batch.process_id,
+            drop_during_sync: false,
-            queued_backfill_batch.blocks,
+            work: Work::ChainSegmentBackfill(queued_backfill_batch.0),
-        )
+        }
     }
 }
 
 /// Unifies the different messages processed by the block delay queue.
-enum InboundEvent<T: BeaconChainTypes> {
+enum InboundEvent {
     /// A gossip block that was queued for later processing and is ready for import.
-    ReadyGossipBlock(QueuedGossipBlock<T>),
+    ReadyGossipBlock(QueuedGossipBlock),
     /// A rpc block that was queued because the same gossip block was being imported
     /// will now be retried for import.
-    ReadyRpcBlock(QueuedRpcBlock<T::EthSpec>),
+    ReadyRpcBlock(QueuedRpcBlock),
     /// An aggregated or unaggregated attestation is ready for re-processing.
     ReadyAttestation(QueuedAttestationId),
     /// A light client update that is ready for re-processing.
     ReadyLightClientUpdate(QueuedLightClientUpdateId),
     /// A backfill batch that was queued is ready for processing.
-    ReadyBackfillSync(QueuedBackfillBatch<T::EthSpec>),
+    ReadyBackfillSync(QueuedBackfillBatch),
     /// A `DelayQueue` returned an error.
     DelayQueueError(TimeError, &'static str),
     /// A message sent to the `ReprocessQueue`
-    Msg(ReprocessQueueMessage<T>),
+    Msg(ReprocessQueueMessage),
 }
 
 /// Manages scheduling works that need to be later re-processed.
-struct ReprocessQueue<T: BeaconChainTypes> {
+struct ReprocessQueue<S> {
     /// Receiver of messages relevant to schedule works for reprocessing.
-    work_reprocessing_rx: Receiver<ReprocessQueueMessage<T>>,
+    work_reprocessing_rx: Receiver<ReprocessQueueMessage>,
     /// Sender of works once they become ready
-    ready_work_tx: Sender<ReadyWork<T>>,
+    ready_work_tx: Sender<ReadyWork>,
 
     /* Queues */
     /// Queue to manage scheduled early blocks.
-    gossip_block_delay_queue: DelayQueue<QueuedGossipBlock<T>>,
+    gossip_block_delay_queue: DelayQueue<QueuedGossipBlock>,
     /// Queue to manage scheduled early blocks.
-    rpc_block_delay_queue: DelayQueue<QueuedRpcBlock<T::EthSpec>>,
+    rpc_block_delay_queue: DelayQueue<QueuedRpcBlock>,
     /// Queue to manage scheduled attestations.
     attestations_delay_queue: DelayQueue<QueuedAttestationId>,
     /// Queue to manage scheduled light client updates.
@@ -238,17 +219,17 @@ struct ReprocessQueue<T: BeaconChainTypes> {
     /// Queued blocks.
     queued_gossip_block_roots: HashSet<Hash256>,
     /// Queued aggregated attestations.
-    queued_aggregates: FnvHashMap<usize, (QueuedAggregate<T::EthSpec>, DelayKey)>,
+    queued_aggregates: FnvHashMap<usize, (QueuedAggregate, DelayKey)>,
     /// Queued attestations.
-    queued_unaggregates: FnvHashMap<usize, (QueuedUnaggregate<T::EthSpec>, DelayKey)>,
+    queued_unaggregates: FnvHashMap<usize, (QueuedUnaggregate, DelayKey)>,
     /// Attestations (aggregated and unaggregated) per root.
     awaiting_attestations_per_root: HashMap<Hash256, Vec<QueuedAttestationId>>,
     /// Queued Light Client Updates.
-    queued_lc_updates: FnvHashMap<usize, (QueuedLightClientUpdate<T::EthSpec>, DelayKey)>,
+    queued_lc_updates: FnvHashMap<usize, (QueuedLightClientUpdate, DelayKey)>,
     /// Light Client Updates per parent_root.
     awaiting_lc_updates_per_parent_root: HashMap<Hash256, Vec<QueuedLightClientUpdateId>>,
     /// Queued backfill batches
-    queued_backfill_batches: Vec<QueuedBackfillBatch<T::EthSpec>>,
+    queued_backfill_batches: Vec<QueuedBackfillBatch>,
 
     /* Aux */
     /// Next attestation id, used for both aggregated and unaggregated attestations
@@ -259,7 +240,7 @@ struct ReprocessQueue<T: BeaconChainTypes> {
     attestation_delay_debounce: TimeLatch,
     lc_update_delay_debounce: TimeLatch,
     next_backfill_batch_event: Option<Pin<Box<tokio::time::Sleep>>>,
-    slot_clock: Pin<Box<T::SlotClock>>,
+    slot_clock: Pin<Box<S>>,
 }
 
 pub type QueuedLightClientUpdateId = usize;
@@ -270,20 +251,20 @@ enum QueuedAttestationId {
     Unaggregate(usize),
 }
 
-impl<T: EthSpec> QueuedAggregate<T> {
+impl QueuedAggregate {
     pub fn beacon_block_root(&self) -> &Hash256 {
-        &self.attestation.message.aggregate.data.beacon_block_root
+        &self.beacon_block_root
     }
 }
 
-impl<T: EthSpec> QueuedUnaggregate<T> {
+impl QueuedUnaggregate {
     pub fn beacon_block_root(&self) -> &Hash256 {
-        &self.attestation.data.beacon_block_root
+        &self.beacon_block_root
     }
 }
 
-impl<T: BeaconChainTypes> Stream for ReprocessQueue<T> {
+impl<S: SlotClock> Stream for ReprocessQueue<S> {
-    type Item = InboundEvent<T>;
+    type Item = InboundEvent;
 
     fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
         // NOTE: implementing `Stream` is not necessary but allows to maintain the future selection
@@ -374,16 +355,13 @@ impl<T: BeaconChainTypes> Stream for ReprocessQueue<T> {
 /// Starts the job that manages scheduling works that need re-processing. The returned `Sender`
 /// gives the communicating channel to receive those works. Once a work is ready, it is sent back
 /// via `ready_work_tx`.
-pub fn spawn_reprocess_scheduler<T: BeaconChainTypes>(
+pub fn spawn_reprocess_scheduler<S: SlotClock + 'static>(
-    ready_work_tx: Sender<ReadyWork<T>>,
+    ready_work_tx: Sender<ReadyWork>,
+    work_reprocessing_rx: Receiver<ReprocessQueueMessage>,
     executor: &TaskExecutor,
-    slot_clock: T::SlotClock,
+    slot_clock: S,
     log: Logger,
-) -> Sender<ReprocessQueueMessage<T>> {
+) {
-    let (work_reprocessing_tx, work_reprocessing_rx) = mpsc::channel(MAX_SCHEDULED_WORK_QUEUE_LEN);
-    // Basic sanity check.
-    assert!(ADDITIONAL_QUEUED_BLOCK_DELAY < MAXIMUM_GOSSIP_CLOCK_DISPARITY);
-
     let mut queue = ReprocessQueue {
         work_reprocessing_rx,
         ready_work_tx,
@@ -422,19 +400,17 @@ pub fn spawn_reprocess_scheduler<T: BeaconChainTypes>(
         },
         TASK_NAME,
     );
-
-    work_reprocessing_tx
 }
 
-impl<T: BeaconChainTypes> ReprocessQueue<T> {
+impl<S: SlotClock> ReprocessQueue<S> {
-    fn handle_message(&mut self, msg: InboundEvent<T>, slot_clock: &T::SlotClock, log: &Logger) {
+    fn handle_message(&mut self, msg: InboundEvent, slot_clock: &S, log: &Logger) {
         use ReprocessQueueMessage::*;
         match msg {
             // Some block has been indicated as "early" and should be processed when the
             // appropriate slot arrives.
             InboundEvent::Msg(EarlyBlock(early_block)) => {
-                let block_slot = early_block.block.block.slot();
+                let block_slot = early_block.beacon_block_slot;
-                let block_root = early_block.block.block_root;
+                let block_root = early_block.beacon_block_root;
 
                 // Don't add the same block to the queue twice. This prevents DoS attacks.
                 if self.queued_gossip_block_roots.contains(&block_root) {
@@ -478,7 +454,7 @@ impl<T: BeaconChainTypes> ReprocessQueue<T> {
                         if block_slot <= now
                             && self
                                 .ready_work_tx
-                                .try_send(ReadyWork::GossipBlock(early_block))
+                                .try_send(ReadyWork::Block(early_block))
                                 .is_err()
                         {
                             error!(
@@ -493,7 +469,7 @@ impl<T: BeaconChainTypes> ReprocessQueue<T> {
             // for the same block hash is being imported. We wait for `QUEUED_RPC_BLOCK_DELAY`
             // and then send the rpc block back for processing assuming the gossip import
             // has completed by then.
-            InboundEvent::Msg(RpcBlock(mut rpc_block)) => {
+            InboundEvent::Msg(RpcBlock(rpc_block)) => {
                 // Check to ensure this won't over-fill the queue.
                 if self.rpc_block_delay_queue.len() >= MAXIMUM_QUEUED_BLOCKS {
                     if self.rpc_block_debounce.elapsed() {
@@ -506,10 +482,11 @@ impl<T: BeaconChainTypes> ReprocessQueue<T> {
                     }
                     // Return the block to the beacon processor signalling to
                     // ignore processing for this block
-                    rpc_block.should_process = false;
                     if self
                         .ready_work_tx
-                        .try_send(ReadyWork::RpcBlock(rpc_block))
+                        .try_send(ReadyWork::IgnoredRpcBlock(IgnoredRpcBlock {
+                            process_fn: rpc_block.ignore_fn,
+                        }))
                         .is_err()
                     {
                         error!(
@@ -528,7 +505,7 @@ impl<T: BeaconChainTypes> ReprocessQueue<T> {
                 debug!(
                     log,
                     "Sending rpc block for reprocessing";
-                    "block_root" => %queued_rpc_block.block_root
+                    "block_root" => %queued_rpc_block.beacon_block_root
                 );
                 if self
                     .ready_work_tx
@@ -766,7 +743,7 @@ impl<T: BeaconChainTypes> ReprocessQueue<T> {
             }
             // A block that was queued for later processing is now ready to be processed.
             InboundEvent::ReadyGossipBlock(ready_block) => {
-                let block_root = ready_block.block.block_root;
+                let block_root = ready_block.beacon_block_root;
 
                 if !self.queued_gossip_block_roots.remove(&block_root) {
                     // Log an error to alert that we've made a bad assumption about how this
@@ -780,7 +757,7 @@ impl<T: BeaconChainTypes> ReprocessQueue<T> {
 
                 if self
                     .ready_work_tx
-                    .try_send(ReadyWork::GossipBlock(ready_block))
+                    .try_send(ReadyWork::Block(ready_block))
                     .is_err()
                 {
                     error!(
@@ -884,18 +861,28 @@ impl<T: BeaconChainTypes> ReprocessQueue<T> {
                     "millis_from_slot_start" => millis_from_slot_start
                 );
 
-                if self
+                match self
                     .ready_work_tx
-                    .try_send(ReadyWork::BackfillSync(queued_backfill_batch.clone()))
+                    .try_send(ReadyWork::BackfillSync(queued_backfill_batch))
-                    .is_err()
                 {
-                    error!(
+                    // The message was sent successfully.
+                    Ok(()) => (),
+                    // The message was not sent, recover it from the returned `Err`.
+                    Err(mpsc::error::TrySendError::Full(ReadyWork::BackfillSync(batch)))
+                    | Err(mpsc::error::TrySendError::Closed(ReadyWork::BackfillSync(batch))) => {
+                        error!(
+                            log,
+                            "Failed to send scheduled backfill work";
+                            "info" => "sending work back to queue"
+                        );
+                        self.queued_backfill_batches.insert(0, batch)
+                    }
+                    // The message was not sent and we didn't get the correct
+                    // return result. This is a logic error.
+                    _ => crit!(
                         log,
-                        "Failed to send scheduled backfill work";
+                        "Unexpected return from try_send error";
-                        "info" => "sending work back to queue"
+                    ),
-                    );
-                    self.queued_backfill_batches
-                        .insert(0, queued_backfill_batch);
                 }
             }
         }
@@ -926,7 +913,7 @@ impl<T: BeaconChainTypes> ReprocessQueue<T> {
         // only recompute the `next_backfill_batch_event` if there are backfill batches in the queue
         if !self.queued_backfill_batches.is_empty() {
             self.next_backfill_batch_event = Some(Box::pin(tokio::time::sleep(
-                ReprocessQueue::<T>::duration_until_next_backfill_batch_event(&self.slot_clock),
+                ReprocessQueue::<S>::duration_until_next_backfill_batch_event(&self.slot_clock),
             )));
         } else {
             self.next_backfill_batch_event = None
@@ -935,7 +922,7 @@ impl<T: BeaconChainTypes> ReprocessQueue<T> {
 
     /// Returns duration until the next scheduled processing time. The schedule ensure that backfill
     /// processing is done in windows of time that aren't critical
-    fn duration_until_next_backfill_batch_event(slot_clock: &T::SlotClock) -> Duration {
+    fn duration_until_next_backfill_batch_event(slot_clock: &S) -> Duration {
         let slot_duration = slot_clock.slot_duration();
         slot_clock
             .millis_from_current_slot_start()
@@ -965,16 +952,9 @@ impl<T: BeaconChainTypes> ReprocessQueue<T> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use beacon_chain::builder::Witness;
-    use beacon_chain::eth1_chain::CachingEth1Backend;
     use slot_clock::TestingSlotClock;
-    use store::MemoryStore;
-    use types::MainnetEthSpec as E;
     use types::Slot;
 
-    type TestBeaconChainType =
-        Witness<TestingSlotClock, CachingEth1Backend<E>, E, MemoryStore<E>, MemoryStore<E>>;
-
     #[test]
     fn backfill_processing_schedule_calculation() {
         let slot_duration = Duration::from_secs(12);
@@ -987,7 +967,7 @@ mod tests {
 
         for &event_duration_from_slot_start in event_times.iter() {
             let duration_to_next_event =
-                ReprocessQueue::<TestBeaconChainType>::duration_until_next_backfill_batch_event(
+                ReprocessQueue::<TestingSlotClock>::duration_until_next_backfill_batch_event(
                     &slot_clock,
                 );
 
@@ -1004,7 +984,7 @@ mod tests {
         // check for next event beyond the current slot
         let duration_to_next_slot = slot_clock.duration_to_next_slot().unwrap();
         let duration_to_next_event =
-            ReprocessQueue::<TestBeaconChainType>::duration_until_next_backfill_batch_event(
+            ReprocessQueue::<TestingSlotClock>::duration_until_next_backfill_batch_event(
                 &slot_clock,
             );
         assert_eq!(

beacon_node/client/Cargo.toml

@@ -43,3 +43,5 @@ slasher = { path = "../../slasher" }
 slasher_service = { path = "../../slasher/service" }
 monitoring_api = {path = "../../common/monitoring_api"}
 execution_layer = { path = "../execution_layer" }
+beacon_processor =  { path = "../beacon_processor" }
+num_cpus = "1.13.0"

beacon_node/client/src/builder.rs

@@ -12,7 +12,11 @@ use beacon_chain::{
     slot_clock::{SlotClock, SystemTimeSlotClock},
     state_advance_timer::spawn_state_advance_timer,
     store::{HotColdDB, ItemStore, LevelDB, StoreConfig},
-    BeaconChain, BeaconChainTypes, Eth1ChainBackend, ServerSentEventHandler,
+    BeaconChain, BeaconChainTypes, Eth1ChainBackend, MigratorConfig, ServerSentEventHandler,
+};
+use beacon_processor::{
+    work_reprocessing_queue::ReprocessQueueMessage, BeaconProcessor, BeaconProcessorSend,
+    WorkEvent, MAX_SCHEDULED_WORK_QUEUE_LEN, MAX_WORK_EVENT_QUEUE_LEN,
 };
 use environment::RuntimeContext;
 use eth1::{Config as Eth1Config, Service as Eth1Service};
@@ -29,12 +33,13 @@ use slasher::Slasher;
 use slasher_service::SlasherService;
 use slog::{debug, info, warn, Logger};
 use state_processing::per_slot_processing;
+use std::cmp;
 use std::net::TcpListener;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::Duration;
 use timer::spawn_timer;
-use tokio::sync::oneshot;
+use tokio::sync::{mpsc, oneshot};
 use types::{
     test_utils::generate_deterministic_keypairs, BeaconState, ChainSpec, EthSpec,
     ExecutionBlockHash, Hash256, SignedBeaconBlock,
@@ -75,6 +80,10 @@ pub struct ClientBuilder<T: BeaconChainTypes> {
     http_metrics_config: http_metrics::Config,
     slasher: Option<Arc<Slasher<T::EthSpec>>>,
     eth_spec_instance: T::EthSpec,
+    beacon_processor_send: BeaconProcessorSend<T::EthSpec>,
+    beacon_processor_receive: mpsc::Receiver<WorkEvent<T::EthSpec>>,
+    work_reprocessing_tx: mpsc::Sender<ReprocessQueueMessage>,
+    work_reprocessing_rx: mpsc::Receiver<ReprocessQueueMessage>,
 }
 
 impl<TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>
@@ -90,6 +99,10 @@ where
     ///
     /// The `eth_spec_instance` parameter is used to concretize `TEthSpec`.
     pub fn new(eth_spec_instance: TEthSpec) -> Self {
+        let (beacon_processor_send, beacon_processor_receive) =
+            mpsc::channel(MAX_WORK_EVENT_QUEUE_LEN);
+        let (work_reprocessing_tx, work_reprocessing_rx) =
+            mpsc::channel(MAX_SCHEDULED_WORK_QUEUE_LEN);
         Self {
             slot_clock: None,
             store: None,
@@ -108,6 +121,10 @@ where
             http_metrics_config: <_>::default(),
             slasher: None,
             eth_spec_instance,
+            beacon_processor_send: BeaconProcessorSend(beacon_processor_send),
+            beacon_processor_receive,
+            work_reprocessing_tx,
+            work_reprocessing_rx,
         }
     }
 
@@ -171,6 +188,9 @@ where
             .store(store)
             .task_executor(context.executor.clone())
             .custom_spec(spec.clone())
+            .store_migrator_config(
+                MigratorConfig::default().epochs_per_migration(chain_config.epochs_per_migration),
+            )
             .chain_config(chain_config)
             .graffiti(graffiti)
             .event_handler(event_handler)
@@ -551,6 +571,8 @@ where
             gossipsub_registry
                 .as_mut()
                 .map(|registry| registry.sub_registry_with_prefix("gossipsub")),
+            self.beacon_processor_send.clone(),
+            self.work_reprocessing_tx.clone(),
         )
         .await
         .map_err(|e| format!("Failed to start network: {:?}", e))?;
@@ -738,6 +760,27 @@ where
         }
 
         if let Some(beacon_chain) = self.beacon_chain.as_ref() {
+            if let Some(network_globals) = &self.network_globals {
+                let beacon_processor_context = runtime_context.service_context("bproc".into());
+                BeaconProcessor {
+                    network_globals: network_globals.clone(),
+                    executor: beacon_processor_context.executor.clone(),
+                    max_workers: cmp::max(1, num_cpus::get()),
+                    current_workers: 0,
+                    enable_backfill_rate_limiting: beacon_chain
+                        .config
+                        .enable_backfill_rate_limiting,
+                    log: beacon_processor_context.log().clone(),
+                }
+                .spawn_manager(
+                    self.beacon_processor_receive,
+                    self.work_reprocessing_tx,
+                    self.work_reprocessing_rx,
+                    None,
+                    beacon_chain.slot_clock.clone(),
+                );
+            }
+
             let state_advance_context = runtime_context.service_context("state_advance".into());
             let state_advance_log = state_advance_context.log().clone();
             spawn_state_advance_timer(

beacon_node/execution_layer/Cargo.toml

@@ -42,9 +42,9 @@ lazy_static = "1.4.0"
 ethers-core = "1.0.2"
 builder_client = { path = "../builder_client" }
 fork_choice = { path = "../../consensus/fork_choice" }
-mev-rs = { git = "https://github.com/ralexstokes/mev-rs" }
+mev-rs = { git = "https://github.com/ralexstokes/mev-rs", rev = "216657016d5c0889b505857c89ae42c7aa2764af" }
-ethereum-consensus = { git = "https://github.com/ralexstokes/ethereum-consensus" }
+ethereum-consensus = { git = "https://github.com/ralexstokes/ethereum-consensus", rev = "e380108" }
-ssz-rs = { git = "https://github.com/ralexstokes/ssz-rs" }
+ssz_rs = "0.9.0"
 tokio-stream = { version = "0.1.9", features = [ "sync" ] }
 strum = "0.24.0"
 keccak-hash = "0.10.0"

beacon_node/execution_layer/src/lib.rs

@@ -42,13 +42,14 @@ use tokio_stream::wrappers::WatchStream;
 use tree_hash::TreeHash;
 use types::beacon_block_body::KzgCommitments;
 use types::blob_sidecar::Blobs;
-use types::KzgProofs;
-use types::{AbstractExecPayload, BeaconStateError, ExecPayload, VersionedHash};
 use types::{
-    BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionPayloadCapella, ExecutionPayloadDeneb,
+    AbstractExecPayload, BeaconStateError, ExecPayload, ExecutionPayloadDeneb, VersionedHash,
-    ExecutionPayloadMerge,
 };
-use types::{ProposerPreparationData, PublicKeyBytes, Signature, Slot, Transaction};
+use types::{
+    BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionPayloadCapella, ExecutionPayloadMerge,
+};
+use types::{KzgProofs, Withdrawals};
+use types::{ProposerPreparationData, PublicKeyBytes, Signature, Slot, Transaction, Uint256};
 
 mod block_hash;
 mod engine_api;
@@ -2125,6 +2126,15 @@ async fn timed_future<F: Future<Output = T>, T>(metric: &str, future: F) -> (T,
     (result, duration)
 }
 
+#[cfg(test)]
+/// Returns the duration since the unix epoch.
+fn timestamp_now() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_else(|_| Duration::from_secs(0))
+        .as_secs()
+}
+
 #[derive(Debug)]
 pub enum BlobTxConversionError {
     /// The transaction type was not set.
@@ -2355,12 +2365,3 @@ mod test {
             .await;
     }
 }
-
-#[cfg(test)]
-/// Returns the duration since the unix epoch.
-fn timestamp_now() -> u64 {
-    SystemTime::now()
-        .duration_since(UNIX_EPOCH)
-        .unwrap_or_else(|_| Duration::from_secs(0))
-        .as_secs()
-}

beacon_node/execution_layer/src/test_utils/mock_builder.rs

@@ -11,11 +11,17 @@ use ethereum_consensus::{
 };
 use fork_choice::ForkchoiceUpdateParameters;
 use mev_rs::{
-    bellatrix::{BuilderBid as BuilderBidBellatrix, SignedBuilderBid as SignedBuilderBidBellatrix},
+    blinded_block_provider::Server as BlindedBlockProviderServer,
-    capella::{BuilderBid as BuilderBidCapella, SignedBuilderBid as SignedBuilderBidCapella},
+    signing::{sign_builder_message, verify_signed_builder_message},
-    sign_builder_message, verify_signed_builder_message, BidRequest, BlindedBlockProviderError,
+    types::{
-    BlindedBlockProviderServer, BuilderBid, ExecutionPayload as ServerPayload,
+        bellatrix::{
-    SignedBlindedBeaconBlock, SignedBuilderBid, SignedValidatorRegistration,
+            BuilderBid as BuilderBidBellatrix, SignedBuilderBid as SignedBuilderBidBellatrix,
+        },
+        capella::{BuilderBid as BuilderBidCapella, SignedBuilderBid as SignedBuilderBidCapella},
+        BidRequest, BuilderBid, ExecutionPayload as ServerPayload, SignedBlindedBeaconBlock,
+        SignedBuilderBid, SignedValidatorRegistration,
+    },
+    Error as MevError,
 };
 use parking_lot::RwLock;
 use sensitive_url::SensitiveUrl;
@@ -47,7 +53,7 @@ pub enum Operation {
 }
 
 impl Operation {
-    fn apply<B: BidStuff>(self, bid: &mut B) -> Result<(), BlindedBlockProviderError> {
+    fn apply<B: BidStuff>(self, bid: &mut B) -> Result<(), MevError> {
         match self {
             Operation::FeeRecipient(fee_recipient) => {
                 *bid.fee_recipient_mut() = to_ssz_rs(&fee_recipient)?
@@ -73,7 +79,7 @@ pub trait BidStuff {
     fn prev_randao_mut(&mut self) -> &mut Hash32;
     fn block_number_mut(&mut self) -> &mut u64;
     fn timestamp_mut(&mut self) -> &mut u64;
-    fn withdrawals_root_mut(&mut self) -> Result<&mut Root, BlindedBlockProviderError>;
+    fn withdrawals_root_mut(&mut self) -> Result<&mut Root, MevError>;
 
     fn sign_builder_message(
         &mut self,
@@ -134,11 +140,9 @@ impl BidStuff for BuilderBid {
         }
     }
 
-    fn withdrawals_root_mut(&mut self) -> Result<&mut Root, BlindedBlockProviderError> {
+    fn withdrawals_root_mut(&mut self) -> Result<&mut Root, MevError> {
         match self {
-            Self::Bellatrix(_) => Err(BlindedBlockProviderError::Custom(
+            Self::Bellatrix(_) => Err(MevError::InvalidFork),
-                "withdrawals_root called on bellatrix bid".to_string(),
-            )),
             Self::Capella(bid) => Ok(&mut bid.header.withdrawals_root),
         }
     }
@@ -274,7 +278,7 @@ impl<E: EthSpec> MockBuilder<E> {
         *self.invalidate_signatures.write() = false;
     }
 
-    fn apply_operations<B: BidStuff>(&self, bid: &mut B) -> Result<(), BlindedBlockProviderError> {
+    fn apply_operations<B: BidStuff>(&self, bid: &mut B) -> Result<(), MevError> {
         let mut guard = self.operations.write();
         while let Some(op) = guard.pop() {
             op.apply(bid)?;
@@ -288,7 +292,7 @@ impl<E: EthSpec> mev_rs::BlindedBlockProvider for MockBuilder<E> {
     async fn register_validators(
         &self,
         registrations: &mut [SignedValidatorRegistration],
-    ) -> Result<(), BlindedBlockProviderError> {
+    ) -> Result<(), MevError> {
         for registration in registrations {
             let pubkey = registration.message.public_key.clone();
             let message = &mut registration.message;
@@ -307,10 +311,7 @@ impl<E: EthSpec> mev_rs::BlindedBlockProvider for MockBuilder<E> {
         Ok(())
     }
 
-    async fn fetch_best_bid(
+    async fn fetch_best_bid(&self, bid_request: &BidRequest) -> Result<SignedBuilderBid, MevError> {
-        &self,
-        bid_request: &BidRequest,
-    ) -> Result<SignedBuilderBid, BlindedBlockProviderError> {
         let slot = Slot::new(bid_request.slot);
         let fork = self.spec.fork_name_at_slot::<E>(slot);
         let signed_cached_data = self
@@ -336,7 +337,7 @@ impl<E: EthSpec> mev_rs::BlindedBlockProvider for MockBuilder<E> {
             .map_err(convert_err)?
             .block_hash();
         if head_execution_hash != from_ssz_rs(&bid_request.parent_hash)? {
-            return Err(BlindedBlockProviderError::Custom(format!(
+            return Err(custom_err(format!(
                 "head mismatch: {} {}",
                 head_execution_hash, bid_request.parent_hash
             )));
@@ -396,7 +397,7 @@ impl<E: EthSpec> mev_rs::BlindedBlockProvider for MockBuilder<E> {
             .get_debug_beacon_states(StateId::Head)
             .await
             .map_err(convert_err)?
-            .ok_or_else(|| BlindedBlockProviderError::Custom("missing head state".to_string()))?
+            .ok_or_else(|| custom_err("missing head state".to_string()))?
             .data;
         let prev_randao = head_state
             .get_randao_mix(head_state.current_epoch())
@@ -409,10 +410,7 @@ impl<E: EthSpec> mev_rs::BlindedBlockProvider for MockBuilder<E> {
                 PayloadAttributes::new(timestamp, *prev_randao, fee_recipient, Some(vec![]))
             }
             ForkName::Base | ForkName::Altair => {
-                return Err(BlindedBlockProviderError::Custom(format!(
+                return Err(MevError::InvalidFork);
-                    "Unsupported fork: {}",
-                    fork
-                )));
             }
         };
 
@@ -453,10 +451,7 @@ impl<E: EthSpec> mev_rs::BlindedBlockProvider for MockBuilder<E> {
                 public_key: self.builder_sk.public_key(),
             }),
             ForkName::Base | ForkName::Altair | ForkName::Deneb => {
-                return Err(BlindedBlockProviderError::Custom(format!(
+                return Err(MevError::InvalidFork)
-                    "Unsupported fork: {}",
-                    fork
-                )))
             }
         };
         *message.gas_limit_mut() = cached_data.gas_limit;
@@ -475,7 +470,7 @@ impl<E: EthSpec> mev_rs::BlindedBlockProvider for MockBuilder<E> {
     async fn open_bid(
         &self,
         signed_block: &mut SignedBlindedBeaconBlock,
-    ) -> Result<ServerPayload, BlindedBlockProviderError> {
+    ) -> Result<ServerPayload, MevError> {
         let node = match signed_block {
             SignedBlindedBeaconBlock::Bellatrix(block) => {
                 block.message.body.execution_payload_header.hash_tree_root()
@@ -496,9 +491,7 @@ impl<E: EthSpec> mev_rs::BlindedBlockProvider for MockBuilder<E> {
     }
 }
 
-pub fn from_ssz_rs<T: SimpleSerialize, U: Decode>(
+pub fn from_ssz_rs<T: SimpleSerialize, U: Decode>(ssz_rs_data: &T) -> Result<U, MevError> {
-    ssz_rs_data: &T,
-) -> Result<U, BlindedBlockProviderError> {
     U::from_ssz_bytes(
         ssz_rs::serialize(ssz_rs_data)
             .map_err(convert_err)?
@@ -507,12 +500,17 @@ pub fn from_ssz_rs<T: SimpleSerialize, U: Decode>(
     .map_err(convert_err)
 }
 
-pub fn to_ssz_rs<T: Encode, U: SimpleSerialize>(
+pub fn to_ssz_rs<T: Encode, U: SimpleSerialize>(ssz_data: &T) -> Result<U, MevError> {
-    ssz_data: &T,
-) -> Result<U, BlindedBlockProviderError> {
     ssz_rs::deserialize::<U>(&ssz_data.as_ssz_bytes()).map_err(convert_err)
 }
 
-fn convert_err<E: Debug>(e: E) -> BlindedBlockProviderError {
+fn convert_err<E: Debug>(e: E) -> MevError {
-    BlindedBlockProviderError::Custom(format!("{e:?}"))
+    custom_err(format!("{e:?}"))
+}
+
+// This is a bit of a hack since the `Custom` variant was removed from `mev_rs::Error`.
+fn custom_err(s: String) -> MevError {
+    MevError::Consensus(ethereum_consensus::state_transition::Error::Io(
+        std::io::Error::new(std::io::ErrorKind::Other, s),
+    ))
 }

beacon_node/http_api/src/block_packing_efficiency.rs

@@ -75,7 +75,7 @@ impl<T: EthSpec> PackingEfficiencyHandler<T> {
             available_attestations: HashSet::new(),
             included_attestations: HashMap::new(),
             committee_store: CommitteeStore::new(),
-            _phantom: PhantomData::default(),
+            _phantom: PhantomData,
         };
 
         handler.compute_epoch(start_epoch, &starting_state, spec)?;

beacon_node/http_api/src/state_id.rs

@@ -70,15 +70,32 @@ impl StateId {
                     .map_err(BeaconChainError::DBError)
                     .map_err(warp_utils::reject::beacon_chain_error)?
                 {
-                    let execution_optimistic = chain
+                    let finalization_status = chain
-                        .canonical_head
+                        .state_finalization_and_canonicity(root, hot_summary.slot)
-                        .fork_choice_read_lock()
-                        .is_optimistic_or_invalid_block_no_fallback(&hot_summary.latest_block_root)
-                        .map_err(BeaconChainError::ForkChoiceError)
-                        .map_err(warp_utils::reject::beacon_chain_error)?;
-                    let finalized = chain
-                        .is_finalized_state(root, hot_summary.slot)
                         .map_err(warp_utils::reject::beacon_chain_error)?;
+                    let finalized = finalization_status.is_finalized();
+                    let fork_choice = chain.canonical_head.fork_choice_read_lock();
+                    let execution_optimistic = if finalization_status.slot_is_finalized
+                        && !finalization_status.canonical
+                    {
+                        // This block is permanently orphaned and has likely been pruned from fork
+                        // choice. If it isn't found in fork choice, mark it optimistic to be on the
+                        // safe side.
+                        fork_choice
+                            .is_optimistic_or_invalid_block_no_fallback(
+                                &hot_summary.latest_block_root,
+                            )
+                            .unwrap_or(true)
+                    } else {
+                        // This block is either old and finalized, or recent and unfinalized, so
+                        // it's safe to fallback to the optimistic status of the finalized block.
+                        chain
+                            .canonical_head
+                            .fork_choice_read_lock()
+                            .is_optimistic_or_invalid_block(&hot_summary.latest_block_root)
+                            .map_err(BeaconChainError::ForkChoiceError)
+                            .map_err(warp_utils::reject::beacon_chain_error)?
+                    };
                     return Ok((*root, execution_optimistic, finalized));
                 } else if let Some(_cold_state_slot) = chain
                     .store

beacon_node/http_api/tests/fork_tests.rs

@@ -327,11 +327,8 @@ async fn sync_committee_indices_across_fork() {
 
 /// Assert that an HTTP API error has the given status code and indexed errors for the given indices.
 fn assert_server_indexed_error(error: eth2::Error, status_code: u16, indices: Vec<usize>) {
-    let eth2::Error::ServerIndexedMessage(IndexedErrorMessage {
+    let eth2::Error::ServerIndexedMessage(IndexedErrorMessage { code, failures, .. }) = error
-        code,
+    else {
-        failures,
-        ..
-    }) = error else {
         panic!("wrong error, expected ServerIndexedMessage, got: {error:?}")
     };
     assert_eq!(code, status_code);

beacon_node/http_api/tests/interactive_tests.rs

@@ -2,8 +2,9 @@
 use beacon_chain::{
     chain_config::{DisallowedReOrgOffsets, ReOrgThreshold},
     test_utils::{AttestationStrategy, BlockStrategy, SyncCommitteeStrategy},
+    ChainConfig,
 };
-use eth2::types::DepositContractData;
+use eth2::types::{DepositContractData, StateId};
 use execution_layer::{ForkchoiceState, PayloadAttributes};
 use http_api::test_utils::InteractiveTester;
 use parking_lot::Mutex;
@@ -17,7 +18,7 @@ use std::time::Duration;
 use tree_hash::TreeHash;
 use types::{
     Address, Epoch, EthSpec, ExecPayload, ExecutionBlockHash, ForkName, FullPayload,
-    MainnetEthSpec, ProposerPreparationData, Slot,
+    MainnetEthSpec, MinimalEthSpec, ProposerPreparationData, Slot,
 };
 
 type E = MainnetEthSpec;
@@ -48,6 +49,76 @@ async fn deposit_contract_custom_network() {
     assert_eq!(result, expected);
 }
 
+// Test that state lookups by root function correctly for states that are finalized but still
+// present in the hot database, and have had their block pruned from fork choice.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn state_by_root_pruned_from_fork_choice() {
+    type E = MinimalEthSpec;
+
+    let validator_count = 24;
+    let spec = ForkName::latest().make_genesis_spec(E::default_spec());
+
+    let tester = InteractiveTester::<E>::new_with_initializer_and_mutator(
+        Some(spec.clone()),
+        validator_count,
+        Some(Box::new(move |builder| {
+            builder
+                .deterministic_keypairs(validator_count)
+                .fresh_ephemeral_store()
+                .chain_config(ChainConfig {
+                    epochs_per_migration: 1024,
+                    ..ChainConfig::default()
+                })
+        })),
+        None,
+    )
+    .await;
+
+    let client = &tester.client;
+    let harness = &tester.harness;
+
+    // Create some chain depth and finalize beyond fork choice's pruning depth.
+    let num_epochs = 8_u64;
+    let num_initial = num_epochs * E::slots_per_epoch();
+    harness.advance_slot();
+    harness
+        .extend_chain_with_sync(
+            num_initial as usize,
+            BlockStrategy::OnCanonicalHead,
+            AttestationStrategy::AllValidators,
+            SyncCommitteeStrategy::NoValidators,
+        )
+        .await;
+
+    // Should now be finalized.
+    let finalized_epoch = harness.finalized_checkpoint().epoch;
+    assert_eq!(finalized_epoch, num_epochs - 2);
+
+    // The split slot should still be at 0.
+    assert_eq!(harness.chain.store.get_split_slot(), 0);
+
+    // States that are between the split and the finalized slot should be able to be looked up by
+    // state root.
+    for slot in 0..finalized_epoch.start_slot(E::slots_per_epoch()).as_u64() {
+        let state_root = harness
+            .chain
+            .state_root_at_slot(Slot::new(slot))
+            .unwrap()
+            .unwrap();
+        let response = client
+            .get_debug_beacon_states::<E>(StateId::Root(state_root))
+            .await
+            .unwrap()
+            .unwrap();
+
+        assert!(response.finalized.unwrap());
+        assert!(!response.execution_optimistic.unwrap());
+
+        let mut state = response.data;
+        assert_eq!(state.update_tree_hash_cache().unwrap(), state_root);
+    }
+}
+
 /// Data structure for tracking fork choice updates received by the mock execution layer.
 #[derive(Debug, Default)]
 struct ForkChoiceUpdates {

beacon_node/lighthouse_network/src/discovery/enr_ext.rs

@@ -1,7 +1,10 @@
 //! ENR extension trait to support libp2p integration.
 use crate::{Enr, Multiaddr, PeerId};
 use discv5::enr::{CombinedKey, CombinedPublicKey};
-use libp2p::core::{identity::Keypair, identity::PublicKey, multiaddr::Protocol};
+use libp2p::{
+    core::{identity::Keypair, identity::PublicKey, multiaddr::Protocol},
+    identity::secp256k1,
+};
 use tiny_keccak::{Hasher, Keccak};
 
 /// Extend ENR for libp2p types.
@@ -36,6 +39,8 @@ pub trait CombinedKeyPublicExt {
 pub trait CombinedKeyExt {
     /// Converts a libp2p key into an ENR combined key.
     fn from_libp2p(key: &libp2p::core::identity::Keypair) -> Result<CombinedKey, &'static str>;
+    /// Converts a [`secp256k1::Keypair`] into and Enr [`CombinedKey`].
+    fn from_secp256k1(key: &secp256k1::Keypair) -> CombinedKey;
 }
 
 impl EnrExt for Enr {
@@ -220,12 +225,7 @@ impl CombinedKeyPublicExt for CombinedPublicKey {
 impl CombinedKeyExt for CombinedKey {
     fn from_libp2p(key: &libp2p::core::identity::Keypair) -> Result<CombinedKey, &'static str> {
         match key {
-            Keypair::Secp256k1(key) => {
+            Keypair::Secp256k1(key) => Ok(CombinedKey::from_secp256k1(key)),
-                let secret =
-                    discv5::enr::k256::ecdsa::SigningKey::from_slice(&key.secret().to_bytes())
-                        .expect("libp2p key must be valid");
-                Ok(CombinedKey::Secp256k1(secret))
-            }
             Keypair::Ed25519(key) => {
                 let ed_keypair = discv5::enr::ed25519_dalek::SigningKey::from_bytes(
                     &(key.encode()[..32])
@@ -237,6 +237,11 @@ impl CombinedKeyExt for CombinedKey {
             Keypair::Ecdsa(_) => Err("Ecdsa keypairs not supported"),
         }
     }
+    fn from_secp256k1(key: &secp256k1::Keypair) -> Self {
+        let secret = discv5::enr::k256::ecdsa::SigningKey::from_slice(&key.secret().to_bytes())
+            .expect("libp2p key must be valid");
+        CombinedKey::Secp256k1(secret)
+    }
 }
 
 // helper function to convert a peer_id to a node_id. This is only possible for secp256k1/ed25519 libp2p

beacon_node/lighthouse_network/src/discovery/mod.rs

@@ -1101,6 +1101,7 @@ mod tests {
     use super::*;
     use crate::rpc::methods::{MetaData, MetaDataV2};
     use enr::EnrBuilder;
+    use libp2p::identity::secp256k1;
     use slog::{o, Drain};
     use types::{BitVector, MinimalEthSpec, SubnetId};
 
@@ -1119,10 +1120,10 @@ mod tests {
     }
 
     async fn build_discovery() -> Discovery<E> {
-        let keypair = libp2p::identity::Keypair::generate_secp256k1();
+        let keypair = secp256k1::Keypair::generate();
         let mut config = NetworkConfig::default();
         config.set_listening_addr(crate::ListenAddress::unused_v4_ports());
-        let enr_key: CombinedKey = CombinedKey::from_libp2p(&keypair).unwrap();
+        let enr_key: CombinedKey = CombinedKey::from_secp256k1(&keypair);
         let enr: Enr = build_enr::<E>(&enr_key, &config, &EnrForkId::default()).unwrap();
         let log = build_log(slog::Level::Debug, false);
         let globals = NetworkGlobals::new(
@@ -1138,6 +1139,7 @@ mod tests {
             false,
             &log,
         );
+        let keypair = Keypair::Secp256k1(keypair);
         Discovery::new(&keypair, &config, Arc::new(globals), &log)
             .await
             .unwrap()
@@ -1184,8 +1186,8 @@ mod tests {
 
     fn make_enr(subnet_ids: Vec<usize>) -> Enr {
         let mut builder = EnrBuilder::new("v4");
-        let keypair = libp2p::identity::Keypair::generate_secp256k1();
+        let keypair = secp256k1::Keypair::generate();
-        let enr_key: CombinedKey = CombinedKey::from_libp2p(&keypair).unwrap();
+        let enr_key: CombinedKey = CombinedKey::from_secp256k1(&keypair);
 
         // set the "attnets" field on our ENR
         let mut bitfield = BitVector::<ssz_types::typenum::U64>::new();

beacon_node/lighthouse_network/src/types/globals.rs

@@ -134,9 +134,8 @@ impl<TSpec: EthSpec> NetworkGlobals<TSpec> {
         log: &slog::Logger,
     ) -> NetworkGlobals<TSpec> {
         use crate::CombinedKeyExt;
-        let keypair = libp2p::identity::Keypair::generate_secp256k1();
+        let keypair = libp2p::identity::secp256k1::Keypair::generate();
-        let enr_key: discv5::enr::CombinedKey =
+        let enr_key: discv5::enr::CombinedKey = discv5::enr::CombinedKey::from_secp256k1(&keypair);
-            discv5::enr::CombinedKey::from_libp2p(&keypair).unwrap();
         let enr = discv5::enr::EnrBuilder::new("v4").build(&enr_key).unwrap();
         NetworkGlobals::new(
             enr,

beacon_node/network/Cargo.toml

@@ -11,7 +11,6 @@ matches = "0.1.8"
 exit-future = "0.2.0"
 slog-term = "2.6.0"
 slog-async = "2.5.0"
-environment = { path = "../../lighthouse/environment" }
 
 [dependencies]
 beacon_chain =  { path = "../beacon_chain" }
@@ -47,6 +46,9 @@ delay_map = "0.3.0"
 ethereum-types = { version = "0.14.1", optional = true }
 operation_pool =  { path = "../operation_pool" }
 execution_layer =  { path = "../execution_layer" }
+beacon_processor =  { path = "../beacon_processor" }
+parking_lot = "0.12.0"
+environment = { path = "../../lighthouse/environment" }
 
 [features]
 spec-minimal = ["beacon_chain/spec-minimal"]

beacon_node/network/src/beacon_processor/worker/mod.rs

@@ -1,51 +0,0 @@
-use super::work_reprocessing_queue::ReprocessQueueMessage;
-use crate::{service::NetworkMessage, sync::SyncMessage};
-use beacon_chain::{BeaconChain, BeaconChainTypes};
-use slog::{debug, Logger};
-use std::sync::Arc;
-use tokio::sync::mpsc;
-
-mod gossip_methods;
-mod rpc_methods;
-mod sync_methods;
-
-pub use gossip_methods::{GossipAggregatePackage, GossipAttestationPackage};
-pub use sync_methods::ChainSegmentProcessId;
-
-pub(crate) const FUTURE_SLOT_TOLERANCE: u64 = 1;
-
-/// Contains the context necessary to import blocks, attestations, etc to the beacon chain.
-pub struct Worker<T: BeaconChainTypes> {
-    pub chain: Arc<BeaconChain<T>>,
-    pub network_tx: mpsc::UnboundedSender<NetworkMessage<T::EthSpec>>,
-    pub sync_tx: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
-    pub log: Logger,
-}
-
-impl<T: BeaconChainTypes> Worker<T> {
-    /// Send a message to `sync_tx`.
-    ///
-    /// Creates a log if there is an internal error.
-    fn send_sync_message(&self, message: SyncMessage<T::EthSpec>) {
-        self.sync_tx.send(message).unwrap_or_else(|e| {
-            debug!(self.log, "Could not send message to the sync service";
-                   "error" => %e)
-        });
-    }
-
-    /// Send a message to `network_tx`.
-    ///
-    /// Creates a log if there is an internal error.
-    fn send_network_message(&self, message: NetworkMessage<T::EthSpec>) {
-        self.network_tx.send(message).unwrap_or_else(|e| {
-            debug!(self.log, "Could not send message to the network service. Likely shutdown";
-                "error" => %e)
-        });
-    }
-}
-
-/// Contains the necessary items for a worker to do their job.
-pub struct Toolbox<T: BeaconChainTypes> {
-    pub idle_tx: mpsc::Sender<()>,
-    pub work_reprocessing_tx: mpsc::Sender<ReprocessQueueMessage<T>>,
-}

beacon_node/network/src/lib.rs

@@ -6,10 +6,10 @@ pub mod error;
 #[allow(clippy::mutable_key_type)] // PeerId in hashmaps are no longer permitted by clippy
 pub mod service;
 
-mod beacon_processor;
 #[allow(clippy::mutable_key_type)] // PeerId in hashmaps are no longer permitted by clippy
 mod metrics;
 mod nat;
+mod network_beacon_processor;
 mod persisted_dht;
 mod router;
 mod status;

beacon_node/network/src/metrics.rs

@@ -49,47 +49,8 @@ lazy_static! {
     /*
      * Gossip processor
      */
-    pub static ref BEACON_PROCESSOR_WORK_EVENTS_RX_COUNT: Result<IntCounterVec> = try_create_int_counter_vec(
+
-        "beacon_processor_work_events_rx_count",
-        "Count of work events received (but not necessarily processed)",
-        &["type"]
-    );
-    pub static ref BEACON_PROCESSOR_WORK_EVENTS_IGNORED_COUNT: Result<IntCounterVec> = try_create_int_counter_vec(
-        "beacon_processor_work_events_ignored_count",
-        "Count of work events purposefully ignored",
-        &["type"]
-    );
-    pub static ref BEACON_PROCESSOR_WORK_EVENTS_STARTED_COUNT: Result<IntCounterVec> = try_create_int_counter_vec(
-        "beacon_processor_work_events_started_count",
-        "Count of work events which have been started by a worker",
-        &["type"]
-    );
-    pub static ref BEACON_PROCESSOR_WORKER_TIME: Result<HistogramVec> = try_create_histogram_vec(
-        "beacon_processor_worker_time",
-        "Time taken for a worker to fully process some parcel of work.",
-        &["type"]
-    );
-    pub static ref BEACON_PROCESSOR_WORKERS_SPAWNED_TOTAL: Result<IntCounter> = try_create_int_counter(
-        "beacon_processor_workers_spawned_total",
-        "The number of workers ever spawned by the gossip processing pool."
-    );
-    pub static ref BEACON_PROCESSOR_WORKERS_ACTIVE_TOTAL: Result<IntGauge> = try_create_int_gauge(
-        "beacon_processor_workers_active_total",
-        "Count of active workers in the gossip processing pool."
-    );
-    pub static ref BEACON_PROCESSOR_IDLE_EVENTS_TOTAL: Result<IntCounter> = try_create_int_counter(
-        "beacon_processor_idle_events_total",
-        "Count of idle events processed by the gossip processor manager."
-    );
-    pub static ref BEACON_PROCESSOR_EVENT_HANDLING_SECONDS: Result<Histogram> = try_create_histogram(
-        "beacon_processor_event_handling_seconds",
-        "Time spent handling a new message and allocating it to a queue or worker."
-    );
     // Gossip blocks.
-    pub static ref BEACON_PROCESSOR_GOSSIP_BLOCK_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
-        "beacon_processor_gossip_block_queue_total",
-        "Count of blocks from gossip waiting to be verified."
-    );
     pub static ref BEACON_PROCESSOR_GOSSIP_BLOCK_VERIFIED_TOTAL: Result<IntCounter> = try_create_int_counter(
         "beacon_processor_gossip_block_verified_total",
         "Total number of gossip blocks verified for propagation."
@@ -106,11 +67,6 @@ lazy_static! {
         "beacon_processor_gossip_block_early_seconds",
         "Whenever a gossip block is received early this metrics is set to how early that block was."
     );
-    // Gossip blobs.
-    pub static ref BEACON_PROCESSOR_GOSSIP_BLOB_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
-        "beacon_processor_gossip_blob_queue_total",
-        "Count of blocks from gossip waiting to be verified."
-    );
     pub static ref BEACON_PROCESSOR_GOSSIP_BLOB_VERIFIED_TOTAL: Result<IntCounter> = try_create_int_counter(
         "beacon_processor_gossip_blob_verified_total",
         "Total number of gossip blob verified for propagation."
@@ -120,10 +76,6 @@ lazy_static! {
         "Total number of gossip blobs imported to fork choice, etc."
     );
     // Gossip Exits.
-    pub static ref BEACON_PROCESSOR_EXIT_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
-        "beacon_processor_exit_queue_total",
-        "Count of exits from gossip waiting to be verified."
-    );
     pub static ref BEACON_PROCESSOR_EXIT_VERIFIED_TOTAL: Result<IntCounter> = try_create_int_counter(
         "beacon_processor_exit_verified_total",
         "Total number of voluntary exits verified for propagation."
@@ -133,10 +85,6 @@ lazy_static! {
         "Total number of voluntary exits imported to the op pool."
     );
     // Gossip proposer slashings.
-    pub static ref BEACON_PROCESSOR_PROPOSER_SLASHING_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
-        "beacon_processor_proposer_slashing_queue_total",
-        "Count of proposer slashings from gossip waiting to be verified."
-    );
     pub static ref BEACON_PROCESSOR_PROPOSER_SLASHING_VERIFIED_TOTAL: Result<IntCounter> = try_create_int_counter(
         "beacon_processor_proposer_slashing_verified_total",
         "Total number of proposer slashings verified for propagation."
@@ -146,10 +94,6 @@ lazy_static! {
         "Total number of proposer slashings imported to the op pool."
     );
     // Gossip attester slashings.
-    pub static ref BEACON_PROCESSOR_ATTESTER_SLASHING_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
-        "beacon_processor_attester_slashing_queue_total",
-        "Count of attester slashings from gossip waiting to be verified."
-    );
     pub static ref BEACON_PROCESSOR_ATTESTER_SLASHING_VERIFIED_TOTAL: Result<IntCounter> = try_create_int_counter(
         "beacon_processor_attester_slashing_verified_total",
         "Total number of attester slashings verified for propagation."
@@ -159,10 +103,6 @@ lazy_static! {
         "Total number of attester slashings imported to the op pool."
     );
     // Gossip BLS to execution changes.
-    pub static ref BEACON_PROCESSOR_BLS_TO_EXECUTION_CHANGE_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
-        "beacon_processor_bls_to_execution_change_queue_total",
-        "Count of address changes from gossip waiting to be verified."
-    );
     pub static ref BEACON_PROCESSOR_BLS_TO_EXECUTION_CHANGE_VERIFIED_TOTAL: Result<IntCounter> = try_create_int_counter(
         "beacon_processor_bls_to_execution_change_verified_total",
         "Total number of address changes verified for propagation."
@@ -176,32 +116,15 @@ lazy_static! {
 // Need to split up this `lazy_static!` due to recursion limits.
 lazy_static! {
     // Rpc blocks.
-    pub static ref BEACON_PROCESSOR_RPC_BLOCK_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
-        "beacon_processor_rpc_block_queue_total",
-        "Count of blocks from the rpc waiting to be verified."
-    );
     pub static ref BEACON_PROCESSOR_RPC_BLOCK_IMPORTED_TOTAL: Result<IntCounter> = try_create_int_counter(
         "beacon_processor_rpc_block_imported_total",
         "Total number of gossip blocks imported to fork choice, etc."
     );
-    // Rpc blobs.
-    pub static ref BEACON_PROCESSOR_RPC_BLOB_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
-        "beacon_processor_rpc_blob_queue_total",
-        "Count of blobs from the rpc waiting to be verified."
-    );
     pub static ref BEACON_PROCESSOR_RPC_BLOB_IMPORTED_TOTAL: Result<IntCounter> = try_create_int_counter(
         "beacon_processor_rpc_blob_imported_total",
         "Total number of gossip blobs imported."
     );
     // Chain segments.
-    pub static ref BEACON_PROCESSOR_CHAIN_SEGMENT_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
-        "beacon_processor_chain_segment_queue_total",
-        "Count of chain segments from the rpc waiting to be verified."
-    );
-    pub static ref BEACON_PROCESSOR_BACKFILL_CHAIN_SEGMENT_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
-        "beacon_processor_backfill_chain_segment_queue_total",
-        "Count of backfill chain segments from the rpc waiting to be verified."
-    );
     pub static ref BEACON_PROCESSOR_CHAIN_SEGMENT_SUCCESS_TOTAL: Result<IntCounter> = try_create_int_counter(
         "beacon_processor_chain_segment_success_total",
         "Total number of chain segments successfully processed."
@@ -219,10 +142,6 @@ lazy_static! {
         "Total number of backfill chain segments that failed processing."
     );
     // Unaggregated attestations.
-    pub static ref BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
-        "beacon_processor_unaggregated_attestation_queue_total",
-        "Count of unagg. attestations waiting to be processed."
-    );
     pub static ref BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_VERIFIED_TOTAL: Result<IntCounter> = try_create_int_counter(
         "beacon_processor_unaggregated_attestation_verified_total",
         "Total number of unaggregated attestations verified for gossip."
@@ -236,10 +155,6 @@ lazy_static! {
         "Total number of unaggregated attestations that referenced an unknown block and were re-queued."
     );
     // Aggregated attestations.
-    pub static ref BEACON_PROCESSOR_AGGREGATED_ATTESTATION_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
-        "beacon_processor_aggregated_attestation_queue_total",
-        "Count of agg. attestations waiting to be processed."
-    );
     pub static ref BEACON_PROCESSOR_AGGREGATED_ATTESTATION_VERIFIED_TOTAL: Result<IntCounter> = try_create_int_counter(
         "beacon_processor_aggregated_attestation_verified_total",
         "Total number of aggregated attestations verified for gossip."
@@ -253,10 +168,6 @@ lazy_static! {
         "Total number of aggregated attestations that referenced an unknown block and were re-queued."
     );
     // Sync committee messages.
-    pub static ref BEACON_PROCESSOR_SYNC_MESSAGE_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
-        "beacon_processor_sync_message_queue_total",
-        "Count of sync committee messages waiting to be processed."
-    );
     pub static ref BEACON_PROCESSOR_SYNC_MESSAGE_VERIFIED_TOTAL: Result<IntCounter> = try_create_int_counter(
         "beacon_processor_sync_message_verified_total",
         "Total number of sync committee messages verified for gossip."
@@ -266,10 +177,6 @@ lazy_static! {
         "Total number of sync committee messages imported to fork choice, etc."
     );
     // Sync contribution.
-    pub static ref BEACON_PROCESSOR_SYNC_CONTRIBUTION_QUEUE_TOTAL: Result<IntGauge> = try_create_int_gauge(
-        "beacon_processor_sync_contribution_queue_total",
-        "Count of sync committee contributions waiting to be processed."
-    );
     pub static ref BEACON_PROCESSOR_SYNC_CONTRIBUTION_VERIFIED_TOTAL: Result<IntCounter> = try_create_int_counter(
         "beacon_processor_sync_contribution_verified_total",
         "Total number of sync committee contributions verified for gossip."
@@ -305,12 +212,6 @@ lazy_static! {
             "Gossipsub light_client_optimistic_update errors per error type",
             &["type"]
         );
-    pub static ref BEACON_PROCESSOR_SEND_ERROR_PER_WORK_TYPE: Result<IntCounterVec> =
-        try_create_int_counter_vec(
-            "beacon_processor_send_error_per_work_type",
-            "Total number of beacon processor send error per work type",
-            &["type"]
-        );
 
 
     /*
@@ -426,35 +327,9 @@ lazy_static! {
         "Count of times when a gossip blob arrived from the network later than the attestation deadline.",
     );
 
-    /*
-     * Attestation reprocessing queue metrics.
-     */
-    pub static ref BEACON_PROCESSOR_REPROCESSING_QUEUE_TOTAL: Result<IntGaugeVec> =
-        try_create_int_gauge_vec(
-        "beacon_processor_reprocessing_queue_total",
-        "Count of items in a reprocessing queue.",
-        &["type"]
-    );
-    pub static ref BEACON_PROCESSOR_REPROCESSING_QUEUE_EXPIRED_ATTESTATIONS: Result<IntCounter> = try_create_int_counter(
-        "beacon_processor_reprocessing_queue_expired_attestations",
-        "Number of queued attestations which have expired before a matching block has been found."
-    );
-    pub static ref BEACON_PROCESSOR_REPROCESSING_QUEUE_MATCHED_ATTESTATIONS: Result<IntCounter> = try_create_int_counter(
-        "beacon_processor_reprocessing_queue_matched_attestations",
-        "Number of queued attestations where as matching block has been imported."
-    );
-
     /*
      * Light client update reprocessing queue metrics.
      */
-    pub static ref BEACON_PROCESSOR_REPROCESSING_QUEUE_EXPIRED_OPTIMISTIC_UPDATES: Result<IntCounter> = try_create_int_counter(
-        "beacon_processor_reprocessing_queue_expired_optimistic_updates",
-        "Number of queued light client optimistic updates which have expired before a matching block has been found."
-    );
-    pub static ref BEACON_PROCESSOR_REPROCESSING_QUEUE_MATCHED_OPTIMISTIC_UPDATES: Result<IntCounter> = try_create_int_counter(
-        "beacon_processor_reprocessing_queue_matched_optimistic_updates",
-        "Number of queued light client optimistic updates where as matching block has been imported."
-    );
     pub static ref BEACON_PROCESSOR_REPROCESSING_QUEUE_SENT_OPTIMISTIC_UPDATES: Result<IntCounter> = try_create_int_counter(
         "beacon_processor_reprocessing_queue_sent_optimistic_updates",
         "Number of queued light client optimistic updates where as matching block has been imported."

beacon_node/network/src/network_beacon_processor/gossip_methods.rs

@@ -1,6 +1,12 @@
-use crate::{metrics, service::NetworkMessage, sync::SyncMessage};
+use crate::{
+    metrics,
+    network_beacon_processor::{InvalidBlockStorage, NetworkBeaconProcessor},
+    service::NetworkMessage,
+    sync::SyncMessage,
+};
 
-use beacon_chain::blob_verification::{AsBlock, BlobError, GossipVerifiedBlob};
+use beacon_chain::blob_verification::AsBlock;
+use beacon_chain::blob_verification::{BlobError, GossipVerifiedBlob};
 use beacon_chain::store::Error;
 use beacon_chain::{
     attestation_verification::{self, Error as AttnError, VerifiedAttestation},
@@ -31,14 +37,13 @@ use types::{
     Slot, SubnetId, SyncCommitteeMessage, SyncSubnetId,
 };
 
-use super::{
+use beacon_processor::{
-    super::work_reprocessing_queue::{
+    work_reprocessing_queue::{
         QueuedAggregate, QueuedGossipBlock, QueuedLightClientUpdate, QueuedUnaggregate,
         ReprocessQueueMessage,
     },
-    Worker,
+    DuplicateCache, GossipAggregatePackage, GossipAttestationPackage,
 };
-use crate::beacon_processor::{DuplicateCache, InvalidBlockStorage};
 
 /// Set to `true` to introduce stricter penalties for peers who send some types of late consensus
 /// messages.
@@ -145,65 +150,7 @@ impl<T: EthSpec> FailedAtt<T> {
     }
 }
 
-/// Items required to verify a batch of unaggregated gossip attestations.
+impl<T: BeaconChainTypes> NetworkBeaconProcessor<T> {
-#[derive(Debug)]
-pub struct GossipAttestationPackage<E: EthSpec> {
-    message_id: MessageId,
-    peer_id: PeerId,
-    attestation: Box<Attestation<E>>,
-    subnet_id: SubnetId,
-    should_import: bool,
-    seen_timestamp: Duration,
-}
-
-impl<E: EthSpec> GossipAttestationPackage<E> {
-    pub fn new(
-        message_id: MessageId,
-        peer_id: PeerId,
-        attestation: Box<Attestation<E>>,
-        subnet_id: SubnetId,
-        should_import: bool,
-        seen_timestamp: Duration,
-    ) -> Self {
-        Self {
-            message_id,
-            peer_id,
-            attestation,
-            subnet_id,
-            should_import,
-            seen_timestamp,
-        }
-    }
-}
-
-/// Items required to verify a batch of aggregated gossip attestations.
-#[derive(Debug)]
-pub struct GossipAggregatePackage<E: EthSpec> {
-    message_id: MessageId,
-    peer_id: PeerId,
-    aggregate: Box<SignedAggregateAndProof<E>>,
-    beacon_block_root: Hash256,
-    seen_timestamp: Duration,
-}
-
-impl<E: EthSpec> GossipAggregatePackage<E> {
-    pub fn new(
-        message_id: MessageId,
-        peer_id: PeerId,
-        aggregate: Box<SignedAggregateAndProof<E>>,
-        seen_timestamp: Duration,
-    ) -> Self {
-        Self {
-            message_id,
-            peer_id,
-            beacon_block_root: aggregate.message.aggregate.data.beacon_block_root,
-            aggregate,
-            seen_timestamp,
-        }
-    }
-}
-
-impl<T: BeaconChainTypes> Worker<T> {
     /* Auxiliary functions */
 
     /// Penalizes a peer for misbehaviour.
@@ -246,13 +193,13 @@ impl<T: BeaconChainTypes> Worker<T> {
     /// Raises a log if there are errors.
     #[allow(clippy::too_many_arguments)]
     pub fn process_gossip_attestation(
-        self,
+        self: Arc<Self>,
         message_id: MessageId,
         peer_id: PeerId,
         attestation: Box<Attestation<T::EthSpec>>,
         subnet_id: SubnetId,
         should_import: bool,
-        reprocess_tx: Option<mpsc::Sender<ReprocessQueueMessage<T>>>,
+        reprocess_tx: Option<mpsc::Sender<ReprocessQueueMessage>>,
         seen_timestamp: Duration,
     ) {
         let result = match self
@@ -278,9 +225,9 @@ impl<T: BeaconChainTypes> Worker<T> {
     }
 
     pub fn process_gossip_attestation_batch(
-        self,
+        self: Arc<Self>,
         packages: Vec<GossipAttestationPackage<T::EthSpec>>,
-        reprocess_tx: Option<mpsc::Sender<ReprocessQueueMessage<T>>>,
+        reprocess_tx: Option<mpsc::Sender<ReprocessQueueMessage>>,
     ) {
         let attestations_and_subnets = packages
             .iter()
@@ -349,12 +296,12 @@ impl<T: BeaconChainTypes> Worker<T> {
     // cant' be mixed-up) and creating a struct would result in more complexity.
     #[allow(clippy::too_many_arguments)]
     fn process_gossip_attestation_result(
-        &self,
+        self: &Arc<Self>,
         result: Result<VerifiedUnaggregate<T>, RejectedUnaggregate<T::EthSpec>>,
         message_id: MessageId,
         peer_id: PeerId,
         subnet_id: SubnetId,
-        reprocess_tx: Option<mpsc::Sender<ReprocessQueueMessage<T>>>,
+        reprocess_tx: Option<mpsc::Sender<ReprocessQueueMessage>>,
         should_import: bool,
         seen_timestamp: Duration,
     ) {
@@ -457,11 +404,11 @@ impl<T: BeaconChainTypes> Worker<T> {
     ///
     /// Raises a log if there are errors.
     pub fn process_gossip_aggregate(
-        self,
+        self: Arc<Self>,
         message_id: MessageId,
         peer_id: PeerId,
         aggregate: Box<SignedAggregateAndProof<T::EthSpec>>,
-        reprocess_tx: Option<mpsc::Sender<ReprocessQueueMessage<T>>>,
+        reprocess_tx: Option<mpsc::Sender<ReprocessQueueMessage>>,
         seen_timestamp: Duration,
     ) {
         let beacon_block_root = aggregate.message.aggregate.data.beacon_block_root;
@@ -491,9 +438,9 @@ impl<T: BeaconChainTypes> Worker<T> {
     }
 
     pub fn process_gossip_aggregate_batch(
-        self,
+        self: Arc<Self>,
         packages: Vec<GossipAggregatePackage<T::EthSpec>>,
-        reprocess_tx: Option<mpsc::Sender<ReprocessQueueMessage<T>>>,
+        reprocess_tx: Option<mpsc::Sender<ReprocessQueueMessage>>,
     ) {
         let aggregates = packages.iter().map(|package| package.aggregate.as_ref());
 
@@ -556,12 +503,12 @@ impl<T: BeaconChainTypes> Worker<T> {
     }
 
     fn process_gossip_aggregate_result(
-        &self,
+        self: &Arc<Self>,
         result: Result<VerifiedAggregate<T>, RejectedAggregate<T::EthSpec>>,
         beacon_block_root: Hash256,
         message_id: MessageId,
         peer_id: PeerId,
-        reprocess_tx: Option<mpsc::Sender<ReprocessQueueMessage<T>>>,
+        reprocess_tx: Option<mpsc::Sender<ReprocessQueueMessage>>,
         seen_timestamp: Duration,
     ) {
         match result {
@@ -654,7 +601,7 @@ impl<T: BeaconChainTypes> Worker<T> {
     // TODO: docs
     #[allow(clippy::too_many_arguments)]
     pub async fn process_gossip_blob(
-        self,
+        self: &Arc<Self>,
         message_id: MessageId,
         peer_id: PeerId,
         _peer_client: Client,
@@ -753,7 +700,7 @@ impl<T: BeaconChainTypes> Worker<T> {
     }
 
     pub async fn process_gossip_verified_blob(
-        self,
+        self: &Arc<Self>,
         peer_id: PeerId,
         verified_blob: GossipVerifiedBlob<T>,
         // This value is not used presently, but it might come in handy for debugging.
@@ -803,12 +750,12 @@ impl<T: BeaconChainTypes> Worker<T> {
     /// Raises a log if there are errors.
     #[allow(clippy::too_many_arguments)]
     pub async fn process_gossip_block(
-        self,
+        self: Arc<Self>,
         message_id: MessageId,
         peer_id: PeerId,
         peer_client: Client,
         block: Arc<SignedBeaconBlock<T::EthSpec>>,
-        reprocess_tx: mpsc::Sender<ReprocessQueueMessage<T>>,
+        reprocess_tx: mpsc::Sender<ReprocessQueueMessage>,
         duplicate_cache: DuplicateCache,
         invalid_block_storage: InvalidBlockStorage,
         seen_duration: Duration,
@@ -852,12 +799,12 @@ impl<T: BeaconChainTypes> Worker<T> {
     ///
     /// Returns the `GossipVerifiedBlock` if verification passes and raises a log if there are errors.
     pub async fn process_gossip_unverified_block(
-        &self,
+        self: &Arc<Self>,
         message_id: MessageId,
         peer_id: PeerId,
         peer_client: Client,
         block: Arc<SignedBeaconBlock<T::EthSpec>>,
-        reprocess_tx: mpsc::Sender<ReprocessQueueMessage<T>>,
+        reprocess_tx: mpsc::Sender<ReprocessQueueMessage>,
         seen_duration: Duration,
     ) -> Option<GossipVerifiedBlock<T>> {
         let block_delay =
@@ -1066,11 +1013,25 @@ impl<T: BeaconChainTypes> Worker<T> {
 
                 metrics::inc_counter(&metrics::BEACON_PROCESSOR_GOSSIP_BLOCK_REQUEUED_TOTAL);
 
+                let inner_self = self.clone();
+                let process_fn = Box::pin(async move {
+                    let reprocess_tx = inner_self.reprocess_tx.clone();
+                    let invalid_block_storage = inner_self.invalid_block_storage.clone();
+                    inner_self
+                        .process_gossip_verified_block(
+                            peer_id,
+                            verified_block,
+                            reprocess_tx,
+                            invalid_block_storage,
+                            seen_duration,
+                        )
+                        .await;
+                });
                 if reprocess_tx
                     .try_send(ReprocessQueueMessage::EarlyBlock(QueuedGossipBlock {
-                        peer_id,
+                        beacon_block_slot: block_slot,
-                        block: Box::new(verified_block),
+                        beacon_block_root: block_root,
-                        seen_timestamp: seen_duration,
+                        process_fn,
                     }))
                     .is_err()
                 {
@@ -1103,10 +1064,10 @@ impl<T: BeaconChainTypes> Worker<T> {
     ///
     /// Raises a log if there are errors.
     pub async fn process_gossip_verified_block(
-        self,
+        self: Arc<Self>,
         peer_id: PeerId,
         verified_block: GossipVerifiedBlock<T>,
-        reprocess_tx: mpsc::Sender<ReprocessQueueMessage<T>>,
+        reprocess_tx: mpsc::Sender<ReprocessQueueMessage>,
         invalid_block_storage: InvalidBlockStorage,
         // This value is not used presently, but it might come in handy for debugging.
         _seen_duration: Duration,
@@ -1214,7 +1175,7 @@ impl<T: BeaconChainTypes> Worker<T> {
     }
 
     pub fn process_gossip_voluntary_exit(
-        self,
+        self: &Arc<Self>,
         message_id: MessageId,
         peer_id: PeerId,
         voluntary_exit: SignedVoluntaryExit,
@@ -1272,7 +1233,7 @@ impl<T: BeaconChainTypes> Worker<T> {
     }
 
     pub fn process_gossip_proposer_slashing(
-        self,
+        self: &Arc<Self>,
         message_id: MessageId,
         peer_id: PeerId,
         proposer_slashing: ProposerSlashing,
@@ -1334,7 +1295,7 @@ impl<T: BeaconChainTypes> Worker<T> {
     }
 
     pub fn process_gossip_attester_slashing(
-        self,
+        self: &Arc<Self>,
         message_id: MessageId,
         peer_id: PeerId,
         attester_slashing: AttesterSlashing<T::EthSpec>,
@@ -1388,7 +1349,7 @@ impl<T: BeaconChainTypes> Worker<T> {
     }
 
     pub fn process_gossip_bls_to_execution_change(
-        self,
+        self: &Arc<Self>,
         message_id: MessageId,
         peer_id: PeerId,
         bls_to_execution_change: SignedBlsToExecutionChange,
@@ -1471,7 +1432,7 @@ impl<T: BeaconChainTypes> Worker<T> {
     ///
     /// Raises a log if there are errors.
     pub fn process_gossip_sync_committee_signature(
-        self,
+        self: &Arc<Self>,
         message_id: MessageId,
         peer_id: PeerId,
         sync_signature: SyncCommitteeMessage,
@@ -1534,7 +1495,7 @@ impl<T: BeaconChainTypes> Worker<T> {
     ///
     /// Raises a log if there are errors.
     pub fn process_sync_committee_contribution(
-        self,
+        self: &Arc<Self>,
         message_id: MessageId,
         peer_id: PeerId,
         sync_contribution: SignedContributionAndProof<T::EthSpec>,
@@ -1589,7 +1550,7 @@ impl<T: BeaconChainTypes> Worker<T> {
     }
 
     pub fn process_gossip_finality_update(
-        self,
+        self: &Arc<Self>,
         message_id: MessageId,
         peer_id: PeerId,
         light_client_finality_update: LightClientFinalityUpdate<T::EthSpec>,
@@ -1655,11 +1616,11 @@ impl<T: BeaconChainTypes> Worker<T> {
     }
 
     pub fn process_gossip_optimistic_update(
-        self,
+        self: &Arc<Self>,
         message_id: MessageId,
         peer_id: PeerId,
         light_client_optimistic_update: LightClientOptimisticUpdate<T::EthSpec>,
-        reprocess_tx: Option<mpsc::Sender<ReprocessQueueMessage<T>>>,
+        reprocess_tx: Option<mpsc::Sender<ReprocessQueueMessage>>,
         seen_timestamp: Duration,
     ) {
         match self.chain.verify_optimistic_update_for_gossip(
@@ -1690,15 +1651,19 @@ impl<T: BeaconChainTypes> Worker<T> {
                         );
 
                         if let Some(sender) = reprocess_tx {
+                            let processor = self.clone();
                             let msg = ReprocessQueueMessage::UnknownLightClientOptimisticUpdate(
                                 QueuedLightClientUpdate {
-                                    peer_id,
-                                    message_id,
-                                    light_client_optimistic_update: Box::new(
-                                        light_client_optimistic_update,
-                                    ),
                                     parent_root,
-                                    seen_timestamp,
+                                    process_fn: Box::new(move || {
+                                        processor.process_gossip_optimistic_update(
+                                            message_id,
+                                            peer_id,
+                                            light_client_optimistic_update,
+                                            None, // Do not reprocess this message again.
+                                            seen_timestamp,
+                                        )
+                                    }),
                                 },
                             );
 
@@ -1787,11 +1752,11 @@ impl<T: BeaconChainTypes> Worker<T> {
     /// Handle an error whilst verifying an `Attestation` or `SignedAggregateAndProof` from the
     /// network.
     fn handle_attestation_verification_failure(
-        &self,
+        self: &Arc<Self>,
         peer_id: PeerId,
         message_id: MessageId,
         failed_att: FailedAtt<T::EthSpec>,
-        reprocess_tx: Option<mpsc::Sender<ReprocessQueueMessage<T>>>,
+        reprocess_tx: Option<mpsc::Sender<ReprocessQueueMessage>>,
         error: AttnError,
         seen_timestamp: Duration,
     ) {
@@ -2026,11 +1991,18 @@ impl<T: BeaconChainTypes> Worker<T> {
                             metrics::inc_counter(
                                 &metrics::BEACON_PROCESSOR_AGGREGATED_ATTESTATION_REQUEUED_TOTAL,
                             );
+                            let processor = self.clone();
                             ReprocessQueueMessage::UnknownBlockAggregate(QueuedAggregate {
-                                peer_id,
+                                beacon_block_root: *beacon_block_root,
-                                message_id,
+                                process_fn: Box::new(move || {
-                                attestation,
+                                    processor.process_gossip_aggregate(
-                                seen_timestamp,
+                                        message_id,
+                                        peer_id,
+                                        attestation,
+                                        None, // Do not allow this attestation to be re-processed beyond this point.
+                                        seen_timestamp,
+                                    )
+                                }),
                             })
                         }
                         FailedAtt::Unaggregate {
@@ -2042,13 +2014,20 @@ impl<T: BeaconChainTypes> Worker<T> {
                             metrics::inc_counter(
                                 &metrics::BEACON_PROCESSOR_UNAGGREGATED_ATTESTATION_REQUEUED_TOTAL,
                             );
+                            let processor = self.clone();
                             ReprocessQueueMessage::UnknownBlockUnaggregate(QueuedUnaggregate {
-                                peer_id,
+                                beacon_block_root: *beacon_block_root,
-                                message_id,
+                                process_fn: Box::new(move || {
-                                attestation,
+                                    processor.process_gossip_attestation(
-                                subnet_id,
+                                        message_id,
-                                should_import,
+                                        peer_id,
-                                seen_timestamp,
+                                        attestation,
+                                        subnet_id,
+                                        should_import,
+                                        None, // Do not allow this attestation to be re-processed beyond this point.
+                                        seen_timestamp,
+                                    )
+                                }),
                             })
                         }
                     };

beacon_node/network/src/network_beacon_processor/mod.rs

@@ -0,0 +1,680 @@
+use crate::{
+    service::NetworkMessage,
+    sync::{manager::BlockProcessType, SyncMessage},
+};
+use beacon_chain::blob_verification::BlockWrapper;
+use beacon_chain::{
+    builder::Witness, eth1_chain::CachingEth1Backend, test_utils::BeaconChainHarness, BeaconChain,
+};
+use beacon_chain::{BeaconChainTypes, NotifyExecutionLayer};
+use beacon_processor::{
+    work_reprocessing_queue::ReprocessQueueMessage, BeaconProcessorSend, DuplicateCache,
+    GossipAggregatePackage, GossipAttestationPackage, Work, WorkEvent as BeaconWorkEvent,
+    MAX_SCHEDULED_WORK_QUEUE_LEN, MAX_WORK_EVENT_QUEUE_LEN,
+};
+use environment::null_logger;
+use lighthouse_network::rpc::methods::{BlobsByRangeRequest, BlobsByRootRequest};
+use lighthouse_network::{
+    rpc::{BlocksByRangeRequest, BlocksByRootRequest, LightClientBootstrapRequest, StatusMessage},
+    Client, MessageId, NetworkGlobals, PeerId, PeerRequestId,
+};
+use slog::{debug, Logger};
+use slot_clock::ManualSlotClock;
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::time::Duration;
+use store::MemoryStore;
+use task_executor::test_utils::TestRuntime;
+use task_executor::TaskExecutor;
+use tokio::sync::mpsc::{self, error::TrySendError};
+use types::*;
+
+pub use sync_methods::ChainSegmentProcessId;
+use types::blob_sidecar::FixedBlobSidecarList;
+
+pub type Error<T> = TrySendError<BeaconWorkEvent<T>>;
+
+mod gossip_methods;
+mod rpc_methods;
+mod sync_methods;
+mod tests;
+
+pub(crate) const FUTURE_SLOT_TOLERANCE: u64 = 1;
+
+/// Defines if and where we will store the SSZ files of invalid blocks.
+#[derive(Clone)]
+pub enum InvalidBlockStorage {
+    Enabled(PathBuf),
+    Disabled,
+}
+
+/// Provides an interface to a `BeaconProcessor` running in some other thread.
+/// The wider `networking` crate should use this struct to interface with the
+/// beacon processor.
+pub struct NetworkBeaconProcessor<T: BeaconChainTypes> {
+    pub beacon_processor_send: BeaconProcessorSend<T::EthSpec>,
+    pub duplicate_cache: DuplicateCache,
+    pub chain: Arc<BeaconChain<T>>,
+    pub network_tx: mpsc::UnboundedSender<NetworkMessage<T::EthSpec>>,
+    pub sync_tx: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+    pub reprocess_tx: mpsc::Sender<ReprocessQueueMessage>,
+    pub network_globals: Arc<NetworkGlobals<T::EthSpec>>,
+    pub invalid_block_storage: InvalidBlockStorage,
+    pub executor: TaskExecutor,
+    pub log: Logger,
+}
+
+impl<T: BeaconChainTypes> NetworkBeaconProcessor<T> {
+    fn try_send(&self, event: BeaconWorkEvent<T::EthSpec>) -> Result<(), Error<T::EthSpec>> {
+        self.beacon_processor_send
+            .try_send(event)
+            .map_err(Into::into)
+    }
+
+    /// Create a new `Work` event for some unaggregated attestation.
+    pub fn send_unaggregated_attestation(
+        self: &Arc<Self>,
+        message_id: MessageId,
+        peer_id: PeerId,
+        attestation: Attestation<T::EthSpec>,
+        subnet_id: SubnetId,
+        should_import: bool,
+        seen_timestamp: Duration,
+    ) -> Result<(), Error<T::EthSpec>> {
+        // Define a closure for processing individual attestations.
+        let processor = self.clone();
+        let process_individual = move |package: GossipAttestationPackage<T::EthSpec>| {
+            let reprocess_tx = processor.reprocess_tx.clone();
+            processor.process_gossip_attestation(
+                package.message_id,
+                package.peer_id,
+                package.attestation,
+                package.subnet_id,
+                package.should_import,
+                Some(reprocess_tx),
+                package.seen_timestamp,
+            )
+        };
+
+        // Define a closure for processing batches of attestations.
+        let processor = self.clone();
+        let process_batch = move |attestations| {
+            let reprocess_tx = processor.reprocess_tx.clone();
+            processor.process_gossip_attestation_batch(attestations, Some(reprocess_tx))
+        };
+
+        self.try_send(BeaconWorkEvent {
+            drop_during_sync: true,
+            work: Work::GossipAttestation {
+                attestation: GossipAttestationPackage {
+                    message_id,
+                    peer_id,
+                    attestation: Box::new(attestation),
+                    subnet_id,
+                    should_import,
+                    seen_timestamp,
+                },
+                process_individual: Box::new(process_individual),
+                process_batch: Box::new(process_batch),
+            },
+        })
+    }
+
+    /// Create a new `Work` event for some aggregated attestation.
+    pub fn send_aggregated_attestation(
+        self: &Arc<Self>,
+        message_id: MessageId,
+        peer_id: PeerId,
+        aggregate: SignedAggregateAndProof<T::EthSpec>,
+        seen_timestamp: Duration,
+    ) -> Result<(), Error<T::EthSpec>> {
+        // Define a closure for processing individual attestations.
+        let processor = self.clone();
+        let process_individual = move |package: GossipAggregatePackage<T::EthSpec>| {
+            let reprocess_tx = processor.reprocess_tx.clone();
+            processor.process_gossip_aggregate(
+                package.message_id,
+                package.peer_id,
+                package.aggregate,
+                Some(reprocess_tx),
+                package.seen_timestamp,
+            )
+        };
+
+        // Define a closure for processing batches of attestations.
+        let processor = self.clone();
+        let process_batch = move |aggregates| {
+            let reprocess_tx = processor.reprocess_tx.clone();
+            processor.process_gossip_aggregate_batch(aggregates, Some(reprocess_tx))
+        };
+
+        let beacon_block_root = aggregate.message.aggregate.data.beacon_block_root;
+        self.try_send(BeaconWorkEvent {
+            drop_during_sync: true,
+            work: Work::GossipAggregate {
+                aggregate: GossipAggregatePackage {
+                    message_id,
+                    peer_id,
+                    aggregate: Box::new(aggregate),
+                    beacon_block_root,
+                    seen_timestamp,
+                },
+                process_individual: Box::new(process_individual),
+                process_batch: Box::new(process_batch),
+            },
+        })
+    }
+
+    /// Create a new `Work` event for some block.
+    pub fn send_gossip_beacon_block(
+        self: &Arc<Self>,
+        message_id: MessageId,
+        peer_id: PeerId,
+        peer_client: Client,
+        block: Arc<SignedBeaconBlock<T::EthSpec>>,
+        seen_timestamp: Duration,
+    ) -> Result<(), Error<T::EthSpec>> {
+        let processor = self.clone();
+        let process_fn = async move {
+            let reprocess_tx = processor.reprocess_tx.clone();
+            let invalid_block_storage = processor.invalid_block_storage.clone();
+            let duplicate_cache = processor.duplicate_cache.clone();
+            processor
+                .process_gossip_block(
+                    message_id,
+                    peer_id,
+                    peer_client,
+                    block,
+                    reprocess_tx,
+                    duplicate_cache,
+                    invalid_block_storage,
+                    seen_timestamp,
+                )
+                .await
+        };
+
+        self.try_send(BeaconWorkEvent {
+            drop_during_sync: false,
+            work: Work::GossipBlock(Box::pin(process_fn)),
+        })
+    }
+
+    /// Create a new `Work` event for some blob sidecar.
+    pub fn send_gossip_blob_sidecar(
+        self: &Arc<Self>,
+        message_id: MessageId,
+        peer_id: PeerId,
+        peer_client: Client,
+        blob_index: u64,
+        blob: SignedBlobSidecar<T::EthSpec>,
+        seen_timestamp: Duration,
+    ) -> Result<(), Error<T::EthSpec>> {
+        let processor = self.clone();
+        let process_fn = async move {
+            processor
+                .process_gossip_blob(
+                    message_id,
+                    peer_id,
+                    peer_client,
+                    blob_index,
+                    blob,
+                    seen_timestamp,
+                )
+                .await
+        };
+
+        self.try_send(BeaconWorkEvent {
+            drop_during_sync: false,
+            work: Work::GossipSignedBlobSidecar(Box::pin(process_fn)),
+        })
+    }
+
+    /// Create a new `Work` event for some sync committee signature.
+    pub fn send_gossip_sync_signature(
+        self: &Arc<Self>,
+        message_id: MessageId,
+        peer_id: PeerId,
+        sync_signature: SyncCommitteeMessage,
+        subnet_id: SyncSubnetId,
+        seen_timestamp: Duration,
+    ) -> Result<(), Error<T::EthSpec>> {
+        let processor = self.clone();
+        let process_fn = move || {
+            processor.process_gossip_sync_committee_signature(
+                message_id,
+                peer_id,
+                sync_signature,
+                subnet_id,
+                seen_timestamp,
+            )
+        };
+
+        self.try_send(BeaconWorkEvent {
+            drop_during_sync: true,
+            work: Work::GossipSyncSignature(Box::new(process_fn)),
+        })
+    }
+
+    /// Create a new `Work` event for some sync committee contribution.
+    pub fn send_gossip_sync_contribution(
+        self: &Arc<Self>,
+        message_id: MessageId,
+        peer_id: PeerId,
+        sync_contribution: SignedContributionAndProof<T::EthSpec>,
+        seen_timestamp: Duration,
+    ) -> Result<(), Error<T::EthSpec>> {
+        let processor = self.clone();
+        let process_fn = move || {
+            processor.process_sync_committee_contribution(
+                message_id,
+                peer_id,
+                sync_contribution,
+                seen_timestamp,
+            )
+        };
+
+        self.try_send(BeaconWorkEvent {
+            drop_during_sync: true,
+            work: Work::GossipSyncContribution(Box::new(process_fn)),
+        })
+    }
+
+    /// Create a new `Work` event for some exit.
+    pub fn send_gossip_voluntary_exit(
+        self: &Arc<Self>,
+        message_id: MessageId,
+        peer_id: PeerId,
+        voluntary_exit: Box<SignedVoluntaryExit>,
+    ) -> Result<(), Error<T::EthSpec>> {
+        let processor = self.clone();
+        let process_fn =
+            move || processor.process_gossip_voluntary_exit(message_id, peer_id, *voluntary_exit);
+
+        self.try_send(BeaconWorkEvent {
+            drop_during_sync: false,
+            work: Work::GossipVoluntaryExit(Box::new(process_fn)),
+        })
+    }
+
+    /// Create a new `Work` event for some proposer slashing.
+    pub fn send_gossip_proposer_slashing(
+        self: &Arc<Self>,
+        message_id: MessageId,
+        peer_id: PeerId,
+        proposer_slashing: Box<ProposerSlashing>,
+    ) -> Result<(), Error<T::EthSpec>> {
+        let processor = self.clone();
+        let process_fn = move || {
+            processor.process_gossip_proposer_slashing(message_id, peer_id, *proposer_slashing)
+        };
+
+        self.try_send(BeaconWorkEvent {
+            drop_during_sync: false,
+            work: Work::GossipProposerSlashing(Box::new(process_fn)),
+        })
+    }
+
+    /// Create a new `Work` event for some light client finality update.
+    pub fn send_gossip_light_client_finality_update(
+        self: &Arc<Self>,
+        message_id: MessageId,
+        peer_id: PeerId,
+        light_client_finality_update: LightClientFinalityUpdate<T::EthSpec>,
+        seen_timestamp: Duration,
+    ) -> Result<(), Error<T::EthSpec>> {
+        let processor = self.clone();
+        let process_fn = move || {
+            processor.process_gossip_finality_update(
+                message_id,
+                peer_id,
+                light_client_finality_update,
+                seen_timestamp,
+            )
+        };
+
+        self.try_send(BeaconWorkEvent {
+            drop_during_sync: true,
+            work: Work::GossipLightClientFinalityUpdate(Box::new(process_fn)),
+        })
+    }
+
+    /// Create a new `Work` event for some light client optimistic update.
+    pub fn send_gossip_light_client_optimistic_update(
+        self: &Arc<Self>,
+        message_id: MessageId,
+        peer_id: PeerId,
+        light_client_optimistic_update: LightClientOptimisticUpdate<T::EthSpec>,
+        seen_timestamp: Duration,
+    ) -> Result<(), Error<T::EthSpec>> {
+        let processor = self.clone();
+        let process_fn = move || {
+            let reprocess_tx = processor.reprocess_tx.clone();
+            processor.process_gossip_optimistic_update(
+                message_id,
+                peer_id,
+                light_client_optimistic_update,
+                Some(reprocess_tx),
+                seen_timestamp,
+            )
+        };
+
+        self.try_send(BeaconWorkEvent {
+            drop_during_sync: true,
+            work: Work::GossipLightClientOptimisticUpdate(Box::new(process_fn)),
+        })
+    }
+
+    /// Create a new `Work` event for some attester slashing.
+    pub fn send_gossip_attester_slashing(
+        self: &Arc<Self>,
+        message_id: MessageId,
+        peer_id: PeerId,
+        attester_slashing: Box<AttesterSlashing<T::EthSpec>>,
+    ) -> Result<(), Error<T::EthSpec>> {
+        let processor = self.clone();
+        let process_fn = move || {
+            processor.process_gossip_attester_slashing(message_id, peer_id, *attester_slashing)
+        };
+
+        self.try_send(BeaconWorkEvent {
+            drop_during_sync: false,
+            work: Work::GossipAttesterSlashing(Box::new(process_fn)),
+        })
+    }
+
+    /// Create a new `Work` event for some BLS to execution change.
+    pub fn send_gossip_bls_to_execution_change(
+        self: &Arc<Self>,
+        message_id: MessageId,
+        peer_id: PeerId,
+        bls_to_execution_change: Box<SignedBlsToExecutionChange>,
+    ) -> Result<(), Error<T::EthSpec>> {
+        let processor = self.clone();
+        let process_fn = move || {
+            processor.process_gossip_bls_to_execution_change(
+                message_id,
+                peer_id,
+                *bls_to_execution_change,
+            )
+        };
+
+        self.try_send(BeaconWorkEvent {
+            drop_during_sync: false,
+            work: Work::GossipBlsToExecutionChange(Box::new(process_fn)),
+        })
+    }
+
+    /// Create a new `Work` event for some block, where the result from computation (if any) is
+    /// sent to the other side of `result_tx`.
+    pub fn send_rpc_beacon_block(
+        self: &Arc<Self>,
+        block_root: Hash256,
+        block: BlockWrapper<T::EthSpec>,
+        seen_timestamp: Duration,
+        process_type: BlockProcessType,
+    ) -> Result<(), Error<T::EthSpec>> {
+        let process_fn = self.clone().generate_rpc_beacon_block_process_fn(
+            block_root,
+            block,
+            seen_timestamp,
+            process_type,
+        );
+        self.try_send(BeaconWorkEvent {
+            drop_during_sync: false,
+            work: Work::RpcBlock { process_fn },
+        })
+    }
+
+    /// Create a new `Work` event for some blobs, where the result from computation (if any) is
+    /// sent to the other side of `result_tx`.
+    pub fn send_rpc_blobs(
+        self: &Arc<Self>,
+        block_root: Hash256,
+        blobs: FixedBlobSidecarList<T::EthSpec>,
+        seen_timestamp: Duration,
+        process_type: BlockProcessType,
+    ) -> Result<(), Error<T::EthSpec>> {
+        let process_fn = self.clone().generate_rpc_blobs_process_fn(
+            block_root,
+            blobs,
+            seen_timestamp,
+            process_type,
+        );
+        self.try_send(BeaconWorkEvent {
+            drop_during_sync: false,
+            work: Work::RpcBlobs { process_fn },
+        })
+    }
+
+    /// Create a new work event to import `blocks` as a beacon chain segment.
+    pub fn send_chain_segment(
+        self: &Arc<Self>,
+        process_id: ChainSegmentProcessId,
+        blocks: Vec<BlockWrapper<T::EthSpec>>,
+    ) -> Result<(), Error<T::EthSpec>> {
+        let is_backfill = matches!(&process_id, ChainSegmentProcessId::BackSyncBatchId { .. });
+        let processor = self.clone();
+        let process_fn = async move {
+            let notify_execution_layer = if processor
+                .network_globals
+                .sync_state
+                .read()
+                .is_syncing_finalized()
+            {
+                NotifyExecutionLayer::No
+            } else {
+                NotifyExecutionLayer::Yes
+            };
+            processor
+                .process_chain_segment(process_id, blocks, notify_execution_layer)
+                .await;
+        };
+        let process_fn = Box::pin(process_fn);
+
+        // Back-sync batches are dispatched with a different `Work` variant so
+        // they can be rate-limited.
+        let work = if is_backfill {
+            Work::ChainSegmentBackfill(process_fn)
+        } else {
+            Work::ChainSegment(process_fn)
+        };
+
+        self.try_send(BeaconWorkEvent {
+            drop_during_sync: false,
+            work,
+        })
+    }
+
+    /// Create a new work event to process `StatusMessage`s from the RPC network.
+    pub fn send_status_message(
+        self: &Arc<Self>,
+        peer_id: PeerId,
+        message: StatusMessage,
+    ) -> Result<(), Error<T::EthSpec>> {
+        let processor = self.clone();
+        let process_fn = move || processor.process_status(peer_id, message);
+
+        self.try_send(BeaconWorkEvent {
+            drop_during_sync: false,
+            work: Work::Status(Box::new(process_fn)),
+        })
+    }
+
+    /// Create a new work event to process `BlocksByRangeRequest`s from the RPC network.
+    pub fn send_blocks_by_range_request(
+        self: &Arc<Self>,
+        peer_id: PeerId,
+        request_id: PeerRequestId,
+        request: BlocksByRangeRequest,
+    ) -> Result<(), Error<T::EthSpec>> {
+        let processor = self.clone();
+        let process_fn = move |send_idle_on_drop| {
+            let executor = processor.executor.clone();
+            processor.handle_blocks_by_range_request(
+                executor,
+                send_idle_on_drop,
+                peer_id,
+                request_id,
+                request,
+            )
+        };
+
+        self.try_send(BeaconWorkEvent {
+            drop_during_sync: false,
+            work: Work::BlocksByRangeRequest(Box::new(process_fn)),
+        })
+    }
+
+    /// Create a new work event to process `BlocksByRootRequest`s from the RPC network.
+    pub fn send_blocks_by_roots_request(
+        self: &Arc<Self>,
+        peer_id: PeerId,
+        request_id: PeerRequestId,
+        request: BlocksByRootRequest,
+    ) -> Result<(), Error<T::EthSpec>> {
+        let processor = self.clone();
+        let process_fn = move |send_idle_on_drop| {
+            let executor = processor.executor.clone();
+            processor.handle_blocks_by_root_request(
+                executor,
+                send_idle_on_drop,
+                peer_id,
+                request_id,
+                request,
+            )
+        };
+
+        self.try_send(BeaconWorkEvent {
+            drop_during_sync: false,
+            work: Work::BlocksByRootsRequest(Box::new(process_fn)),
+        })
+    }
+
+    /// Create a new work event to process `BlobsByRangeRequest`s from the RPC network.
+    pub fn send_blobs_by_range_request(
+        self: &Arc<Self>,
+        peer_id: PeerId,
+        request_id: PeerRequestId,
+        request: BlobsByRangeRequest,
+    ) -> Result<(), Error<T::EthSpec>> {
+        let processor = self.clone();
+        let process_fn = move |send_idle_on_drop| {
+            processor.handle_blobs_by_range_request(send_idle_on_drop, peer_id, request_id, request)
+        };
+
+        self.try_send(BeaconWorkEvent {
+            drop_during_sync: false,
+            work: Work::BlobsByRangeRequest(Box::new(process_fn)),
+        })
+    }
+
+    /// Create a new work event to process `BlobsByRootRequest`s from the RPC network.
+    pub fn send_blobs_by_roots_request(
+        self: &Arc<Self>,
+        peer_id: PeerId,
+        request_id: PeerRequestId,
+        request: BlobsByRootRequest,
+    ) -> Result<(), Error<T::EthSpec>> {
+        let processor = self.clone();
+        let process_fn = move |send_idle_on_drop| {
+            processor.handle_blobs_by_root_request(send_idle_on_drop, peer_id, request_id, request)
+        };
+
+        self.try_send(BeaconWorkEvent {
+            drop_during_sync: false,
+            work: Work::BlobsByRootsRequest(Box::new(process_fn)),
+        })
+    }
+
+    /// Create a new work event to process `LightClientBootstrap`s from the RPC network.
+    pub fn send_lightclient_bootstrap_request(
+        self: &Arc<Self>,
+        peer_id: PeerId,
+        request_id: PeerRequestId,
+        request: LightClientBootstrapRequest,
+    ) -> Result<(), Error<T::EthSpec>> {
+        let processor = self.clone();
+        let process_fn =
+            move || processor.handle_light_client_bootstrap(peer_id, request_id, request);
+
+        self.try_send(BeaconWorkEvent {
+            drop_during_sync: true,
+            work: Work::LightClientBootstrapRequest(Box::new(process_fn)),
+        })
+    }
+
+    /// Send a message to `sync_tx`.
+    ///
+    /// Creates a log if there is an internal error.
+    fn send_sync_message(&self, message: SyncMessage<T::EthSpec>) {
+        self.sync_tx.send(message).unwrap_or_else(|e| {
+            debug!(self.log, "Could not send message to the sync service";
+                   "error" => %e)
+        });
+    }
+
+    /// Send a message to `network_tx`.
+    ///
+    /// Creates a log if there is an internal error.
+    fn send_network_message(&self, message: NetworkMessage<T::EthSpec>) {
+        self.network_tx.send(message).unwrap_or_else(|e| {
+            debug!(self.log, "Could not send message to the network service. Likely shutdown";
+                "error" => %e)
+        });
+    }
+}
+
+type TestBeaconChainType<E> =
+    Witness<ManualSlotClock, CachingEth1Backend<E>, E, MemoryStore<E>, MemoryStore<E>>;
+
+impl<E: EthSpec> NetworkBeaconProcessor<TestBeaconChainType<E>> {
+    // Instantiates a mostly non-functional version of `Self` and returns the
+    // event receiver that would normally go to the beacon processor. This is
+    // useful for testing that messages are actually being sent to the beacon
+    // processor (but not much else).
+    pub fn null_for_testing(
+        network_globals: Arc<NetworkGlobals<E>>,
+    ) -> (Self, mpsc::Receiver<BeaconWorkEvent<E>>) {
+        let (beacon_processor_send, beacon_processor_receive) =
+            mpsc::channel(MAX_WORK_EVENT_QUEUE_LEN);
+        let (network_tx, _network_rx) = mpsc::unbounded_channel();
+        let (sync_tx, _sync_rx) = mpsc::unbounded_channel();
+        let (reprocess_tx, _reprocess_rx) = mpsc::channel(MAX_SCHEDULED_WORK_QUEUE_LEN);
+        let log = null_logger().unwrap();
+        let harness: BeaconChainHarness<TestBeaconChainType<E>> =
+            BeaconChainHarness::builder(E::default())
+                .spec(E::default_spec())
+                .deterministic_keypairs(8)
+                .logger(log.clone())
+                .fresh_ephemeral_store()
+                .mock_execution_layer()
+                .build();
+        let runtime = TestRuntime::default();
+
+        let network_beacon_processor = Self {
+            beacon_processor_send: BeaconProcessorSend(beacon_processor_send),
+            duplicate_cache: DuplicateCache::default(),
+            chain: harness.chain,
+            network_tx,
+            sync_tx,
+            reprocess_tx,
+            network_globals,
+            invalid_block_storage: InvalidBlockStorage::Disabled,
+            executor: runtime.task_executor.clone(),
+            log,
+        };
+
+        (network_beacon_processor, beacon_processor_receive)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    #[test]
+    fn queued_block_delay_is_sane() {
+        assert!(
+            beacon_processor::work_reprocessing_queue::ADDITIONAL_QUEUED_BLOCK_DELAY
+                < beacon_chain::MAXIMUM_GOSSIP_CLOCK_DISPARITY
+        );
+    }
+}

beacon_node/network/src/network_beacon_processor/rpc_methods.rs

@@ -1,8 +1,9 @@
-use crate::beacon_processor::{worker::FUTURE_SLOT_TOLERANCE, SendOnDrop};
+use crate::network_beacon_processor::{NetworkBeaconProcessor, FUTURE_SLOT_TOLERANCE};
 use crate::service::NetworkMessage;
 use crate::status::ToStatusMessage;
 use crate::sync::SyncMessage;
 use beacon_chain::{BeaconChainError, BeaconChainTypes, HistoricalBlockError, WhenSlotSkipped};
+use beacon_processor::SendOnDrop;
 use itertools::process_results;
 use lighthouse_network::rpc::methods::{
     BlobsByRangeRequest, BlobsByRootRequest, MAX_REQUEST_BLOB_SIDECARS, MAX_REQUEST_BLOCKS_DENEB,
@@ -13,14 +14,13 @@ use lighthouse_network::{PeerId, PeerRequestId, ReportSource, Response, SyncInfo
 use slog::{debug, error, trace, warn};
 use slot_clock::SlotClock;
 use std::collections::{hash_map::Entry, HashMap};
+use std::sync::Arc;
 use task_executor::TaskExecutor;
 use tokio_stream::StreamExt;
 use types::blob_sidecar::BlobIdentifier;
 use types::{light_client_bootstrap::LightClientBootstrap, Epoch, EthSpec, Hash256, Slot};
 
-use super::Worker;
+impl<T: BeaconChainTypes> NetworkBeaconProcessor<T> {
-
-impl<T: BeaconChainTypes> Worker<T> {
     /* Auxiliary functions */
 
     /// Disconnects and ban's a peer, sending a Goodbye request with the associated reason.
@@ -132,7 +132,7 @@ impl<T: BeaconChainTypes> Worker<T> {
 
     /// Handle a `BlocksByRoot` request from the peer.
     pub fn handle_blocks_by_root_request(
-        self,
+        self: Arc<Self>,
         executor: TaskExecutor,
         send_on_drop: SendOnDrop,
         peer_id: PeerId,
@@ -217,7 +217,7 @@ impl<T: BeaconChainTypes> Worker<T> {
     }
     /// Handle a `BlobsByRoot` request from the peer.
     pub fn handle_blobs_by_root_request(
-        self,
+        self: Arc<Self>,
         send_on_drop: SendOnDrop,
         peer_id: PeerId,
         request_id: PeerRequestId,
@@ -297,7 +297,7 @@ impl<T: BeaconChainTypes> Worker<T> {
 
     /// Handle a `BlocksByRoot` request from the peer.
     pub fn handle_light_client_bootstrap(
-        self,
+        self: &Arc<Self>,
         peer_id: PeerId,
         request_id: PeerRequestId,
         request: LightClientBootstrapRequest,
@@ -370,7 +370,7 @@ impl<T: BeaconChainTypes> Worker<T> {
 
     /// Handle a `BlocksByRange` request from the peer.
     pub fn handle_blocks_by_range_request(
-        self,
+        self: Arc<Self>,
         executor: TaskExecutor,
         send_on_drop: SendOnDrop,
         peer_id: PeerId,
@@ -616,7 +616,7 @@ impl<T: BeaconChainTypes> Worker<T> {
 
     /// Handle a `BlobsByRange` request from the peer.
     pub fn handle_blobs_by_range_request(
-        self,
+        self: Arc<Self>,
         send_on_drop: SendOnDrop,
         peer_id: PeerId,
         request_id: PeerRequestId,

beacon_node/network/src/network_beacon_processor/sync_methods.rs

@@ -1,24 +1,27 @@
-use std::time::Duration;
-
-use super::{super::work_reprocessing_queue::ReprocessQueueMessage, Worker};
-use crate::beacon_processor::work_reprocessing_queue::QueuedRpcBlock;
-use crate::beacon_processor::worker::FUTURE_SLOT_TOLERANCE;
-use crate::beacon_processor::DuplicateCache;
 use crate::metrics;
-use crate::sync::manager::{BlockProcessType, ResponseType, SyncMessage};
+use crate::network_beacon_processor::{NetworkBeaconProcessor, FUTURE_SLOT_TOLERANCE};
-use crate::sync::{BatchProcessResult, ChainId};
+use crate::sync::manager::ResponseType;
-use beacon_chain::blob_verification::BlockWrapper;
+use crate::sync::BatchProcessResult;
-use beacon_chain::blob_verification::{AsBlock, MaybeAvailableBlock};
+use crate::sync::{
+    manager::{BlockProcessType, SyncMessage},
+    ChainId,
+};
+use beacon_chain::blob_verification::{AsBlock, BlockWrapper, MaybeAvailableBlock};
 use beacon_chain::data_availability_checker::AvailabilityCheckError;
-use beacon_chain::AvailabilityProcessingStatus;
 use beacon_chain::{
     observed_block_producers::Error as ObserveError, validator_monitor::get_block_delay_ms,
-    BeaconChainError, BeaconChainTypes, BlockError, ChainSegmentResult, HistoricalBlockError,
+    AvailabilityProcessingStatus, BeaconChainError, BeaconChainTypes, BlockError,
-    NotifyExecutionLayer,
+    ChainSegmentResult, HistoricalBlockError, NotifyExecutionLayer,
+};
+use beacon_processor::{
+    work_reprocessing_queue::{QueuedRpcBlock, ReprocessQueueMessage},
+    AsyncFn, BlockingFn, DuplicateCache,
 };
 use lighthouse_network::PeerAction;
 use slog::{debug, error, info, warn};
 use slot_clock::SlotClock;
+use std::sync::Arc;
+use std::time::Duration;
 use std::time::{SystemTime, UNIX_EPOCH};
 use tokio::sync::mpsc;
 use types::blob_sidecar::FixedBlobSidecarList;
@@ -43,28 +46,72 @@ struct ChainSegmentFailed {
     peer_action: Option<PeerAction>,
 }
 
-impl<T: BeaconChainTypes> Worker<T> {
+impl<T: BeaconChainTypes> NetworkBeaconProcessor<T> {
-    /// Attempt to process a block received from a direct RPC request.
+    /// Returns an async closure which processes a beacon block received via RPC.
-    #[allow(clippy::too_many_arguments)]
+    ///
-    pub async fn process_rpc_block(
+    /// This separate function was required to prevent a cycle during compiler
-        self,
+    /// type checking.
+    pub fn generate_rpc_beacon_block_process_fn(
+        self: Arc<Self>,
         block_root: Hash256,
         block: BlockWrapper<T::EthSpec>,
         seen_timestamp: Duration,
         process_type: BlockProcessType,
-        reprocess_tx: mpsc::Sender<ReprocessQueueMessage<T>>,
+    ) -> AsyncFn {
-        duplicate_cache: DuplicateCache,
+        let process_fn = async move {
-        should_process: bool,
+            let reprocess_tx = self.reprocess_tx.clone();
-    ) {
+            let duplicate_cache = self.duplicate_cache.clone();
-        if !should_process {
+            self.process_rpc_block(
+                block_root,
+                block,
+                seen_timestamp,
+                process_type,
+                reprocess_tx,
+                duplicate_cache,
+            )
+            .await;
+        };
+        Box::pin(process_fn)
+    }
+
+    /// Returns the `process_fn` and `ignore_fn` required when requeuing an RPC block.
+    pub fn generate_rpc_beacon_block_fns(
+        self: Arc<Self>,
+        block_root: Hash256,
+        block: BlockWrapper<T::EthSpec>,
+        seen_timestamp: Duration,
+        process_type: BlockProcessType,
+    ) -> (AsyncFn, BlockingFn) {
+        // An async closure which will import the block.
+        let process_fn = self.clone().generate_rpc_beacon_block_process_fn(
+            block_root,
+            block,
+            seen_timestamp,
+            process_type.clone(),
+        );
+        // A closure which will ignore the block.
+        let ignore_fn = move || {
             // Sync handles these results
             self.send_sync_message(SyncMessage::BlockComponentProcessed {
                 process_type,
                 result: crate::sync::manager::BlockProcessingResult::Ignored,
                 response_type: crate::sync::manager::ResponseType::Block,
             });
-            return;
+        };
-        }
+        (process_fn, Box::new(ignore_fn))
+    }
+
+    /// Attempt to process a block received from a direct RPC request.
+    #[allow(clippy::too_many_arguments)]
+    pub async fn process_rpc_block(
+        self: Arc<NetworkBeaconProcessor<T>>,
+        block_root: Hash256,
+        block: BlockWrapper<T::EthSpec>,
+        seen_timestamp: Duration,
+        process_type: BlockProcessType,
+        reprocess_tx: mpsc::Sender<ReprocessQueueMessage>,
+        duplicate_cache: DuplicateCache,
+    ) {
         // Check if the block is already being imported through another source
         let handle = match duplicate_cache.check_and_insert(block_root) {
             Some(handle) => handle,
@@ -75,13 +122,18 @@ impl<T: BeaconChainTypes> Worker<T> {
                     "action" => "sending rpc block to reprocessing queue",
                     "block_root" => %block_root,
                 );
+
                 // Send message to work reprocess queue to retry the block
-                let reprocess_msg = ReprocessQueueMessage::RpcBlock(QueuedRpcBlock {
+                let (process_fn, ignore_fn) = self.clone().generate_rpc_beacon_block_fns(
                     block_root,
-                    block: block.clone(),
+                    block,
-                    process_type,
                     seen_timestamp,
-                    should_process: true,
+                    process_type,
+                );
+                let reprocess_msg = ReprocessQueueMessage::RpcBlock(QueuedRpcBlock {
+                    beacon_block_root: block_root,
+                    process_fn,
+                    ignore_fn,
                 });
 
                 if reprocess_tx.try_send(reprocess_msg).is_err() {
@@ -135,12 +187,16 @@ impl<T: BeaconChainTypes> Worker<T> {
             );
 
             // Send message to work reprocess queue to retry the block
-            let reprocess_msg = ReprocessQueueMessage::RpcBlock(QueuedRpcBlock {
+            let (process_fn, ignore_fn) = self.clone().generate_rpc_beacon_block_fns(
                 block_root,
-                block: block.clone(),
+                block,
-                process_type,
                 seen_timestamp,
-                should_process: true,
+                process_type,
+            );
+            let reprocess_msg = ReprocessQueueMessage::RpcBlock(QueuedRpcBlock {
+                beacon_block_root: block_root,
+                process_fn,
+                ignore_fn,
             });
 
             if reprocess_tx.try_send(reprocess_msg).is_err() {
@@ -201,8 +257,27 @@ impl<T: BeaconChainTypes> Worker<T> {
         drop(handle);
     }
 
+    /// Returns an async closure which processes a list of blobs received via RPC.
+    ///
+    /// This separate function was required to prevent a cycle during compiler
+    /// type checking.
+    pub fn generate_rpc_blobs_process_fn(
+        self: Arc<Self>,
+        block_root: Hash256,
+        block: FixedBlobSidecarList<T::EthSpec>,
+        seen_timestamp: Duration,
+        process_type: BlockProcessType,
+    ) -> AsyncFn {
+        let process_fn = async move {
+            self.clone()
+                .process_rpc_blobs(block_root, block, seen_timestamp, process_type)
+                .await;
+        };
+        Box::pin(process_fn)
+    }
+
     pub async fn process_rpc_blobs(
-        self,
+        self: Arc<NetworkBeaconProcessor<T>>,
         block_root: Hash256,
         blobs: FixedBlobSidecarList<T::EthSpec>,
         _seen_timestamp: Duration,
@@ -231,6 +306,10 @@ impl<T: BeaconChainTypes> Worker<T> {
         });
     }
 
+    pub fn send_delayed_lookup(&self, block_root: Hash256) {
+        self.send_sync_message(SyncMessage::MissingGossipBlockComponentsDelayed(block_root))
+    }
+
     /// Attempt to import the chain segment (`blocks`) to the beacon chain, informing the sync
     /// thread if more blocks are needed to process it.
     pub async fn process_chain_segment(

beacon_node/network/src/network_beacon_processor/tests.rs

@@ -1,21 +1,25 @@
 #![cfg(test)]
 
-use crate::beacon_processor::work_reprocessing_queue::{
+use crate::{
-    QUEUED_ATTESTATION_DELAY, QUEUED_RPC_BLOCK_DELAY,
+    network_beacon_processor::{
+        ChainSegmentProcessId, DuplicateCache, InvalidBlockStorage, NetworkBeaconProcessor,
+    },
+    service::NetworkMessage,
+    sync::{manager::BlockProcessType, SyncMessage},
 };
-use crate::beacon_processor::*;
-use crate::{service::NetworkMessage, sync::SyncMessage};
 use beacon_chain::test_utils::{
     test_spec, AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType,
 };
 use beacon_chain::{BeaconChain, ChainConfig, WhenSlotSkipped, MAXIMUM_GOSSIP_CLOCK_DISPARITY};
+use beacon_processor::{work_reprocessing_queue::*, *};
 use lighthouse_network::discovery::ConnectionId;
+use lighthouse_network::rpc::methods::BlobsByRangeRequest;
 use lighthouse_network::rpc::SubstreamId;
 use lighthouse_network::{
     discv5::enr::{CombinedKey, EnrBuilder},
     rpc::methods::{MetaData, MetaDataV2},
     types::{EnrAttestationBitfield, EnrSyncCommitteeBitfield},
-    MessageId, NetworkGlobals, PeerId, Response,
+    Client, MessageId, NetworkGlobals, PeerId, Response,
 };
 use slot_clock::SlotClock;
 use std::cmp;
@@ -23,9 +27,11 @@ use std::iter::Iterator;
 use std::sync::Arc;
 use std::time::Duration;
 use tokio::sync::mpsc;
+use types::blob_sidecar::FixedBlobSidecarList;
 use types::{
-    Attestation, AttesterSlashing, Epoch, MainnetEthSpec, ProposerSlashing, SignedBeaconBlock,
+    Attestation, AttesterSlashing, Epoch, Hash256, MainnetEthSpec, ProposerSlashing,
-    SignedBlobSidecarList, SignedVoluntaryExit, Slot, SubnetId,
+    SignedAggregateAndProof, SignedBeaconBlock, SignedBlobSidecarList, SignedVoluntaryExit, Slot,
+    SubnetId,
 };
 
 type E = MainnetEthSpec;
@@ -53,11 +59,12 @@ struct TestRig {
     attester_slashing: AttesterSlashing<E>,
     proposer_slashing: ProposerSlashing,
     voluntary_exit: SignedVoluntaryExit,
-    beacon_processor_tx: mpsc::Sender<WorkEvent<T>>,
+    beacon_processor_tx: BeaconProcessorSend<E>,
     work_journal_rx: mpsc::Receiver<&'static str>,
     _network_rx: mpsc::UnboundedReceiver<NetworkMessage<E>>,
     _sync_rx: mpsc::UnboundedReceiver<SyncMessage<E>>,
     duplicate_cache: DuplicateCache,
+    network_beacon_processor: Arc<NetworkBeaconProcessor<T>>,
     _harness: BeaconChainHarness<T>,
 }
 
@@ -66,7 +73,7 @@ struct TestRig {
 impl Drop for TestRig {
     fn drop(&mut self) {
         // Causes the beacon processor to shutdown.
-        self.beacon_processor_tx = mpsc::channel(MAX_WORK_EVENT_QUEUE_LEN).0;
+        self.beacon_processor_tx = BeaconProcessorSend(mpsc::channel(MAX_WORK_EVENT_QUEUE_LEN).0);
     }
 }
 
@@ -172,6 +179,7 @@ impl TestRig {
         let log = harness.logger().clone();
 
         let (beacon_processor_tx, beacon_processor_rx) = mpsc::channel(MAX_WORK_EVENT_QUEUE_LEN);
+        let beacon_processor_tx = BeaconProcessorSend(beacon_processor_tx);
         let (sync_tx, _sync_rx) = mpsc::unbounded_channel();
 
         // Default metadata
@@ -194,22 +202,40 @@ impl TestRig {
 
         let executor = harness.runtime.task_executor.clone();
 
+        let (work_reprocessing_tx, work_reprocessing_rx) =
+            mpsc::channel(MAX_SCHEDULED_WORK_QUEUE_LEN);
         let (work_journal_tx, work_journal_rx) = mpsc::channel(16_364);
 
         let duplicate_cache = DuplicateCache::default();
-        BeaconProcessor {
+        let network_beacon_processor = NetworkBeaconProcessor {
-            beacon_chain: Arc::downgrade(&chain),
+            beacon_processor_send: beacon_processor_tx.clone(),
+            duplicate_cache: duplicate_cache.clone(),
+            chain: harness.chain.clone(),
             network_tx,
             sync_tx,
+            reprocess_tx: work_reprocessing_tx.clone(),
+            network_globals: network_globals.clone(),
+            invalid_block_storage: InvalidBlockStorage::Disabled,
+            executor: executor.clone(),
+            log: log.clone(),
+        };
+        let network_beacon_processor = Arc::new(network_beacon_processor);
+
+        BeaconProcessor {
             network_globals,
             executor,
             max_workers: cmp::max(1, num_cpus::get()),
             current_workers: 0,
-            importing_blocks: duplicate_cache.clone(),
+            enable_backfill_rate_limiting: harness.chain.config.enable_backfill_rate_limiting,
-            invalid_block_storage: InvalidBlockStorage::Disabled,
             log: log.clone(),
         }
-        .spawn_manager(beacon_processor_rx, Some(work_journal_tx));
+        .spawn_manager(
+            beacon_processor_rx,
+            work_reprocessing_tx,
+            work_reprocessing_rx,
+            Some(work_journal_tx),
+            harness.chain.slot_clock.clone(),
+        );
 
         Self {
             chain,
@@ -226,6 +252,7 @@ impl TestRig {
             _network_rx,
             _sync_rx,
             duplicate_cache,
+            network_beacon_processor,
             _harness: harness,
         }
     }
@@ -239,55 +266,56 @@ impl TestRig {
     }
 
     pub fn enqueue_gossip_block(&self) {
-        self.beacon_processor_tx
+        self.network_beacon_processor
-            .try_send(WorkEvent::gossip_beacon_block(
+            .send_gossip_beacon_block(
                 junk_message_id(),
                 junk_peer_id(),
                 Client::default(),
                 self.next_block.clone(),
                 Duration::from_secs(0),
-            ))
+            )
             .unwrap();
     }
 
     pub fn enqueue_gossip_blob(&self, blob_index: usize) {
         if let Some(blobs) = self.next_blobs.as_ref() {
             let blob = blobs.get(blob_index).unwrap();
-            self.beacon_processor_tx
+            self.network_beacon_processor
-                .try_send(WorkEvent::gossip_signed_blob_sidecar(
+                .send_gossip_blob_sidecar(
                     junk_message_id(),
                     junk_peer_id(),
                     Client::default(),
-                    blob_index as u64,
+                    blob.message.index,
                     blob.clone(),
                     Duration::from_secs(0),
-                ))
+                )
                 .unwrap();
         }
     }
 
     pub fn enqueue_rpc_block(&self) {
-        let event = WorkEvent::rpc_beacon_block(
+        self.network_beacon_processor
-            self.next_block.canonical_root(),
+            .send_rpc_beacon_block(
-            self.next_block.clone().into(),
+                self.next_block.canonical_root(),
-            std::time::Duration::default(),
+                self.next_block.clone().into(),
-            BlockProcessType::ParentLookup {
+                std::time::Duration::default(),
-                chain_hash: Hash256::random(),
+                BlockProcessType::ParentLookup {
-            },
+                    chain_hash: Hash256::random(),
-        );
+                },
-        self.beacon_processor_tx.try_send(event).unwrap();
+            )
+            .unwrap();
     }
 
     pub fn enqueue_single_lookup_rpc_block(&self) {
-        let event = WorkEvent::rpc_beacon_block(
+        self.network_beacon_processor
-            self.next_block.canonical_root(),
+            .send_rpc_beacon_block(
-            self.next_block.clone().into(),
+                self.next_block.canonical_root(),
-            std::time::Duration::default(),
+                self.next_block.clone().into(),
-            BlockProcessType::SingleBlock { id: 1 },
+                std::time::Duration::default(),
-        );
+                BlockProcessType::SingleBlock { id: 1 },
-        self.beacon_processor_tx.try_send(event).unwrap();
+            )
+            .unwrap();
     }
-
     pub fn enqueue_single_lookup_rpc_blobs(&self) {
         if let Some(blobs) = self.next_blobs.clone() {
             let blobs = FixedBlobSidecarList::from(
@@ -296,91 +324,94 @@ impl TestRig {
                     .map(|b| Some(b.message))
                     .collect::<Vec<_>>(),
             );
-            let event = WorkEvent::rpc_blobs(
+            self.network_beacon_processor
-                self.next_block.canonical_root(),
+                .send_rpc_blobs(
-                blobs,
+                    self.next_block.canonical_root(),
-                std::time::Duration::default(),
+                    blobs,
-                BlockProcessType::SingleBlock { id: 1 },
+                    std::time::Duration::default(),
-            );
+                    BlockProcessType::SingleBlock { id: 1 },
-            self.beacon_processor_tx.try_send(event).unwrap();
+                )
+                .unwrap();
         }
     }
 
     pub fn enqueue_blobs_by_range_request(&self, count: u64) {
-        let event = WorkEvent::blobs_by_range_request(
+        self.network_beacon_processor
-            PeerId::random(),
+            .send_blobs_by_range_request(
-            (ConnectionId::new(42), SubstreamId::new(24)),
+                PeerId::random(),
-            BlobsByRangeRequest {
+                (ConnectionId::new(42), SubstreamId::new(24)),
-                start_slot: 0,
+                BlobsByRangeRequest {
-                count,
+                    start_slot: 0,
-            },
+                    count,
-        );
+                },
-        self.beacon_processor_tx.try_send(event).unwrap();
+            )
+            .unwrap();
     }
 
     pub fn enqueue_backfill_batch(&self) {
-        let event = WorkEvent::chain_segment(
+        self.network_beacon_processor
-            ChainSegmentProcessId::BackSyncBatchId(Epoch::default()),
+            .send_chain_segment(
-            Vec::default(),
+                ChainSegmentProcessId::BackSyncBatchId(Epoch::default()),
-        );
+                Vec::default(),
-        self.beacon_processor_tx.try_send(event).unwrap();
+            )
+            .unwrap();
     }
 
     pub fn enqueue_unaggregated_attestation(&self) {
         let (attestation, subnet_id) = self.attestations.first().unwrap().clone();
-        self.beacon_processor_tx
+        self.network_beacon_processor
-            .try_send(WorkEvent::unaggregated_attestation(
+            .send_unaggregated_attestation(
                 junk_message_id(),
                 junk_peer_id(),
                 attestation,
                 subnet_id,
                 true,
                 Duration::from_secs(0),
-            ))
+            )
             .unwrap();
     }
 
     pub fn enqueue_gossip_attester_slashing(&self) {
-        self.beacon_processor_tx
+        self.network_beacon_processor
-            .try_send(WorkEvent::gossip_attester_slashing(
+            .send_gossip_attester_slashing(
                 junk_message_id(),
                 junk_peer_id(),
                 Box::new(self.attester_slashing.clone()),
-            ))
+            )
             .unwrap();
     }
 
     pub fn enqueue_gossip_proposer_slashing(&self) {
-        self.beacon_processor_tx
+        self.network_beacon_processor
-            .try_send(WorkEvent::gossip_proposer_slashing(
+            .send_gossip_proposer_slashing(
                 junk_message_id(),
                 junk_peer_id(),
                 Box::new(self.proposer_slashing.clone()),
-            ))
+            )
             .unwrap();
     }
 
     pub fn enqueue_gossip_voluntary_exit(&self) {
-        self.beacon_processor_tx
+        self.network_beacon_processor
-            .try_send(WorkEvent::gossip_voluntary_exit(
+            .send_gossip_voluntary_exit(
                 junk_message_id(),
                 junk_peer_id(),
                 Box::new(self.voluntary_exit.clone()),
-            ))
+            )
             .unwrap();
     }
 
     pub fn enqueue_next_block_unaggregated_attestation(&self) {
         let (attestation, subnet_id) = self.next_block_attestations.first().unwrap().clone();
-        self.beacon_processor_tx
+        self.network_beacon_processor
-            .try_send(WorkEvent::unaggregated_attestation(
+            .send_unaggregated_attestation(
                 junk_message_id(),
                 junk_peer_id(),
                 attestation,
                 subnet_id,
                 true,
                 Duration::from_secs(0),
-            ))
+            )
             .unwrap();
     }
 
@@ -390,13 +421,13 @@ impl TestRig {
             .first()
             .unwrap()
             .clone();
-        self.beacon_processor_tx
+        self.network_beacon_processor
-            .try_send(WorkEvent::aggregated_attestation(
+            .send_aggregated_attestation(
                 junk_message_id(),
                 junk_peer_id(),
                 aggregate,
                 Duration::from_secs(0),
-            ))
+            )
             .unwrap();
     }
 
@@ -709,7 +740,7 @@ async fn attestation_to_unknown_block_processed(import_method: BlockImportMethod
             events.push(RPC_BLOCK);
             if num_blobs > 0 {
                 rig.enqueue_single_lookup_rpc_blobs();
-                events.push(RPC_BLOB);
+                events.push(RPC_BLOBS);
             }
         }
     };
@@ -792,7 +823,7 @@ async fn aggregate_attestation_to_unknown_block(import_method: BlockImportMethod
             events.push(RPC_BLOCK);
             if num_blobs > 0 {
                 rig.enqueue_single_lookup_rpc_blobs();
-                events.push(RPC_BLOB);
+                events.push(RPC_BLOBS);
             }
         }
     };
@@ -972,7 +1003,7 @@ async fn test_rpc_block_reprocessing() {
 
     rig.enqueue_single_lookup_rpc_blobs();
     if rig.next_blobs.as_ref().map(|b| b.len()).unwrap_or(0) > 0 {
-        rig.assert_event_journal(&[RPC_BLOB, WORKER_FREED, NOTHING_TO_DO])
+        rig.assert_event_journal(&[RPC_BLOBS, WORKER_FREED, NOTHING_TO_DO])
             .await;
     }
 

beacon_node/network/src/router.rs

@@ -5,16 +5,16 @@
 //! syncing-related responses to the Sync manager.
 #![allow(clippy::unit_arg)]
 
-use crate::beacon_processor::{
-    BeaconProcessor, BeaconProcessorSend, InvalidBlockStorage, WorkEvent as BeaconWorkEvent,
-    MAX_WORK_EVENT_QUEUE_LEN,
-};
 use crate::error;
+use crate::network_beacon_processor::{InvalidBlockStorage, NetworkBeaconProcessor};
 use crate::service::{NetworkMessage, RequestId};
 use crate::status::status_message;
 use crate::sync::manager::RequestId as SyncId;
 use crate::sync::SyncMessage;
 use beacon_chain::{BeaconChain, BeaconChainTypes};
+use beacon_processor::{
+    work_reprocessing_queue::ReprocessQueueMessage, BeaconProcessorSend, DuplicateCache,
+};
 use futures::prelude::*;
 use lighthouse_network::rpc::*;
 use lighthouse_network::{
@@ -23,7 +23,6 @@ use lighthouse_network::{
 use logging::TimeLatch;
 use slog::{debug, o, trace};
 use slog::{error, warn};
-use std::cmp;
 use std::sync::Arc;
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
 use tokio::sync::mpsc;
@@ -41,7 +40,7 @@ pub struct Router<T: BeaconChainTypes> {
     /// A network context to return and handle RPC requests.
     network: HandlerNetworkContext<T::EthSpec>,
     /// A multi-threaded, non-blocking processor for applying messages to the beacon chain.
-    beacon_processor_send: BeaconProcessorSend<T>,
+    network_beacon_processor: Arc<NetworkBeaconProcessor<T>>,
     /// The `Router` logger.
     log: slog::Logger,
     /// Provides de-bounce functionality for logging.
@@ -81,12 +80,15 @@ pub enum RouterMessage<T: EthSpec> {
 
 impl<T: BeaconChainTypes> Router<T> {
     /// Initializes and runs the Router.
+    #[allow(clippy::too_many_arguments)]
     pub fn spawn(
         beacon_chain: Arc<BeaconChain<T>>,
         network_globals: Arc<NetworkGlobals<T::EthSpec>>,
         network_send: mpsc::UnboundedSender<NetworkMessage<T::EthSpec>>,
         executor: task_executor::TaskExecutor,
         invalid_block_storage: InvalidBlockStorage,
+        beacon_processor_send: BeaconProcessorSend<T::EthSpec>,
+        beacon_processor_reprocess_tx: mpsc::Sender<ReprocessQueueMessage>,
         log: slog::Logger,
     ) -> error::Result<mpsc::UnboundedSender<RouterMessage<T::EthSpec>>> {
         let message_handler_log = log.new(o!("service"=> "router"));
@@ -94,34 +96,33 @@ impl<T: BeaconChainTypes> Router<T> {
 
         let (handler_send, handler_recv) = mpsc::unbounded_channel();
 
-        let (beacon_processor_send, beacon_processor_receive) =
-            mpsc::channel(MAX_WORK_EVENT_QUEUE_LEN);
-
         let sync_logger = log.new(o!("service"=> "sync"));
+        // generate the message channel
+        let (sync_send, sync_recv) = mpsc::unbounded_channel::<SyncMessage<T::EthSpec>>();
 
-        // spawn the sync thread
+        let network_beacon_processor = NetworkBeaconProcessor {
-        let sync_send = crate::sync::manager::spawn(
+            beacon_processor_send,
-            executor.clone(),
+            duplicate_cache: DuplicateCache::default(),
-            beacon_chain.clone(),
+            chain: beacon_chain.clone(),
-            network_globals.clone(),
-            network_send.clone(),
-            BeaconProcessorSend(beacon_processor_send.clone()),
-            sync_logger,
-        );
-
-        BeaconProcessor {
-            beacon_chain: Arc::downgrade(&beacon_chain),
             network_tx: network_send.clone(),
             sync_tx: sync_send.clone(),
+            reprocess_tx: beacon_processor_reprocess_tx,
             network_globals: network_globals.clone(),
-            executor: executor.clone(),
-            max_workers: cmp::max(1, num_cpus::get()),
-            current_workers: 0,
-            importing_blocks: Default::default(),
             invalid_block_storage,
+            executor: executor.clone(),
             log: log.clone(),
-        }
+        };
-        .spawn_manager(beacon_processor_receive, None);
+        let network_beacon_processor = Arc::new(network_beacon_processor);
+
+        // spawn the sync thread
+        crate::sync::manager::spawn(
+            executor.clone(),
+            beacon_chain.clone(),
+            network_send.clone(),
+            network_beacon_processor.clone(),
+            sync_recv,
+            sync_logger,
+        );
 
         // generate the Message handler
         let mut handler = Router {
@@ -129,7 +130,7 @@ impl<T: BeaconChainTypes> Router<T> {
             chain: beacon_chain,
             sync_send,
             network: HandlerNetworkContext::new(network_send, log.clone()),
-            beacon_processor_send: BeaconProcessorSend(beacon_processor_send),
+            network_beacon_processor,
             log: message_handler_log,
             logger_debounce: TimeLatch::default(),
         };
@@ -199,20 +200,25 @@ impl<T: BeaconChainTypes> Router<T> {
             Request::Status(status_message) => {
                 self.on_status_request(peer_id, request_id, status_message)
             }
-            Request::BlocksByRange(request) => self.send_beacon_processor_work(
+            Request::BlocksByRange(request) => self.handle_beacon_processor_send_result(
-                BeaconWorkEvent::blocks_by_range_request(peer_id, request_id, request),
+                self.network_beacon_processor
+                    .send_blocks_by_range_request(peer_id, request_id, request),
             ),
-            Request::BlocksByRoot(request) => self.send_beacon_processor_work(
+            Request::BlocksByRoot(request) => self.handle_beacon_processor_send_result(
-                BeaconWorkEvent::blocks_by_roots_request(peer_id, request_id, request),
+                self.network_beacon_processor
+                    .send_blocks_by_roots_request(peer_id, request_id, request),
             ),
-            Request::BlobsByRange(request) => self.send_beacon_processor_work(
+            Request::BlobsByRange(request) => self.handle_beacon_processor_send_result(
-                BeaconWorkEvent::blobs_by_range_request(peer_id, request_id, request),
+                self.network_beacon_processor
+                    .send_blobs_by_range_request(peer_id, request_id, request),
             ),
-            Request::BlobsByRoot(request) => self.send_beacon_processor_work(
+            Request::BlobsByRoot(request) => self.handle_beacon_processor_send_result(
-                BeaconWorkEvent::blobs_by_root_request(peer_id, request_id, request),
+                self.network_beacon_processor
+                    .send_blobs_by_roots_request(peer_id, request_id, request),
             ),
-            Request::LightClientBootstrap(request) => self.send_beacon_processor_work(
+            Request::LightClientBootstrap(request) => self.handle_beacon_processor_send_result(
-                BeaconWorkEvent::lightclient_bootstrap_request(peer_id, request_id, request),
+                self.network_beacon_processor
+                    .send_lightclient_bootstrap_request(peer_id, request_id, request),
             ),
         }
     }
@@ -227,10 +233,10 @@ impl<T: BeaconChainTypes> Router<T> {
         match response {
             Response::Status(status_message) => {
                 debug!(self.log, "Received Status Response"; "peer_id" => %peer_id, &status_message);
-                self.send_beacon_processor_work(BeaconWorkEvent::status_message(
+                self.handle_beacon_processor_send_result(
-                    peer_id,
+                    self.network_beacon_processor
-                    status_message,
+                        .send_status_message(peer_id, status_message),
-                ))
+                )
             }
             Response::BlocksByRange(beacon_block) => {
                 self.on_blocks_by_range_response(peer_id, request_id, beacon_block);
@@ -261,48 +267,53 @@ impl<T: BeaconChainTypes> Router<T> {
     ) {
         match gossip_message {
             PubsubMessage::AggregateAndProofAttestation(aggregate_and_proof) => self
-                .send_beacon_processor_work(BeaconWorkEvent::aggregated_attestation(
+                .handle_beacon_processor_send_result(
-                    message_id,
+                    self.network_beacon_processor.send_aggregated_attestation(
-                    peer_id,
+                        message_id,
-                    *aggregate_and_proof,
+                        peer_id,
-                    timestamp_now(),
+                        *aggregate_and_proof,
-                )),
+                        timestamp_now(),
-            PubsubMessage::Attestation(subnet_attestation) => {
+                    ),
-                self.send_beacon_processor_work(BeaconWorkEvent::unaggregated_attestation(
+                ),
-                    message_id,
+            PubsubMessage::Attestation(subnet_attestation) => self
-                    peer_id,
+                .handle_beacon_processor_send_result(
-                    subnet_attestation.1,
+                    self.network_beacon_processor.send_unaggregated_attestation(
-                    subnet_attestation.0,
+                        message_id,
-                    should_process,
+                        peer_id,
-                    timestamp_now(),
+                        subnet_attestation.1,
-                ))
+                        subnet_attestation.0,
-            }
+                        should_process,
-            PubsubMessage::BeaconBlock(block) => {
+                        timestamp_now(),
-                self.send_beacon_processor_work(BeaconWorkEvent::gossip_beacon_block(
+                    ),
+                ),
+            PubsubMessage::BeaconBlock(block) => self.handle_beacon_processor_send_result(
+                self.network_beacon_processor.send_gossip_beacon_block(
                     message_id,
                     peer_id,
                     self.network_globals.client(&peer_id),
                     block,
                     timestamp_now(),
-                ))
+                ),
-            }
+            ),
             PubsubMessage::BlobSidecar(data) => {
                 let (blob_index, signed_blob) = *data;
-                let peer_client = self.network_globals.client(&peer_id);
+                self.handle_beacon_processor_send_result(
-                self.send_beacon_processor_work(BeaconWorkEvent::gossip_signed_blob_sidecar(
+                    self.network_beacon_processor.send_gossip_blob_sidecar(
-                    message_id,
+                        message_id,
-                    peer_id,
+                        peer_id,
-                    peer_client,
+                        self.network_globals.client(&peer_id),
-                    blob_index,
+                        blob_index,
-                    signed_blob,
+                        signed_blob,
-                    timestamp_now(),
+                        timestamp_now(),
-                ))
+                    ),
+                )
             }
             PubsubMessage::VoluntaryExit(exit) => {
                 debug!(self.log, "Received a voluntary exit"; "peer_id" => %peer_id);
-                self.send_beacon_processor_work(BeaconWorkEvent::gossip_voluntary_exit(
+                self.handle_beacon_processor_send_result(
-                    message_id, peer_id, exit,
+                    self.network_beacon_processor
-                ))
+                        .send_gossip_voluntary_exit(message_id, peer_id, exit),
+                )
             }
             PubsubMessage::ProposerSlashing(proposer_slashing) => {
                 debug!(
@@ -310,11 +321,13 @@ impl<T: BeaconChainTypes> Router<T> {
                     "Received a proposer slashing";
                     "peer_id" => %peer_id
                 );
-                self.send_beacon_processor_work(BeaconWorkEvent::gossip_proposer_slashing(
+                self.handle_beacon_processor_send_result(
-                    message_id,
+                    self.network_beacon_processor.send_gossip_proposer_slashing(
-                    peer_id,
+                        message_id,
-                    proposer_slashing,
+                        peer_id,
-                ))
+                        proposer_slashing,
+                    ),
+                )
             }
             PubsubMessage::AttesterSlashing(attester_slashing) => {
                 debug!(
@@ -322,11 +335,13 @@ impl<T: BeaconChainTypes> Router<T> {
                     "Received a attester slashing";
                     "peer_id" => %peer_id
                 );
-                self.send_beacon_processor_work(BeaconWorkEvent::gossip_attester_slashing(
+                self.handle_beacon_processor_send_result(
-                    message_id,
+                    self.network_beacon_processor.send_gossip_attester_slashing(
-                    peer_id,
+                        message_id,
-                    attester_slashing,
+                        peer_id,
-                ))
+                        attester_slashing,
+                    ),
+                )
             }
             PubsubMessage::SignedContributionAndProof(contribution_and_proof) => {
                 trace!(
@@ -334,12 +349,14 @@ impl<T: BeaconChainTypes> Router<T> {
                     "Received sync committee aggregate";
                     "peer_id" => %peer_id
                 );
-                self.send_beacon_processor_work(BeaconWorkEvent::gossip_sync_contribution(
+                self.handle_beacon_processor_send_result(
-                    message_id,
+                    self.network_beacon_processor.send_gossip_sync_contribution(
-                    peer_id,
+                        message_id,
-                    *contribution_and_proof,
+                        peer_id,
-                    timestamp_now(),
+                        *contribution_and_proof,
-                ))
+                        timestamp_now(),
+                    ),
+                )
             }
             PubsubMessage::SyncCommitteeMessage(sync_committtee_msg) => {
                 trace!(
@@ -347,13 +364,15 @@ impl<T: BeaconChainTypes> Router<T> {
                     "Received sync committee signature";
                     "peer_id" => %peer_id
                 );
-                self.send_beacon_processor_work(BeaconWorkEvent::gossip_sync_signature(
+                self.handle_beacon_processor_send_result(
-                    message_id,
+                    self.network_beacon_processor.send_gossip_sync_signature(
-                    peer_id,
+                        message_id,
-                    sync_committtee_msg.1,
+                        peer_id,
-                    sync_committtee_msg.0,
+                        sync_committtee_msg.1,
-                    timestamp_now(),
+                        sync_committtee_msg.0,
-                ))
+                        timestamp_now(),
+                    ),
+                )
             }
             PubsubMessage::LightClientFinalityUpdate(light_client_finality_update) => {
                 trace!(
@@ -361,13 +380,14 @@ impl<T: BeaconChainTypes> Router<T> {
                     "Received light client finality update";
                     "peer_id" => %peer_id
                 );
-                self.send_beacon_processor_work(
+                self.handle_beacon_processor_send_result(
-                    BeaconWorkEvent::gossip_light_client_finality_update(
+                    self.network_beacon_processor
-                        message_id,
+                        .send_gossip_light_client_finality_update(
-                        peer_id,
+                            message_id,
-                        light_client_finality_update,
+                            peer_id,
-                        timestamp_now(),
+                            *light_client_finality_update,
-                    ),
+                            timestamp_now(),
+                        ),
                 )
             }
             PubsubMessage::LightClientOptimisticUpdate(light_client_optimistic_update) => {
@@ -376,21 +396,25 @@ impl<T: BeaconChainTypes> Router<T> {
                     "Received light client optimistic update";
                     "peer_id" => %peer_id
                 );
-                self.send_beacon_processor_work(
+                self.handle_beacon_processor_send_result(
-                    BeaconWorkEvent::gossip_light_client_optimistic_update(
+                    self.network_beacon_processor
-                        message_id,
+                        .send_gossip_light_client_optimistic_update(
-                        peer_id,
+                            message_id,
-                        light_client_optimistic_update,
+                            peer_id,
-                        timestamp_now(),
+                            *light_client_optimistic_update,
-                    ),
+                            timestamp_now(),
+                        ),
                 )
             }
             PubsubMessage::BlsToExecutionChange(bls_to_execution_change) => self
-                .send_beacon_processor_work(BeaconWorkEvent::gossip_bls_to_execution_change(
+                .handle_beacon_processor_send_result(
-                    message_id,
+                    self.network_beacon_processor
-                    peer_id,
+                        .send_gossip_bls_to_execution_change(
-                    bls_to_execution_change,
+                            message_id,
-                )),
+                            peer_id,
+                            bls_to_execution_change,
+                        ),
+                ),
         }
     }
 
@@ -442,7 +466,10 @@ impl<T: BeaconChainTypes> Router<T> {
             request_id,
         );
 
-        self.send_beacon_processor_work(BeaconWorkEvent::status_message(peer_id, status))
+        self.handle_beacon_processor_send_result(
+            self.network_beacon_processor
+                .send_status_message(peer_id, status),
+        )
     }
 
     /// Handle a `BlocksByRange` response from the peer.
@@ -573,20 +600,22 @@ impl<T: BeaconChainTypes> Router<T> {
         });
     }
 
-    fn send_beacon_processor_work(&mut self, work: BeaconWorkEvent<T>) {
+    fn handle_beacon_processor_send_result(
-        self.beacon_processor_send
+        &mut self,
-            .try_send(work)
+        result: Result<(), crate::network_beacon_processor::Error<T::EthSpec>>,
-            .unwrap_or_else(|e| {
+    ) {
-                let work_type = match &*e {
+        if let Err(e) = result {
-                    mpsc::error::TrySendError::Closed(work)
+            let work_type = match &e {
-                    | mpsc::error::TrySendError::Full(work) => work.work_type(),
+                mpsc::error::TrySendError::Closed(work) | mpsc::error::TrySendError::Full(work) => {
-                };
+                    work.work_type()
-
-                if self.logger_debounce.elapsed() {
-                    error!(&self.log, "Unable to send message to the beacon processor";
-                        "error" => %e, "type" => work_type)
                 }
-            })
+            };
+
+            if self.logger_debounce.elapsed() {
+                error!(&self.log, "Unable to send message to the beacon processor";
+                    "error" => %e, "type" => work_type)
+            }
+        }
     }
 }
 

beacon_node/network/src/service.rs

@@ -1,5 +1,5 @@
 use super::sync::manager::RequestId as SyncId;
-use crate::beacon_processor::InvalidBlockStorage;
+use crate::network_beacon_processor::InvalidBlockStorage;
 use crate::persisted_dht::{clear_dht, load_dht, persist_dht};
 use crate::router::{Router, RouterMessage};
 use crate::subnet_service::SyncCommitteeService;
@@ -9,6 +9,7 @@ use crate::{
     NetworkConfig,
 };
 use beacon_chain::{BeaconChain, BeaconChainTypes};
+use beacon_processor::{work_reprocessing_queue::ReprocessQueueMessage, BeaconProcessorSend};
 use futures::channel::mpsc::Sender;
 use futures::future::OptionFuture;
 use futures::prelude::*;
@@ -224,6 +225,8 @@ impl<T: BeaconChainTypes> NetworkService<T> {
         config: &NetworkConfig,
         executor: task_executor::TaskExecutor,
         gossipsub_registry: Option<&'_ mut Registry>,
+        beacon_processor_send: BeaconProcessorSend<T::EthSpec>,
+        beacon_processor_reprocess_tx: mpsc::Sender<ReprocessQueueMessage>,
     ) -> error::Result<(Arc<NetworkGlobals<T::EthSpec>>, NetworkSenders<T::EthSpec>)> {
         let network_log = executor.log().clone();
         // build the channels for external comms
@@ -311,6 +314,8 @@ impl<T: BeaconChainTypes> NetworkService<T> {
             network_senders.network_send(),
             executor.clone(),
             invalid_block_storage,
+            beacon_processor_send,
+            beacon_processor_reprocess_tx,
             network_log.clone(),
         )?;
 

beacon_node/network/src/service/tests.rs

@@ -4,12 +4,15 @@ mod tests {
     use crate::persisted_dht::load_dht;
     use crate::{NetworkConfig, NetworkService};
     use beacon_chain::test_utils::EphemeralHarnessType;
+    use beacon_processor::{
+        BeaconProcessorSend, MAX_SCHEDULED_WORK_QUEUE_LEN, MAX_WORK_EVENT_QUEUE_LEN,
+    };
     use lighthouse_network::Enr;
     use slog::{o, Drain, Level, Logger};
     use sloggers::{null::NullLoggerBuilder, Build};
     use std::str::FromStr;
     use std::sync::Arc;
-    use tokio::runtime::Runtime;
+    use tokio::{runtime::Runtime, sync::mpsc};
     use types::MinimalEthSpec as E;
 
     type BeaconChainHarness = beacon_chain::test_utils::BeaconChainHarness<EphemeralHarnessType<E>>;
@@ -69,10 +72,20 @@ mod tests {
             // Create a new network service which implicitly gets dropped at the
             // end of the block.
 
-            let _network_service =
+            let (beacon_processor_send, _beacon_processor_receive) =
-                NetworkService::start(beacon_chain.clone(), &config, executor, None)
+                mpsc::channel(MAX_WORK_EVENT_QUEUE_LEN);
-                    .await
+            let (beacon_processor_reprocess_tx, _beacon_processor_reprocess_rx) =
-                    .unwrap();
+                mpsc::channel(MAX_SCHEDULED_WORK_QUEUE_LEN);
+            let _network_service = NetworkService::start(
+                beacon_chain.clone(),
+                &config,
+                executor,
+                None,
+                BeaconProcessorSend(beacon_processor_send),
+                beacon_processor_reprocess_tx,
+            )
+            .await
+            .unwrap();
             drop(signal);
         });
 

beacon_node/network/src/sync/backfill_sync/mod.rs

@@ -8,7 +8,7 @@
 //! If a batch fails, the backfill sync cannot progress. In this scenario, we mark the backfill
 //! sync as failed, log an error and attempt to retry once a new peer joins the node.
 
-use crate::beacon_processor::{ChainSegmentProcessId, WorkEvent as BeaconWorkEvent};
+use crate::network_beacon_processor::ChainSegmentProcessId;
 use crate::sync::manager::{BatchProcessResult, Id};
 use crate::sync::network_context::SyncNetworkContext;
 use crate::sync::range_sync::{
@@ -537,8 +537,10 @@ impl<T: BeaconChainTypes> BackFillSync<T> {
         let process_id = ChainSegmentProcessId::BackSyncBatchId(batch_id);
         self.current_processing_batch = Some(batch_id);
 
-        let work_event = BeaconWorkEvent::chain_segment(process_id, blocks);
+        if let Err(e) = network
-        if let Err(e) = network.processor_channel().try_send(work_event) {
+            .beacon_processor()
+            .send_chain_segment(process_id, blocks)
+        {
             crit!(self.log, "Failed to send backfill segment to processor."; "msg" => "process_batch",
                 "error" => %e, "batch" => self.processing_target);
             // This is unlikely to happen but it would stall syncing since the batch now has no
@@ -1097,12 +1099,7 @@ impl<T: BeaconChainTypes> BackFillSync<T> {
         match self.batches.entry(batch_id) {
             Entry::Occupied(_) => {
                 // this batch doesn't need downloading, let this same function decide the next batch
-                if batch_id
+                if self.would_complete(batch_id) {
-                    == self
-                        .beacon_chain
-                        .genesis_backfill_slot
-                        .epoch(T::EthSpec::slots_per_epoch())
-                {
                     self.last_batch_downloaded = true;
                 }
 
@@ -1118,12 +1115,7 @@ impl<T: BeaconChainTypes> BackFillSync<T> {
                     BACKFILL_EPOCHS_PER_BATCH,
                     batch_type,
                 ));
-                if batch_id
+                if self.would_complete(batch_id) {
-                    == self
-                        .beacon_chain
-                        .genesis_backfill_slot
-                        .epoch(T::EthSpec::slots_per_epoch())
-                {
                     self.last_batch_downloaded = true;
                 }
                 self.to_be_downloaded = self
@@ -1155,14 +1147,8 @@ impl<T: BeaconChainTypes> BackFillSync<T> {
 
     /// Checks with the beacon chain if backfill sync has completed.
     fn check_completed(&mut self) -> bool {
-        if self.current_start
+        if self.would_complete(self.current_start) {
-            == self
-                .beacon_chain
-                .genesis_backfill_slot
-                .epoch(T::EthSpec::slots_per_epoch())
-        {
             // Check that the beacon chain agrees
-
             if let Some(anchor_info) = self.beacon_chain.store.get_anchor_info() {
                 // Conditions that we have completed a backfill sync
                 if anchor_info.block_backfill_complete(self.beacon_chain.genesis_backfill_slot) {
@@ -1175,6 +1161,15 @@ impl<T: BeaconChainTypes> BackFillSync<T> {
         false
     }
 
+    /// Checks if backfill would complete by syncing to `start_epoch`.
+    fn would_complete(&self, start_epoch: Epoch) -> bool {
+        start_epoch
+            <= self
+                .beacon_chain
+                .genesis_backfill_slot
+                .epoch(T::EthSpec::slots_per_epoch())
+    }
+
     /// Updates the global network state indicating the current state of a backfill sync.
     fn set_state(&self, state: BackFillState) {
         *self.network_globals.backfill_state.write() = state;

beacon_node/network/src/sync/block_lookups/delayed_lookup.rs

@@ -1,6 +1,6 @@
-use crate::sync::SyncMessage;
+use crate::network_beacon_processor::NetworkBeaconProcessor;
 use beacon_chain::{BeaconChain, BeaconChainTypes};
-use slog::{crit, warn};
+use slog::crit;
 use slot_clock::SlotClock;
 use std::sync::Arc;
 use tokio::sync::mpsc;
@@ -35,7 +35,7 @@ pub fn spawn_delayed_lookup_service<T: BeaconChainTypes>(
     executor: &task_executor::TaskExecutor,
     beacon_chain: Arc<BeaconChain<T>>,
     mut delayed_lookups_recv: mpsc::Receiver<DelayedLookupMessage>,
-    sync_send: mpsc::UnboundedSender<SyncMessage<T::EthSpec>>,
+    beacon_processor: Arc<NetworkBeaconProcessor<T>>,
     log: slog::Logger,
 ) {
     executor.spawn(
@@ -52,8 +52,8 @@ pub fn spawn_delayed_lookup_service<T: BeaconChainTypes>(
                     } else {
                         delay - seconds_from_current_slot_start
                     };
-                    tokio::time::Instant::now() + duration_until_start
+                    Instant::now() + duration_until_start
-                                  }
+                }
                 _ => {
                     crit!(log,
                         "Failed to read slot clock, delayed lookup service timing will be inaccurate.\
@@ -69,11 +69,8 @@ pub fn spawn_delayed_lookup_service<T: BeaconChainTypes>(
                 while let Ok(msg) = delayed_lookups_recv.try_recv() {
                     match msg {
                         DelayedLookupMessage::MissingComponents(block_root) => {
-                            if let Err(e) = sync_send
+                            beacon_processor
-                                .send(SyncMessage::MissingGossipBlockComponentsDelayed(block_root))
+                                .send_delayed_lookup(block_root)
-                            {
-                                warn!(log, "Failed to send delayed lookup message"; "error" => ?e);
-                            }
                         }
                     }
                 }

beacon_node/network/src/sync/block_lookups/mod.rs

@@ -1,20 +1,3 @@
-use beacon_chain::blob_verification::{AsBlock, BlockWrapper};
-use beacon_chain::data_availability_checker::{AvailabilityCheckError, DataAvailabilityChecker};
-use beacon_chain::{AvailabilityProcessingStatus, BeaconChainTypes, BlockError};
-use lighthouse_network::rpc::RPCError;
-use lighthouse_network::{PeerAction, PeerId};
-use lru_cache::LRUTimeCache;
-use slog::{debug, error, trace, warn, Logger};
-use smallvec::SmallVec;
-use std::collections::HashMap;
-use std::fmt::Debug;
-use std::sync::Arc;
-use std::time::Duration;
-use store::Hash256;
-use strum::Display;
-use types::blob_sidecar::FixedBlobSidecarList;
-use types::{BlobSidecar, SignedBeaconBlock, Slot};
-
 use self::parent_lookup::PARENT_FAIL_TOLERANCE;
 use self::parent_lookup::{ParentLookup, ParentVerifyError};
 use self::single_block_lookup::{LookupVerifyError, SingleBlockLookup};
@@ -24,10 +7,26 @@ use super::{
     manager::{BlockProcessType, Id},
     network_context::SyncNetworkContext,
 };
-use crate::beacon_processor::{ChainSegmentProcessId, WorkEvent};
 use crate::metrics;
+use crate::network_beacon_processor::ChainSegmentProcessId;
 use crate::sync::block_lookups::single_block_lookup::LookupId;
+use beacon_chain::blob_verification::{AsBlock, BlockWrapper};
+use beacon_chain::data_availability_checker::{AvailabilityCheckError, DataAvailabilityChecker};
+use beacon_chain::{AvailabilityProcessingStatus, BeaconChainTypes, BlockError};
+use lighthouse_network::rpc::RPCError;
+use lighthouse_network::{PeerAction, PeerId};
+use lru_cache::LRUTimeCache;
 pub use single_block_lookup::UnknownParentComponents;
+use slog::{debug, error, trace, warn, Logger};
+use smallvec::SmallVec;
+use std::collections::HashMap;
+use std::fmt::Debug;
+use std::sync::Arc;
+use std::time::Duration;
+use store::{Hash256, SignedBeaconBlock};
+use strum::Display;
+use types::blob_sidecar::FixedBlobSidecarList;
+use types::{BlobSidecar, Slot};
 
 pub(crate) mod delayed_lookup;
 mod parent_lookup;
@@ -380,13 +379,13 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
                 };
 
                 if !has_pending_parent_request {
-                    let block_wrapper = request_ref
+                    let rpc_block = request_ref
                         .get_downloaded_block()
                         .unwrap_or(BlockWrapper::Block(block));
                     // This is the correct block, send it for processing
                     match self.send_block_for_processing(
                         block_root,
-                        block_wrapper,
+                        rpc_block,
                         seen_timestamp,
                         BlockProcessType::SingleBlock { id },
                         cx,
@@ -562,14 +561,13 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
         match parent_lookup.verify_block(block, &mut self.failed_chains) {
             Ok(Some((block_root, block))) => {
                 parent_lookup.add_current_request_block(block);
-                if let Some(block_wrapper) =
+                if let Some(rpc_block) = parent_lookup.current_parent_request.get_downloaded_block()
-                    parent_lookup.current_parent_request.get_downloaded_block()
                 {
                     let chain_hash = parent_lookup.chain_hash();
                     if self
                         .send_block_for_processing(
                             block_root,
-                            block_wrapper,
+                            rpc_block,
                             seen_timestamp,
                             BlockProcessType::ParentLookup { chain_hash },
                             cx,
@@ -643,13 +641,12 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
             Ok(Some((block_root, blobs))) => {
                 parent_lookup.add_current_request_blobs(blobs);
                 let chain_hash = parent_lookup.chain_hash();
-                if let Some(block_wrapper) =
+                if let Some(rpc_block) = parent_lookup.current_parent_request.get_downloaded_block()
-                    parent_lookup.current_parent_request.get_downloaded_block()
                 {
                     if self
                         .send_block_for_processing(
                             block_root,
-                            block_wrapper,
+                            rpc_block,
                             seen_timestamp,
                             BlockProcessType::ParentLookup { chain_hash },
                             cx,
@@ -914,10 +911,10 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
                         let slot = block.slot();
                         let parent_root = block.parent_root();
                         let (block, blobs) = block.deconstruct();
-                        request_ref.add_unknown_parent_block(block);
+                        request_ref.add_unknown_parent_components(UnknownParentComponents::new(
-                        if let Some(blobs) = blobs {
+                            Some(block),
-                            request_ref.add_unknown_parent_blobs(blobs);
+                            blobs,
-                        }
+                        ));
                         self.search_parent(slot, root, parent_root, peer_id.to_peer_id(), cx);
                         ShouldRemoveLookup::False
                     }
@@ -1054,8 +1051,8 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
             BlockProcessingResult::Ok(AvailabilityProcessingStatus::Imported(_))
             | BlockProcessingResult::Err(BlockError::BlockIsAlreadyKnown { .. }) => {
                 // Check if the beacon processor is available
-                let beacon_processor_send = match cx.processor_channel_if_enabled() {
+                let beacon_processor = match cx.beacon_processor_if_enabled() {
-                    Some(channel) => channel,
+                    Some(beacon_processor) => beacon_processor,
                     None => {
                         return trace!(
                             self.log,
@@ -1076,9 +1073,8 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
                     blocks.push(child_block);
                 };
                 let process_id = ChainSegmentProcessId::ParentLookup(chain_hash);
-                let work = WorkEvent::chain_segment(process_id, blocks);
 
-                match beacon_processor_send.try_send(work) {
+                match beacon_processor.send_chain_segment(process_id, blocks) {
                     Ok(_) => {
                         self.processing_parent_lookups
                             .insert(chain_hash, (hashes, block_request));
@@ -1170,7 +1166,7 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
                     .enumerate()
                     .find(|(_, req)| req.block_request_state.requested_block_root == chain_hash)
                 {
-                    if let Some((lookup_id, block_wrapper)) =
+                    if let Some((lookup_id, rpc_block)) =
                         self.single_block_lookups.get_mut(index).and_then(|lookup| {
                             lookup
                                 .get_downloaded_block()
@@ -1190,7 +1186,7 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
                         if self
                             .send_block_for_processing(
                                 chain_hash,
-                                block_wrapper,
+                                rpc_block,
                                 Duration::from_secs(0), //TODO(sean) pipe this through
                                 BlockProcessType::SingleBlock { id },
                                 cx,
@@ -1235,11 +1231,15 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
         process_type: BlockProcessType,
         cx: &mut SyncNetworkContext<T>,
     ) -> Result<(), ()> {
-        match cx.processor_channel_if_enabled() {
+        match cx.beacon_processor_if_enabled() {
-            Some(beacon_processor_send) => {
+            Some(beacon_processor) => {
                 trace!(self.log, "Sending block for processing"; "block" => ?block_root, "process" => ?process_type);
-                let event = WorkEvent::rpc_beacon_block(block_root, block, duration, process_type);
+                if let Err(e) = beacon_processor.send_rpc_beacon_block(
-                if let Err(e) = beacon_processor_send.try_send(event) {
+                    block_root,
+                    block,
+                    duration,
+                    process_type,
+                ) {
                     error!(
                         self.log,
                         "Failed to send sync block to processor";
@@ -1269,11 +1269,12 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
         if blob_count == 0 {
             return Ok(());
         }
-        match cx.processor_channel_if_enabled() {
+        match cx.beacon_processor_if_enabled() {
-            Some(beacon_processor_send) => {
+            Some(beacon_processor) => {
                 trace!(self.log, "Sending blobs for processing"; "block" => ?block_root, "process_type" => ?process_type);
-                let event = WorkEvent::rpc_blobs(block_root, blobs, duration, process_type);
+                if let Err(e) =
-                if let Err(e) = beacon_processor_send.try_send(event) {
+                    beacon_processor.send_rpc_blobs(block_root, blobs, duration, process_type)
+                {
                     error!(
                         self.log,
                         "Failed to send sync blobs to processor";

beacon_node/network/src/sync/block_lookups/tests.rs

@@ -1,24 +1,20 @@
 #![cfg(feature = "spec-minimal")]
-use std::sync::Arc;
+use crate::network_beacon_processor::NetworkBeaconProcessor;
-
-use crate::beacon_processor::BeaconProcessorSend;
 use crate::service::RequestId;
 use crate::sync::manager::RequestId as SyncId;
 use crate::NetworkMessage;
+use std::sync::Arc;
 
 use super::*;
 
-use beacon_chain::{
+use beacon_chain::builder::Witness;
-    builder::Witness,
+use beacon_chain::eth1_chain::CachingEth1Backend;
-    eth1_chain::CachingEth1Backend,
+use beacon_chain::test_utils::{build_log, BeaconChainHarness, EphemeralHarnessType};
-    test_utils::{build_log, BeaconChainHarness, EphemeralHarnessType},
+use beacon_processor::WorkEvent;
-};
 use execution_layer::BlobsBundleV1;
-pub use genesis::{interop_genesis_state, DEFAULT_ETH1_BLOCK_HASH};
 use lighthouse_network::rpc::RPCResponseErrorCode;
 use lighthouse_network::{NetworkGlobals, Request};
-use slot_clock::{SlotClock, TestingSlotClock};
+use slot_clock::{ManualSlotClock, SlotClock, TestingSlotClock};
-use std::time::Duration;
 use store::MemoryStore;
 use tokio::sync::mpsc;
 use types::{
@@ -27,10 +23,10 @@ use types::{
     BeaconBlock, EthSpec, ForkName, FullPayloadDeneb, MinimalEthSpec as E, SignedBeaconBlock,
 };
 
-type T = Witness<TestingSlotClock, CachingEth1Backend<E>, E, MemoryStore<E>, MemoryStore<E>>;
+type T = Witness<ManualSlotClock, CachingEth1Backend<E>, E, MemoryStore<E>, MemoryStore<E>>;
 
 struct TestRig {
-    beacon_processor_rx: mpsc::Receiver<WorkEvent<T>>,
+    beacon_processor_rx: mpsc::Receiver<WorkEvent<E>>,
     network_rx: mpsc::UnboundedReceiver<NetworkMessage<E>>,
     rng: XorShiftRng,
     harness: BeaconChainHarness<T>,
@@ -62,8 +58,10 @@ impl TestRig {
 
         let chain = harness.chain.clone();
 
-        let (beacon_processor_tx, beacon_processor_rx) = mpsc::channel(100);
         let (network_tx, network_rx) = mpsc::unbounded_channel();
+        let globals = Arc::new(NetworkGlobals::new_test_globals(Vec::new(), &log));
+        let (network_beacon_processor, beacon_processor_rx) =
+            NetworkBeaconProcessor::null_for_testing(globals);
         let rng = XorShiftRng::from_seed([42; 16]);
         let rig = TestRig {
             beacon_processor_rx,
@@ -77,11 +75,9 @@ impl TestRig {
             log.new(slog::o!("component" => "block_lookups")),
         );
         let cx = {
-            let globals = Arc::new(NetworkGlobals::new_test_globals(Vec::new(), &log));
             SyncNetworkContext::new(
                 network_tx,
-                globals,
+                Arc::new(network_beacon_processor),
-                BeaconProcessorSend(beacon_processor_tx),
                 chain,
                 log.new(slog::o!("component" => "network_context")),
             )
@@ -105,7 +101,7 @@ impl TestRig {
         let mut blob_sidecars = vec![];
         if let Ok(message) = block.message_deneb_mut() {
             // get random number between 0 and Max Blobs
-            let mut payload: &mut FullPayloadDeneb<E> = &mut message.body.execution_payload;
+            let payload: &mut FullPayloadDeneb<E> = &mut message.body.execution_payload;
             let num_blobs = match num_blobs {
                 NumBlobs::Random => {
                     let mut num_blobs = rand::random::<usize>() % E::max_blobs_per_block();
@@ -211,13 +207,13 @@ impl TestRig {
         match response_type {
             ResponseType::Block => match self.beacon_processor_rx.try_recv() {
                 Ok(work) => {
-                    assert_eq!(work.work_type(), crate::beacon_processor::RPC_BLOCK);
+                    assert_eq!(work.work_type(), beacon_processor::RPC_BLOCK);
                 }
                 other => panic!("Expected block process, found {:?}", other),
             },
             ResponseType::Blob => match self.beacon_processor_rx.try_recv() {
                 Ok(work) => {
-                    assert_eq!(work.work_type(), crate::beacon_processor::RPC_BLOB);
+                    assert_eq!(work.work_type(), beacon_processor::RPC_BLOBS);
                 }
                 other => panic!("Expected blob process, found {:?}", other),
             },
@@ -228,7 +224,7 @@ impl TestRig {
     fn expect_parent_chain_process(&mut self) {
         match self.beacon_processor_rx.try_recv() {
             Ok(work) => {
-                assert_eq!(work.work_type(), crate::beacon_processor::CHAIN_SEGMENT);
+                assert_eq!(work.work_type(), beacon_processor::CHAIN_SEGMENT);
             }
             other => panic!("Expected chain segment process, found {:?}", other),
         }

beacon_node/network/src/sync/manager.rs

@@ -38,7 +38,7 @@ use super::block_lookups::{BlockLookups, PeerShouldHave};
 use super::network_context::{BlockOrBlob, SyncNetworkContext};
 use super::peer_sync_info::{remote_sync_type, PeerSyncType};
 use super::range_sync::{RangeSync, RangeSyncType, EPOCHS_PER_BATCH};
-use crate::beacon_processor::{BeaconProcessorSend, ChainSegmentProcessId};
+use crate::network_beacon_processor::{ChainSegmentProcessId, NetworkBeaconProcessor};
 use crate::service::NetworkMessage;
 use crate::status::ToStatusMessage;
 use crate::sync::block_lookups::delayed_lookup;
@@ -207,9 +207,6 @@ pub struct SyncManager<T: BeaconChainTypes> {
     /// A reference to the underlying beacon chain.
     chain: Arc<BeaconChain<T>>,
 
-    /// A reference to the network globals and peer-db.
-    network_globals: Arc<NetworkGlobals<T::EthSpec>>,
-
     /// A receiving channel sent by the message processor thread.
     input_channel: mpsc::UnboundedReceiver<SyncMessage<T::EthSpec>>,
 
@@ -236,29 +233,26 @@ pub struct SyncManager<T: BeaconChainTypes> {
 pub fn spawn<T: BeaconChainTypes>(
     executor: task_executor::TaskExecutor,
     beacon_chain: Arc<BeaconChain<T>>,
-    network_globals: Arc<NetworkGlobals<T::EthSpec>>,
     network_send: mpsc::UnboundedSender<NetworkMessage<T::EthSpec>>,
-    beacon_processor_send: BeaconProcessorSend<T>,
+    beacon_processor: Arc<NetworkBeaconProcessor<T>>,
+    sync_recv: mpsc::UnboundedReceiver<SyncMessage<T::EthSpec>>,
     log: slog::Logger,
-) -> mpsc::UnboundedSender<SyncMessage<T::EthSpec>> {
+) {
     assert!(
         MAX_REQUEST_BLOCKS >= T::EthSpec::slots_per_epoch() * EPOCHS_PER_BATCH,
         "Max blocks that can be requested in a single batch greater than max allowed blocks in a single request"
     );
-    // generate the message channel
-    let (sync_send, sync_recv) = mpsc::unbounded_channel::<SyncMessage<T::EthSpec>>();
     let (delayed_lookups_send, delayed_lookups_recv) =
         mpsc::channel::<DelayedLookupMessage>(DELAY_QUEUE_CHANNEL_SIZE);
 
     // create an instance of the SyncManager
+    let network_globals = beacon_processor.network_globals.clone();
     let mut sync_manager = SyncManager {
         chain: beacon_chain.clone(),
-        network_globals: network_globals.clone(),
         input_channel: sync_recv,
         network: SyncNetworkContext::new(
             network_send,
-            network_globals.clone(),
+            beacon_processor.clone(),
-            beacon_processor_send,
             beacon_chain.clone(),
             log.clone(),
         ),
@@ -273,22 +267,24 @@ pub fn spawn<T: BeaconChainTypes>(
     };
 
     let log_clone = log.clone();
-    let sync_send_clone = sync_send.clone();
     delayed_lookup::spawn_delayed_lookup_service(
         &executor,
         beacon_chain,
         delayed_lookups_recv,
-        sync_send,
+        beacon_processor,
         log,
     );
 
     // spawn the sync manager thread
     debug!(log_clone, "Sync Manager started");
     executor.spawn(async move { Box::pin(sync_manager.main()).await }, "sync");
-    sync_send_clone
 }
 
 impl<T: BeaconChainTypes> SyncManager<T> {
+    fn network_globals(&self) -> &NetworkGlobals<T::EthSpec> {
+        self.network.network_globals()
+    }
+
     /* Input Handling Functions */
 
     /// A peer has connected which has blocks that are unknown to us.
@@ -429,12 +425,12 @@ impl<T: BeaconChainTypes> SyncManager<T> {
         let rpr = new_state.as_str();
         // Drop the write lock
         let update_sync_status = self
-            .network_globals
+            .network_globals()
             .peers
             .write()
             .update_sync_status(peer_id, new_state.clone());
         if let Some(was_updated) = update_sync_status {
-            let is_connected = self.network_globals.peers.read().is_connected(peer_id);
+            let is_connected = self.network_globals().peers.read().is_connected(peer_id);
             if was_updated {
                 debug!(
                     self.log,
@@ -490,7 +486,7 @@ impl<T: BeaconChainTypes> SyncManager<T> {
                         let head = self.chain.best_slot();
                         let current_slot = self.chain.slot().unwrap_or_else(|_| Slot::new(0));
 
-                        let peers = self.network_globals.peers.read();
+                        let peers = self.network_globals().peers.read();
                         if current_slot >= head
                             && current_slot.sub(head) <= (SLOT_IMPORT_TOLERANCE as u64)
                             && head > 0
@@ -552,8 +548,8 @@ impl<T: BeaconChainTypes> SyncManager<T> {
             },
         };
 
-        let old_state = self.network_globals.set_sync_state(new_state);
+        let old_state = self.network_globals().set_sync_state(new_state);
-        let new_state = self.network_globals.sync_state.read();
+        let new_state = self.network_globals().sync_state.read().clone();
         if !new_state.eq(&old_state) {
             info!(self.log, "Sync state updated"; "old_state" => %old_state, "new_state" => %new_state);
             // If we have become synced - Subscribe to all the core subnet topics
@@ -802,7 +798,7 @@ impl<T: BeaconChainTypes> SyncManager<T> {
     }
 
     fn should_search_for_block(&mut self, block_slot: Slot, peer_id: &PeerId) -> bool {
-        if !self.network_globals.sync_state.read().is_synced() {
+        if !self.network_globals().sync_state.read().is_synced() {
             let head_slot = self.chain.canonical_head.cached_head().head_slot();
 
             // if the block is far in the future, ignore it. If its within the slot tolerance of
@@ -816,13 +812,13 @@ impl<T: BeaconChainTypes> SyncManager<T> {
             }
         }
 
-        self.network_globals.peers.read().is_connected(peer_id)
+        self.network_globals().peers.read().is_connected(peer_id)
             && self.network.is_execution_engine_online()
     }
 
     fn synced_and_connected(&mut self, peer_id: &PeerId) -> bool {
-        self.network_globals.sync_state.read().is_synced()
+        self.network_globals().sync_state.read().is_synced()
-            && self.network_globals.peers.read().is_connected(peer_id)
+            && self.network_globals().peers.read().is_connected(peer_id)
             && self.network.is_execution_engine_online()
     }
 

beacon_node/network/src/sync/network_context.rs

@@ -4,7 +4,7 @@
 use super::block_sidecar_coupling::BlocksAndBlobsRequestInfo;
 use super::manager::{Id, RequestId as SyncRequestId};
 use super::range_sync::{BatchId, ByRangeRequestType, ChainId};
-use crate::beacon_processor::BeaconProcessorSend;
+use crate::network_beacon_processor::NetworkBeaconProcessor;
 use crate::service::{NetworkMessage, RequestId};
 use crate::status::ToStatusMessage;
 use crate::sync::block_lookups::{BlobRequestId, BlockRequestId};
@@ -36,9 +36,6 @@ pub struct SyncNetworkContext<T: BeaconChainTypes> {
     /// The network channel to relay messages to the Network service.
     network_send: mpsc::UnboundedSender<NetworkMessage<T::EthSpec>>,
 
-    /// Access to the network global vars.
-    network_globals: Arc<NetworkGlobals<T::EthSpec>>,
-
     /// A sequential ID for all RPC requests.
     request_id: Id,
 
@@ -59,8 +56,8 @@ pub struct SyncNetworkContext<T: BeaconChainTypes> {
     /// `beacon_processor_send`.
     execution_engine_state: EngineState,
 
-    /// Channel to send work to the beacon processor.
+    /// Sends work to the beacon processor via a channel.
-    beacon_processor_send: BeaconProcessorSend<T>,
+    network_beacon_processor: Arc<NetworkBeaconProcessor<T>>,
 
     pub chain: Arc<BeaconChain<T>>,
 
@@ -89,29 +86,31 @@ impl<T: EthSpec> From<Option<Arc<BlobSidecar<T>>>> for BlockOrBlob<T> {
 impl<T: BeaconChainTypes> SyncNetworkContext<T> {
     pub fn new(
         network_send: mpsc::UnboundedSender<NetworkMessage<T::EthSpec>>,
-        network_globals: Arc<NetworkGlobals<T::EthSpec>>,
+        network_beacon_processor: Arc<NetworkBeaconProcessor<T>>,
-        beacon_processor_send: BeaconProcessorSend<T>,
         chain: Arc<BeaconChain<T>>,
         log: slog::Logger,
     ) -> Self {
         SyncNetworkContext {
             network_send,
-            network_globals,
-            request_id: 1,
-            range_requests: Default::default(),
-            backfill_requests: Default::default(),
-            range_blocks_and_blobs_requests: Default::default(),
-            backfill_blocks_and_blobs_requests: Default::default(),
             execution_engine_state: EngineState::Online, // always assume `Online` at the start
-            beacon_processor_send,
+            request_id: 1,
+            range_requests: FnvHashMap::default(),
+            backfill_requests: FnvHashMap::default(),
+            range_blocks_and_blobs_requests: FnvHashMap::default(),
+            backfill_blocks_and_blobs_requests: FnvHashMap::default(),
+            network_beacon_processor,
             chain,
             log,
         }
     }
 
+    pub fn network_globals(&self) -> &NetworkGlobals<T::EthSpec> {
+        &self.network_beacon_processor.network_globals
+    }
+
     /// Returns the Client type of the peer if known
     pub fn client_type(&self, peer_id: &PeerId) -> Client {
-        self.network_globals
+        self.network_globals()
             .peers
             .read()
             .peer_info(peer_id)
@@ -564,13 +563,13 @@ impl<T: BeaconChainTypes> SyncNetworkContext<T> {
         })
     }
 
-    pub fn processor_channel_if_enabled(&self) -> Option<&BeaconProcessorSend<T>> {
+    pub fn beacon_processor_if_enabled(&self) -> Option<&Arc<NetworkBeaconProcessor<T>>> {
         self.is_execution_engine_online()
-            .then_some(&self.beacon_processor_send)
+            .then_some(&self.network_beacon_processor)
     }
 
-    pub fn processor_channel(&self) -> &BeaconProcessorSend<T> {
+    pub fn beacon_processor(&self) -> &Arc<NetworkBeaconProcessor<T>> {
-        &self.beacon_processor_send
+        &self.network_beacon_processor
     }
 
     fn next_id(&mut self) -> Id {

beacon_node/network/src/sync/range_sync/chain.rs

@@ -1,5 +1,5 @@
 use super::batch::{BatchInfo, BatchProcessingResult, BatchState};
-use crate::beacon_processor::{ChainSegmentProcessId, WorkEvent as BeaconWorkEvent};
+use crate::network_beacon_processor::ChainSegmentProcessId;
 use crate::sync::{
     manager::Id, network_context::SyncNetworkContext, BatchOperationOutcome, BatchProcessResult,
 };
@@ -294,8 +294,8 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
             return Ok(KeepChain);
         }
 
-        let beacon_processor_send = match network.processor_channel_if_enabled() {
+        let beacon_processor = match network.beacon_processor_if_enabled() {
-            Some(channel) => channel,
+            Some(beacon_processor) => beacon_processor,
             None => return Ok(KeepChain),
         };
 
@@ -317,9 +317,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
         let process_id = ChainSegmentProcessId::RangeBatchId(self.id, batch_id);
         self.current_processing_batch = Some(batch_id);
 
-        let work_event = BeaconWorkEvent::chain_segment(process_id, blocks);
+        if let Err(e) = beacon_processor.send_chain_segment(process_id, blocks) {
-
-        if let Err(e) = beacon_processor_send.try_send(work_event) {
             crit!(self.log, "Failed to send chain segment to processor."; "msg" => "process_batch",
                 "error" => %e, "batch" => self.processing_target);
             // This is unlikely to happen but it would stall syncing since the batch now has no

beacon_node/network/src/sync/range_sync/range.rs

@@ -379,27 +379,27 @@ where
 
 #[cfg(test)]
 mod tests {
-    use super::*;
+    use crate::network_beacon_processor::NetworkBeaconProcessor;
-
-    use crate::beacon_processor::{BeaconProcessorSend, WorkEvent as BeaconWorkEvent};
     use crate::service::RequestId;
     use crate::NetworkMessage;
-    use beacon_chain::{
+
-        builder::Witness,
+    use super::*;
-        eth1_chain::CachingEth1Backend,
+    use beacon_chain::builder::Witness;
-        parking_lot::RwLock,
+    use beacon_chain::eth1_chain::CachingEth1Backend;
-        test_utils::{build_log, BeaconChainHarness, EphemeralHarnessType},
+    use beacon_chain::parking_lot::RwLock;
-        EngineState,
+    use beacon_chain::EngineState;
-    };
+    use beacon_processor::WorkEvent as BeaconWorkEvent;
-    use lighthouse_network::{
+    use lighthouse_network::rpc::BlocksByRangeRequest;
-        rpc::{BlocksByRangeRequest, StatusMessage},
+    use lighthouse_network::Request;
-        NetworkGlobals, Request,
+    use lighthouse_network::{rpc::StatusMessage, NetworkGlobals};
-    };
+    use slog::{o, Drain};
-    use slog::o;
-    use slot_clock::TestingSlotClock;
-    use std::{collections::HashSet, sync::Arc};
-    use store::MemoryStore;
     use tokio::sync::mpsc;
+
+    use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType};
+    use slot_clock::TestingSlotClock;
+    use std::collections::HashSet;
+    use std::sync::Arc;
+    use store::MemoryStore;
     use types::{Hash256, MinimalEthSpec as E};
 
     #[derive(Debug)]
@@ -449,11 +449,23 @@ mod tests {
     type TestBeaconChainType =
         Witness<TestingSlotClock, CachingEth1Backend<E>, E, MemoryStore<E>, MemoryStore<E>>;
 
+    fn build_log(level: slog::Level, enabled: bool) -> slog::Logger {
+        let decorator = slog_term::TermDecorator::new().build();
+        let drain = slog_term::FullFormat::new(decorator).build().fuse();
+        let drain = slog_async::Async::new(drain).build().fuse();
+
+        if enabled {
+            slog::Logger::root(drain.filter_level(level).fuse(), o!())
+        } else {
+            slog::Logger::root(drain.filter(|_| false).fuse(), o!())
+        }
+    }
+
     #[allow(unused)]
     struct TestRig {
         log: slog::Logger,
         /// To check what does sync send to the beacon processor.
-        beacon_processor_rx: mpsc::Receiver<BeaconWorkEvent<TestBeaconChainType>>,
+        beacon_processor_rx: mpsc::Receiver<BeaconWorkEvent<E>>,
         /// To set up different scenarios where sync is told about known/unkown blocks.
         chain: Arc<FakeStorage>,
         /// Needed by range to handle communication with the network.
@@ -581,7 +593,7 @@ mod tests {
         fn expect_chain_segment(&mut self) {
             match self.beacon_processor_rx.try_recv() {
                 Ok(work) => {
-                    assert_eq!(work.work_type(), crate::beacon_processor::CHAIN_SEGMENT);
+                    assert_eq!(work.work_type(), beacon_processor::CHAIN_SEGMENT);
                 }
                 other => panic!("Expected chain segment process, found {:?}", other),
             }
@@ -591,7 +603,7 @@ mod tests {
     fn range(log_enabled: bool) -> (TestRig, RangeSync<TestBeaconChainType, FakeStorage>) {
         let log = build_log(slog::Level::Trace, log_enabled);
         // Initialise a new beacon chain
-        let harness = BeaconChainHarness::<EphemeralHarnessType<E>>::builder(E::default())
+        let harness = BeaconChainHarness::<EphemeralHarnessType<E>>::builder(E)
             .default_spec()
             .logger(log.clone())
             .deterministic_keypairs(1)
@@ -600,17 +612,17 @@ mod tests {
         let chain = harness.chain;
 
         let fake_store = Arc::new(FakeStorage::default());
-        let (beacon_processor_tx, beacon_processor_rx) = mpsc::channel(10);
         let range_sync = RangeSync::<TestBeaconChainType, FakeStorage>::new(
             fake_store.clone(),
             log.new(o!("component" => "range")),
         );
         let (network_tx, network_rx) = mpsc::unbounded_channel();
         let globals = Arc::new(NetworkGlobals::new_test_globals(Vec::new(), &log));
+        let (network_beacon_processor, beacon_processor_rx) =
+            NetworkBeaconProcessor::null_for_testing(globals.clone());
         let cx = SyncNetworkContext::new(
             network_tx,
-            globals.clone(),
+            Arc::new(network_beacon_processor),
-            BeaconProcessorSend(beacon_processor_tx),
             chain,
             log.new(o!("component" => "network_context")),
         );

beacon_node/src/cli.rs

@@ -540,6 +540,16 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                        [default: 8192 (mainnet) or 64 (minimal)]")
                 .takes_value(true)
         )
+        .arg(
+            Arg::with_name("epochs-per-migration")
+                .long("epochs-per-migration")
+                .value_name("N")
+                .help("The number of epochs to wait between running the migration of data from the \
+                       hot DB to the cold DB. Less frequent runs can be useful for minimizing disk \
+                       writes")
+                .default_value("1")
+                .takes_value(true)
+        )
         .arg(
             Arg::with_name("block-cache-size")
                 .long("block-cache-size")
@@ -890,7 +900,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .help("Set the timeout for checkpoint sync calls to remote beacon node HTTP endpoint.")
                 .value_name("SECONDS")
                 .takes_value(true)
-                .default_value("60")
+                .default_value("180")
         )
         .arg(
             Arg::with_name("reconstruct-historic-states")

beacon_node/src/config.rs

@@ -421,6 +421,12 @@ pub fn get_config<E: EthSpec>(
         client_config.store.prune_payloads = prune_payloads;
     }
 
+    if let Some(epochs_per_migration) =
+        clap_utils::parse_optional(cli_args, "epochs-per-migration")?
+    {
+        client_config.chain.epochs_per_migration = epochs_per_migration;
+    }
+
     if let Some(prune_blobs) = clap_utils::parse_optional(cli_args, "prune-blobs")? {
         client_config.store.prune_blobs = prune_blobs;
     }

book/src/checkpoint-sync.md

@@ -48,6 +48,17 @@ The Ethereum community provides various [public endpoints](https://eth-clients.g
 lighthouse bn --checkpoint-sync-url https://example.com/ ...
 ```
 
+### Adjusting the timeout
+
+If the beacon node fails to start due to a timeout from the checkpoint sync server, you can try
+running it again with a longer timeout by adding the flag `--checkpoint-sync-url-timeout`.
+
+```
+lighthouse bn --checkpoint-sync-url-timeout 300 --checkpoint-sync-url https://example.com/ ...
+```
+
+The flag takes a value in seconds. For more information see `lighthouse bn --help`.
+
 ## Backfilling Blocks
 
 Once forwards sync completes, Lighthouse will commence a "backfill sync" to download the blocks

consensus/state_processing/src/lib.rs

@@ -2,7 +2,7 @@
 #![cfg_attr(
     not(test),
     deny(
-        clippy::integer_arithmetic,
+        clippy::arithmetic_side_effects,
         clippy::disallowed_methods,
         clippy::indexing_slicing,
         clippy::unwrap_used,

consensus/state_processing/src/per_block_processing/block_signature_verifier.rs

@@ -1,4 +1,4 @@
-#![allow(clippy::integer_arithmetic)]
+#![allow(clippy::arithmetic_side_effects)]
 
 use super::signature_sets::{Error as SignatureSetError, *};
 use crate::per_block_processing::errors::{AttestationInvalid, BlockOperationError};

consensus/state_processing/src/per_block_processing/tests.rs

@@ -960,7 +960,7 @@ async fn fork_spanning_exit() {
     spec.bellatrix_fork_epoch = Some(Epoch::new(4));
     spec.shard_committee_period = 0;
 
-    let harness = BeaconChainHarness::builder(MainnetEthSpec::default())
+    let harness = BeaconChainHarness::builder(MainnetEthSpec)
         .spec(spec.clone())
         .deterministic_keypairs(VALIDATOR_COUNT)
         .mock_execution_layer()

consensus/state_processing/src/per_epoch_processing/altair/inactivity_updates.rs

@@ -34,7 +34,7 @@ pub fn process_inactivity_updates<T: EthSpec>(
                 .safe_add_assign(spec.inactivity_score_bias)?;
         }
         // Decrease the score of all validators for forgiveness when not during a leak
-        if !state.is_in_inactivity_leak(previous_epoch, spec) {
+        if !state.is_in_inactivity_leak(previous_epoch, spec)? {
             let inactivity_score = state.get_inactivity_score_mut(index)?;
             inactivity_score
                 .safe_sub_assign(min(spec.inactivity_score_recovery_rate, *inactivity_score))?;

consensus/state_processing/src/per_epoch_processing/altair/rewards_and_penalties.rs

@@ -77,7 +77,7 @@ pub fn get_flag_index_deltas<T: EthSpec>(
         let mut delta = Delta::default();
 
         if unslashed_participating_indices.contains(index)? {
-            if !state.is_in_inactivity_leak(previous_epoch, spec) {
+            if !state.is_in_inactivity_leak(previous_epoch, spec)? {
                 let reward_numerator = base_reward
                     .safe_mul(weight)?
                     .safe_mul(unslashed_participating_increments)?;

consensus/state_processing/src/verify_operation.rs

@@ -138,7 +138,7 @@ impl<E: EthSpec> VerifyOperation<E> for SignedVoluntaryExit {
         Ok(SigVerifiedOp::new(self, state))
     }
 
-    #[allow(clippy::integer_arithmetic)]
+    #[allow(clippy::arithmetic_side_effects)]
     fn verification_epochs(&self) -> SmallVec<[Epoch; MAX_FORKS_VERIFIED_AGAINST]> {
         smallvec![self.message.epoch]
     }
@@ -156,7 +156,7 @@ impl<E: EthSpec> VerifyOperation<E> for AttesterSlashing<E> {
         Ok(SigVerifiedOp::new(self, state))
     }
 
-    #[allow(clippy::integer_arithmetic)]
+    #[allow(clippy::arithmetic_side_effects)]
     fn verification_epochs(&self) -> SmallVec<[Epoch; MAX_FORKS_VERIFIED_AGAINST]> {
         smallvec![
             self.attestation_1.data.target.epoch,
@@ -177,7 +177,7 @@ impl<E: EthSpec> VerifyOperation<E> for ProposerSlashing {
         Ok(SigVerifiedOp::new(self, state))
     }
 
-    #[allow(clippy::integer_arithmetic)]
+    #[allow(clippy::arithmetic_side_effects)]
     fn verification_epochs(&self) -> SmallVec<[Epoch; MAX_FORKS_VERIFIED_AGAINST]> {
         // Only need a single epoch because the slots of the two headers must be equal.
         smallvec![self
@@ -200,7 +200,7 @@ impl<E: EthSpec> VerifyOperation<E> for SignedBlsToExecutionChange {
         Ok(SigVerifiedOp::new(self, state))
     }
 
-    #[allow(clippy::integer_arithmetic)]
+    #[allow(clippy::arithmetic_side_effects)]
     fn verification_epochs(&self) -> SmallVec<[Epoch; MAX_FORKS_VERIFIED_AGAINST]> {
         smallvec![]
     }

consensus/types/src/beacon_state.rs

@@ -455,7 +455,7 @@ impl<T: EthSpec> BeaconState<T> {
     }
 
     /// Specialised deserialisation method that uses the `ChainSpec` as context.
-    #[allow(clippy::integer_arithmetic)]
+    #[allow(clippy::arithmetic_side_effects)]
     pub fn from_ssz_bytes(bytes: &[u8], spec: &ChainSpec) -> Result<Self, ssz::DecodeError> {
         // Slot is after genesis_time (u64) and genesis_validators_root (Hash256).
         let slot_start = <u64 as Decode>::ssz_fixed_len() + <Hash256 as Decode>::ssz_fixed_len();
@@ -1761,16 +1761,22 @@ impl<T: EthSpec> BeaconState<T> {
         previous_epoch: Epoch,
         val_index: usize,
     ) -> Result<bool, Error> {
-        self.get_validator(val_index).map(|val| {
+        let val = self.get_validator(val_index)?;
-            val.is_active_at(previous_epoch)
+        Ok(val.is_active_at(previous_epoch)
-                || (val.slashed && previous_epoch + Epoch::new(1) < val.withdrawable_epoch)
+            || (val.slashed && previous_epoch.safe_add(Epoch::new(1))? < val.withdrawable_epoch))
-        })
     }
 
     /// Passing `previous_epoch` to this function rather than computing it internally provides
     /// a tangible speed improvement in state processing.
-    pub fn is_in_inactivity_leak(&self, previous_epoch: Epoch, spec: &ChainSpec) -> bool {
+    pub fn is_in_inactivity_leak(
-        (previous_epoch - self.finalized_checkpoint().epoch) > spec.min_epochs_to_inactivity_penalty
+        &self,
+        previous_epoch: Epoch,
+        spec: &ChainSpec,
+    ) -> Result<bool, safe_arith::ArithError> {
+        Ok(
+            (previous_epoch.safe_sub(self.finalized_checkpoint().epoch)?)
+                > spec.min_epochs_to_inactivity_penalty,
+        )
     }
 
     /// Get the `SyncCommittee` associated with the next slot. Useful because sync committees

consensus/types/src/beacon_state/committee_cache.rs

@@ -1,4 +1,4 @@
-#![allow(clippy::integer_arithmetic)]
+#![allow(clippy::arithmetic_side_effects)]
 
 use super::BeaconState;
 use crate::*;

consensus/types/src/beacon_state/tree_hash_cache.rs

@@ -1,4 +1,4 @@
-#![allow(clippy::integer_arithmetic)]
+#![allow(clippy::arithmetic_side_effects)]
 #![allow(clippy::disallowed_methods)]
 #![allow(clippy::indexing_slicing)]
 

consensus/types/src/blob_sidecar.rs

@@ -126,7 +126,7 @@ impl<T: EthSpec> BlobSidecar<T> {
         })
     }
 
-    #[allow(clippy::integer_arithmetic)]
+    #[allow(clippy::arithmetic_side_effects)]
     pub fn max_size() -> usize {
         // Fixed part
         Self::empty().as_ssz_bytes().len()

consensus/types/src/chain_spec.rs

@@ -468,7 +468,7 @@ impl ChainSpec {
         Hash256::from(domain)
     }
 
-    #[allow(clippy::integer_arithmetic)]
+    #[allow(clippy::arithmetic_side_effects)]
     pub const fn attestation_subnet_prefix_bits(&self) -> u32 {
         let attestation_subnet_count_bits = self.attestation_subnet_count.ilog2();
         self.attestation_subnet_extra_bits as u32 + attestation_subnet_count_bits

consensus/types/src/execution_payload.rs

@@ -115,7 +115,7 @@ impl<T: EthSpec> ExecutionPayload<T> {
         }
     }
 
-    #[allow(clippy::integer_arithmetic)]
+    #[allow(clippy::arithmetic_side_effects)]
     /// Returns the maximum size of an execution payload.
     pub fn max_execution_payload_merge_size() -> usize {
         // Fixed part
@@ -126,7 +126,7 @@ impl<T: EthSpec> ExecutionPayload<T> {
             + (T::max_transactions_per_payload() * (ssz::BYTES_PER_LENGTH_OFFSET + T::max_bytes_per_transaction()))
     }
 
-    #[allow(clippy::integer_arithmetic)]
+    #[allow(clippy::arithmetic_side_effects)]
     /// Returns the maximum size of an execution payload.
     pub fn max_execution_payload_capella_size() -> usize {
         // Fixed part
@@ -139,7 +139,7 @@ impl<T: EthSpec> ExecutionPayload<T> {
             + (T::max_withdrawals_per_payload() * <Withdrawal as Encode>::ssz_fixed_len())
     }
 
-    #[allow(clippy::integer_arithmetic)]
+    #[allow(clippy::arithmetic_side_effects)]
     /// Returns the maximum size of an execution payload.
     pub fn max_execution_payload_deneb_size() -> usize {
         // Fixed part

consensus/types/src/lib.rs

@@ -3,7 +3,7 @@
 #![cfg_attr(
     not(test),
     deny(
-        clippy::integer_arithmetic,
+        clippy::arithmetic_side_effects,
         clippy::disallowed_methods,
         clippy::indexing_slicing
     )

consensus/types/src/participation_list.rs

@@ -1,4 +1,4 @@
-#![allow(clippy::integer_arithmetic)]
+#![allow(clippy::arithmetic_side_effects)]
 
 use crate::{Hash256, ParticipationFlags, Unsigned, VariableList};
 use cached_tree_hash::{int_log, CacheArena, CachedTreeHash, Error, TreeHashCache};

consensus/types/src/subnet_id.rs

@@ -72,7 +72,7 @@ impl SubnetId {
             .into())
     }
 
-    #[allow(clippy::integer_arithmetic)]
+    #[allow(clippy::arithmetic_side_effects)]
     /// Computes the set of subnets the node should be subscribed to during the current epoch,
     /// along with the first epoch in which these subscriptions are no longer valid.
     pub fn compute_subnets_for_epoch<T: EthSpec>(

consensus/types/src/test_utils/mod.rs

@@ -1,4 +1,4 @@
-#![allow(clippy::integer_arithmetic)]
+#![allow(clippy::arithmetic_side_effects)]
 
 use std::fmt::Debug;
 

consensus/types/src/test_utils/test_random.rs

@@ -30,7 +30,7 @@ pub trait TestRandom {
 
 impl<T> TestRandom for PhantomData<T> {
     fn random_for_test(_rng: &mut impl RngCore) -> Self {
-        PhantomData::default()
+        PhantomData
     }
 }
 

lcli/src/block_root.rs

@@ -31,14 +31,19 @@ use clap::ArgMatches;
 use clap_utils::{parse_optional, parse_required};
 use environment::Environment;
 use eth2::{types::BlockId, BeaconNodeHttpClient, SensitiveUrl, Timeouts};
+use eth2_network_config::Eth2NetworkConfig;
 use std::path::PathBuf;
 use std::time::{Duration, Instant};
 use types::{EthSpec, FullPayload, SignedBeaconBlock};
 
 const HTTP_TIMEOUT: Duration = Duration::from_secs(5);
 
-pub fn run<T: EthSpec>(env: Environment<T>, matches: &ArgMatches) -> Result<(), String> {
+pub fn run<T: EthSpec>(
-    let spec = &T::default_spec();
+    env: Environment<T>,
+    network_config: Eth2NetworkConfig,
+    matches: &ArgMatches,
+) -> Result<(), String> {
+    let spec = &network_config.chain_spec::<T>()?;
     let executor = env.core_context().executor;
 
     /*

lcli/src/generate_bootnode_enr.rs

@@ -1,6 +1,7 @@
 use clap::ArgMatches;
 use lighthouse_network::{
-    discovery::{build_enr, CombinedKey, CombinedKeyExt, Keypair, ENR_FILENAME},
+    discovery::{build_enr, CombinedKey, CombinedKeyExt, ENR_FILENAME},
+    libp2p::identity::secp256k1,
     NetworkConfig, NETWORK_KEY_FILENAME,
 };
 use std::fs::File;
@@ -29,8 +30,8 @@ pub fn run<T: EthSpec>(matches: &ArgMatches) -> Result<(), String> {
     config.enr_udp4_port = Some(udp_port);
     config.enr_tcp6_port = Some(tcp_port);
 
-    let local_keypair = Keypair::generate_secp256k1();
+    let secp256k1_keypair = secp256k1::Keypair::generate();
-    let enr_key = CombinedKey::from_libp2p(&local_keypair)?;
+    let enr_key = CombinedKey::from_secp256k1(&secp256k1_keypair);
     let enr_fork_id = EnrForkId {
         fork_digest: ChainSpec::compute_fork_digest(genesis_fork_version, Hash256::zero()),
         next_fork_version: genesis_fork_version,
@@ -47,13 +48,10 @@ pub fn run<T: EthSpec>(matches: &ArgMatches) -> Result<(), String> {
         .write_all(enr.to_base64().as_bytes())
         .map_err(|e| format!("Unable to write ENR to {}: {:?}", ENR_FILENAME, e))?;
 
-    let secret_bytes = match local_keypair {
-        Keypair::Secp256k1(key) => key.secret().to_bytes(),
-        _ => return Err("Key is not a secp256k1 key".into()),
-    };
-
     let mut key_file = File::create(output_dir.join(NETWORK_KEY_FILENAME))
         .map_err(|e| format!("Unable to create {}: {:?}", NETWORK_KEY_FILENAME, e))?;
+
+    let secret_bytes = secp256k1_keypair.secret().to_bytes();
     key_file
         .write_all(&secret_bytes)
         .map_err(|e| format!("Unable to write key to {}: {:?}", NETWORK_KEY_FILENAME, e))?;

lcli/src/main.rs

@@ -15,11 +15,13 @@ mod new_testnet;
 mod parse_ssz;
 mod replace_state_pubkeys;
 mod skip_slots;
+mod state_root;
 mod transition_blocks;
 
 use clap::{App, Arg, ArgMatches, SubCommand};
-use clap_utils::parse_path_with_default_in_home_dir;
+use clap_utils::parse_optional;
 use environment::{EnvironmentBuilder, LoggerConfig};
+use eth2_network_config::Eth2NetworkConfig;
 use parse_ssz::run_parse_ssz;
 use std::path::PathBuf;
 use std::process;
@@ -38,7 +40,6 @@ fn main() {
                 .long("spec")
                 .value_name("STRING")
                 .takes_value(true)
-                .required(true)
                 .possible_values(&["minimal", "mainnet", "gnosis"])
                 .default_value("mainnet")
                 .global(true),
@@ -50,7 +51,16 @@ fn main() {
                 .value_name("PATH")
                 .takes_value(true)
                 .global(true)
-                .help("The testnet dir. Defaults to ~/.lighthouse/testnet"),
+                .help("The testnet dir."),
+        )
+        .arg(
+            Arg::with_name("network")
+                .long("network")
+                .value_name("NAME")
+                .takes_value(true)
+                .global(true)
+                .help("The network to use. Defaults to mainnet.")
+                .conflicts_with("testnet-dir")
         )
         .subcommand(
             SubCommand::with_name("skip-slots")
@@ -126,7 +136,7 @@ fn main() {
                         .takes_value(true)
                         .conflicts_with("beacon-url")
                         .requires("block-path")
-                        .help("Path to load a BeaconState from file as SSZ."),
+                        .help("Path to load a BeaconState from as SSZ."),
                 )
                 .arg(
                     Arg::with_name("block-path")
@@ -135,7 +145,7 @@ fn main() {
                         .takes_value(true)
                         .conflicts_with("beacon-url")
                         .requires("pre-state-path")
-                        .help("Path to load a SignedBeaconBlock from file as SSZ."),
+                        .help("Path to load a SignedBeaconBlock from as SSZ."),
                 )
                 .arg(
                     Arg::with_name("post-state-output-path")
@@ -361,7 +371,6 @@ fn main() {
                         .index(2)
                         .value_name("BIP39_MNENMONIC")
                         .takes_value(true)
-                        .required(true)
                         .default_value(
                             "replace nephew blur decorate waste convince soup column \
                             orient excite play baby",
@@ -382,7 +391,6 @@ fn main() {
                         .help("The block hash used when generating an execution payload. This \
                             value is used for `execution_payload_header.block_hash` as well as \
                             `execution_payload_header.random`")
-                        .required(true)
                         .default_value(
                             "0x0000000000000000000000000000000000000000000000000000000000000000",
                         ),
@@ -400,7 +408,6 @@ fn main() {
                         .value_name("INTEGER")
                         .takes_value(true)
                         .help("The base fee per gas field in the execution payload generated.")
-                        .required(true)
                         .default_value("1000000000"),
                 )
                 .arg(
@@ -409,7 +416,6 @@ fn main() {
                         .value_name("INTEGER")
                         .takes_value(true)
                         .help("The gas limit field in the execution payload generated.")
-                        .required(true)
                         .default_value("30000000"),
                 )
                 .arg(
@@ -817,14 +823,14 @@ fn main() {
         )
         .subcommand(
             SubCommand::with_name("block-root")
-                .about("Computes the block root of some block")
+                .about("Computes the block root of some block.")
                 .arg(
                     Arg::with_name("block-path")
                         .long("block-path")
                         .value_name("PATH")
                         .takes_value(true)
                         .conflicts_with("beacon-url")
-                        .help("Path to load a SignedBeaconBlock from file as SSZ."),
+                        .help("Path to load a SignedBeaconBlock from as SSZ."),
                 )
                 .arg(
                     Arg::with_name("beacon-url")
@@ -850,6 +856,41 @@ fn main() {
                         .help("Number of repeat runs, useful for benchmarking."),
                 )
         )
+        .subcommand(
+            SubCommand::with_name("state-root")
+                .about("Computes the state root of some state.")
+                .arg(
+                    Arg::with_name("state-path")
+                        .long("state-path")
+                        .value_name("PATH")
+                        .takes_value(true)
+                        .conflicts_with("beacon-url")
+                        .help("Path to load a BeaconState from as SSZ."),
+                )
+                .arg(
+                    Arg::with_name("beacon-url")
+                        .long("beacon-url")
+                        .value_name("URL")
+                        .takes_value(true)
+                        .help("URL to a beacon-API provider."),
+                )
+                .arg(
+                    Arg::with_name("state-id")
+                        .long("state-id")
+                        .value_name("BLOCK_ID")
+                        .takes_value(true)
+                        .requires("beacon-url")
+                        .help("Identifier for a state as per beacon-API standards (slot, root, etc.)"),
+                )
+                .arg(
+                    Arg::with_name("runs")
+                        .long("runs")
+                        .value_name("INTEGER")
+                        .takes_value(true)
+                        .default_value("1")
+                        .help("Number of repeat runs, useful for benchmarking."),
+                )
+        )
         .get_matches();
 
     let result = matches
@@ -896,17 +937,44 @@ fn run<T: EthSpec>(
         .build()
         .map_err(|e| format!("should build env: {:?}", e))?;
 
-    let testnet_dir = parse_path_with_default_in_home_dir(
+    // Determine testnet-dir path or network name depending on CLI flags.
-        matches,
+    let (testnet_dir, network_name) =
-        "testnet-dir",
+        if let Some(testnet_dir) = parse_optional::<PathBuf>(matches, "testnet-dir")? {
-        PathBuf::from(directory::DEFAULT_ROOT_DIR).join("testnet"),
+            (Some(testnet_dir), None)
-    )?;
+        } else {
+            let network_name =
+                parse_optional(matches, "network")?.unwrap_or_else(|| "mainnet".to_string());
+            (None, Some(network_name))
+        };
+
+    // Lazily load either the testnet dir or the network config, as required.
+    // Some subcommands like new-testnet need the testnet dir but not the network config.
+    let get_testnet_dir = || testnet_dir.clone().ok_or("testnet-dir is required");
+    let get_network_config = || {
+        if let Some(testnet_dir) = &testnet_dir {
+            Eth2NetworkConfig::load(testnet_dir.clone()).map_err(|e| {
+                format!(
+                    "Unable to open testnet dir at {}: {}",
+                    testnet_dir.display(),
+                    e
+                )
+            })
+        } else {
+            let network_name = network_name.ok_or("no network name or testnet-dir provided")?;
+            Eth2NetworkConfig::constant(&network_name)?.ok_or("invalid network name".into())
+        }
+    };
 
     match matches.subcommand() {
-        ("transition-blocks", Some(matches)) => transition_blocks::run::<T>(env, matches)
+        ("transition-blocks", Some(matches)) => {
-            .map_err(|e| format!("Failed to transition blocks: {}", e)),
+            let network_config = get_network_config()?;
+            transition_blocks::run::<T>(env, network_config, matches)
+                .map_err(|e| format!("Failed to transition blocks: {}", e))
+        }
         ("skip-slots", Some(matches)) => {
-            skip_slots::run::<T>(env, matches).map_err(|e| format!("Failed to skip slots: {}", e))
+            let network_config = get_network_config()?;
+            skip_slots::run::<T>(env, network_config, matches)
+                .map_err(|e| format!("Failed to skip slots: {}", e))
         }
         ("pretty-ssz", Some(matches)) => {
             run_parse_ssz::<T>(matches).map_err(|e| format!("Failed to pretty print hex: {}", e))
@@ -915,22 +983,33 @@ fn run<T: EthSpec>(
             deploy_deposit_contract::run::<T>(env, matches)
                 .map_err(|e| format!("Failed to run deploy-deposit-contract command: {}", e))
         }
-        ("eth1-genesis", Some(matches)) => eth1_genesis::run::<T>(env, testnet_dir, matches)
+        ("eth1-genesis", Some(matches)) => {
-            .map_err(|e| format!("Failed to run eth1-genesis command: {}", e)),
+            let testnet_dir = get_testnet_dir()?;
-        ("interop-genesis", Some(matches)) => interop_genesis::run::<T>(testnet_dir, matches)
+            eth1_genesis::run::<T>(env, testnet_dir, matches)
-            .map_err(|e| format!("Failed to run interop-genesis command: {}", e)),
+                .map_err(|e| format!("Failed to run eth1-genesis command: {}", e))
+        }
+        ("interop-genesis", Some(matches)) => {
+            let testnet_dir = get_testnet_dir()?;
+            interop_genesis::run::<T>(testnet_dir, matches)
+                .map_err(|e| format!("Failed to run interop-genesis command: {}", e))
+        }
         ("change-genesis-time", Some(matches)) => {
+            let testnet_dir = get_testnet_dir()?;
             change_genesis_time::run::<T>(testnet_dir, matches)
                 .map_err(|e| format!("Failed to run change-genesis-time command: {}", e))
         }
         ("create-payload-header", Some(matches)) => create_payload_header::run::<T>(matches)
             .map_err(|e| format!("Failed to run create-payload-header command: {}", e)),
         ("replace-state-pubkeys", Some(matches)) => {
+            let testnet_dir = get_testnet_dir()?;
             replace_state_pubkeys::run::<T>(testnet_dir, matches)
                 .map_err(|e| format!("Failed to run replace-state-pubkeys command: {}", e))
         }
-        ("new-testnet", Some(matches)) => new_testnet::run::<T>(testnet_dir, matches)
+        ("new-testnet", Some(matches)) => {
-            .map_err(|e| format!("Failed to run new_testnet command: {}", e)),
+            let testnet_dir = get_testnet_dir()?;
+            new_testnet::run::<T>(testnet_dir, matches)
+                .map_err(|e| format!("Failed to run new_testnet command: {}", e))
+        }
         ("check-deposit-data", Some(matches)) => check_deposit_data::run(matches)
             .map_err(|e| format!("Failed to run check-deposit-data command: {}", e)),
         ("generate-bootnode-enr", Some(matches)) => generate_bootnode_enr::run::<T>(matches)
@@ -941,8 +1020,16 @@ fn run<T: EthSpec>(
             .map_err(|e| format!("Failed to run mnemonic-validators command: {}", e)),
         ("indexed-attestations", Some(matches)) => indexed_attestations::run::<T>(matches)
             .map_err(|e| format!("Failed to run indexed-attestations command: {}", e)),
-        ("block-root", Some(matches)) => block_root::run::<T>(env, matches)
+        ("block-root", Some(matches)) => {
-            .map_err(|e| format!("Failed to run block-root command: {}", e)),
+            let network_config = get_network_config()?;
+            block_root::run::<T>(env, network_config, matches)
+                .map_err(|e| format!("Failed to run block-root command: {}", e))
+        }
+        ("state-root", Some(matches)) => {
+            let network_config = get_network_config()?;
+            state_root::run::<T>(env, network_config, matches)
+                .map_err(|e| format!("Failed to run state-root command: {}", e))
+        }
         (other, _) => Err(format!("Unknown subcommand {}. See --help.", other)),
     }
 }

lcli/src/skip_slots.rs

@@ -49,6 +49,7 @@ use clap::ArgMatches;
 use clap_utils::{parse_optional, parse_required};
 use environment::Environment;
 use eth2::{types::StateId, BeaconNodeHttpClient, SensitiveUrl, Timeouts};
+use eth2_network_config::Eth2NetworkConfig;
 use ssz::Encode;
 use state_processing::state_advance::{complete_state_advance, partial_state_advance};
 use std::fs::File;
@@ -59,8 +60,12 @@ use types::{BeaconState, CloneConfig, EthSpec, Hash256};
 
 const HTTP_TIMEOUT: Duration = Duration::from_secs(10);
 
-pub fn run<T: EthSpec>(env: Environment<T>, matches: &ArgMatches) -> Result<(), String> {
+pub fn run<T: EthSpec>(
-    let spec = &T::default_spec();
+    env: Environment<T>,
+    network_config: Eth2NetworkConfig,
+    matches: &ArgMatches,
+) -> Result<(), String> {
+    let spec = &network_config.chain_spec::<T>()?;
     let executor = env.core_context().executor;
 
     let output_path: Option<PathBuf> = parse_optional(matches, "output-path")?;

lcli/src/state_root.rs

@@ -0,0 +1,76 @@
+use crate::transition_blocks::load_from_ssz_with;
+use clap::ArgMatches;
+use clap_utils::{parse_optional, parse_required};
+use environment::Environment;
+use eth2::{types::StateId, BeaconNodeHttpClient, SensitiveUrl, Timeouts};
+use eth2_network_config::Eth2NetworkConfig;
+use std::path::PathBuf;
+use std::time::{Duration, Instant};
+use types::{BeaconState, EthSpec};
+
+const HTTP_TIMEOUT: Duration = Duration::from_secs(10);
+
+pub fn run<T: EthSpec>(
+    env: Environment<T>,
+    network_config: Eth2NetworkConfig,
+    matches: &ArgMatches,
+) -> Result<(), String> {
+    let executor = env.core_context().executor;
+
+    let spec = &network_config.chain_spec::<T>()?;
+
+    let state_path: Option<PathBuf> = parse_optional(matches, "state-path")?;
+    let beacon_url: Option<SensitiveUrl> = parse_optional(matches, "beacon-url")?;
+    let runs: usize = parse_required(matches, "runs")?;
+
+    info!(
+        "Using {} network ({} spec)",
+        spec.config_name.as_deref().unwrap_or("unknown"),
+        T::spec_name()
+    );
+    info!("Doing {} runs", runs);
+
+    let state = match (state_path, beacon_url) {
+        (Some(state_path), None) => {
+            info!("State path: {:?}", state_path);
+            load_from_ssz_with(&state_path, spec, BeaconState::from_ssz_bytes)?
+        }
+        (None, Some(beacon_url)) => {
+            let state_id: StateId = parse_required(matches, "state-id")?;
+            let client = BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(HTTP_TIMEOUT));
+            executor
+                .handle()
+                .ok_or("shutdown in progress")?
+                .block_on(async move {
+                    client
+                        .get_debug_beacon_states::<T>(state_id)
+                        .await
+                        .map_err(|e| format!("Failed to download state: {:?}", e))
+                })
+                .map_err(|e| format!("Failed to complete task: {:?}", e))?
+                .ok_or_else(|| format!("Unable to locate state at {:?}", state_id))?
+                .data
+        }
+        _ => return Err("must supply either --state-path or --beacon-url".into()),
+    };
+
+    /*
+     * Perform the core "runs".
+     */
+    let mut state_root = None;
+    for i in 0..runs {
+        let mut state = state.clone();
+        let timer = Instant::now();
+        state_root = Some(
+            state
+                .update_tree_hash_cache()
+                .map_err(|e| format!("error computing state root: {e:?}"))?,
+        );
+        info!("Run {}: {:?}", i, timer.elapsed());
+    }
+
+    if let Some(state_root) = state_root {
+        info!("State root is {:?}", state_root);
+    }
+    Ok(())
+}

lcli/src/transition_blocks.rs

@@ -71,6 +71,7 @@ use eth2::{
     types::{BlockId, StateId},
     BeaconNodeHttpClient, SensitiveUrl, Timeouts,
 };
+use eth2_network_config::Eth2NetworkConfig;
 use ssz::Encode;
 use state_processing::{
     block_signature_verifier::BlockSignatureVerifier, per_block_processing, per_slot_processing,
@@ -94,8 +95,12 @@ struct Config {
     exclude_post_block_thc: bool,
 }
 
-pub fn run<T: EthSpec>(env: Environment<T>, matches: &ArgMatches) -> Result<(), String> {
+pub fn run<T: EthSpec>(
-    let spec = &T::default_spec();
+    env: Environment<T>,
+    network_config: Eth2NetworkConfig,
+    matches: &ArgMatches,
+) -> Result<(), String> {
+    let spec = &network_config.chain_spec::<T>()?;
     let executor = env.core_context().executor;
 
     /*

lighthouse/tests/beacon_node.rs

@@ -177,7 +177,7 @@ fn checkpoint_sync_url_timeout_default() {
     CommandLineTest::new()
         .run_with_zero_port()
         .with_config(|config| {
-            assert_eq!(config.chain.checkpoint_sync_url_timeout, 60);
+            assert_eq!(config.chain.checkpoint_sync_url_timeout, 180);
         });
 }
 
@@ -1772,6 +1772,24 @@ fn no_reconstruct_historic_states_flag() {
         .run_with_zero_port()
         .with_config(|config| assert!(!config.chain.reconstruct_historic_states));
 }
+#[test]
+fn epochs_per_migration_default() {
+    CommandLineTest::new()
+        .run_with_zero_port()
+        .with_config(|config| {
+            assert_eq!(
+                config.chain.epochs_per_migration,
+                beacon_node::beacon_chain::migrate::DEFAULT_EPOCHS_PER_MIGRATION
+            )
+        });
+}
+#[test]
+fn epochs_per_migration_override() {
+    CommandLineTest::new()
+        .flag("epochs-per-migration", Some("128"))
+        .run_with_zero_port()
+        .with_config(|config| assert_eq!(config.chain.epochs_per_migration, 128));
+}
 
 // Tests for Slasher flags.
 // Using `--slasher-max-db-size` to work around https://github.com/sigp/lighthouse/issues/2342

scripts/local_testnet/clean.sh

@@ -9,5 +9,5 @@ set -Eeuo pipefail
 source ./vars.env
 
 if [ -d $DATADIR ]; then
-  rm -r $DATADIR
+  rm -rf $DATADIR
 fi

slasher/tests/backend.rs

@@ -1,4 +1,4 @@
-#![cfg(all(feature = "lmdb"))]
+#![cfg(feature = "lmdb")]
 
 use slasher::{config::MDBX_DATA_FILENAME, Config, DatabaseBackend, DatabaseBackendOverride};
 use std::fs::File;

testing/ef_tests/Makefile

@@ -13,7 +13,7 @@ BLS_TARBALL = $(patsubst %,%-$(BLS_TEST_TAG).tar.gz,$(BLS_TEST))
 BLS_OUTPUT_DIR := $(OUTPUT_DIR)/$(BLS_TEST_REPO_NAME)
 BLS_BASE_URL := https://github.com/ethereum/$(BLS_TEST_REPO_NAME)/releases/download/$(BLS_TEST_TAG)
 
-WGET := $(if $(LIGHTHOUSE_GITHUB_TOKEN),wget --header="Authorization: $(LIGHTHOUSE_GITHUB_TOKEN)",wget)
+CURL := $(if $(LIGHTHOUSE_GITHUB_TOKEN),curl -L --header "Authorization: $(LIGHTHOUSE_GITHUB_TOKEN)",curl -L)
 
 all:
 	make $(OUTPUT_DIR)
@@ -27,11 +27,11 @@ $(OUTPUT_DIR): $(TARBALLS)
 
 $(BLS_OUTPUT_DIR):
 	mkdir $(BLS_OUTPUT_DIR)
-	$(WGET) $(BLS_BASE_URL)/$(BLS_TEST).tar.gz -O $(BLS_TARBALL)
+	$(CURL) $(BLS_BASE_URL)/$(BLS_TEST).tar.gz -o $(BLS_TARBALL)
 	tar -xzf $(BLS_TARBALL) -C $(BLS_OUTPUT_DIR)
 
 %-$(TESTS_TAG).tar.gz:
-	$(WGET) $(BASE_URL)/$*.tar.gz -O $@
+	$(CURL) $(BASE_URL)/$*.tar.gz -o $@
 
 clean-test-files:
 	rm -rf $(OUTPUT_DIR) $(BLS_OUTPUT_DIR)

testing/node_test_rig/Cargo.toml

@@ -14,3 +14,4 @@ validator_client = { path = "../../validator_client" }
 validator_dir = { path = "../../common/validator_dir", features = ["insecure_keys"] }
 sensitive_url = { path = "../../common/sensitive_url" }
 execution_layer = { path = "../../beacon_node/execution_layer" }
+tokio = { version = "1.14.0", features = ["time"] }

testing/node_test_rig/src/lib.rs

@@ -10,6 +10,7 @@ use std::path::PathBuf;
 use std::time::Duration;
 use std::time::{SystemTime, UNIX_EPOCH};
 use tempfile::{Builder as TempBuilder, TempDir};
+use tokio::time::timeout;
 use types::EthSpec;
 use validator_client::ProductionValidatorClient;
 use validator_dir::insecure_keys::build_deterministic_validator_dirs;
@@ -24,6 +25,8 @@ pub use validator_client::Config as ValidatorConfig;
 
 /// The global timeout for HTTP requests to the beacon node.
 const HTTP_TIMEOUT: Duration = Duration::from_secs(4);
+/// The timeout for a beacon node to start up.
+const STARTUP_TIMEOUT: Duration = Duration::from_secs(60);
 
 /// Provides a beacon node that is running in the current process on a given tokio executor (it
 /// is _local_ to this process).
@@ -51,12 +54,16 @@ impl<E: EthSpec> LocalBeaconNode<E> {
         client_config.set_data_dir(datadir.path().into());
         client_config.network.network_dir = PathBuf::from(datadir.path()).join("network");
 
-        ProductionBeaconNode::new(context, client_config)
+        timeout(
-            .await
+            STARTUP_TIMEOUT,
-            .map(move |client| Self {
+            ProductionBeaconNode::new(context, client_config),
-                client: client.into_inner(),
+        )
-                datadir,
+        .await
-            })
+        .map_err(|_| format!("Beacon node startup timed out after {:?}", STARTUP_TIMEOUT))?
+        .map(move |client| Self {
+            client: client.into_inner(),
+            datadir,
+        })
     }
 }
 

testing/simulator/src/cli.rs

@@ -25,8 +25,8 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                         .default_value("4")
                         .help("Number of beacon nodes"))
                     .arg(Arg::with_name("proposer-nodes")
-                        .short("n")
+                        .short("p")
-                        .long("nodes")
+                        .long("proposer_nodes")
                         .takes_value(true)
                         .default_value("2")
                         .help("Number of proposer-only beacon nodes"))
@@ -64,8 +64,8 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                         .default_value("4")
                         .help("Number of beacon nodes"))
                     .arg(Arg::with_name("proposer-nodes")
-                        .short("n")
+                        .short("p")
-                        .long("nodes")
+                        .long("proposer_nodes")
                         .takes_value(true)
                         .default_value("2")
                         .help("Number of proposer-only beacon nodes"))

testing/simulator/src/eth1_sim.rs

@@ -1,14 +1,16 @@
 use crate::local_network::{EXECUTION_PORT, TERMINAL_BLOCK, TERMINAL_DIFFICULTY};
-use crate::{checks, LocalNetwork, E};
+use crate::{checks, LocalNetwork};
 use clap::ArgMatches;
 use eth1::{Eth1Endpoint, DEFAULT_CHAIN_ID};
 use eth1_test_rig::AnvilEth1Instance;
 
+use crate::retry::with_retry;
 use execution_layer::http::deposit_methods::Eth1Id;
 use futures::prelude::*;
+use node_test_rig::environment::RuntimeContext;
 use node_test_rig::{
     environment::{EnvironmentBuilder, LoggerConfig},
-    testing_client_config, testing_validator_config, ClientGenesis, ValidatorFiles,
+    testing_client_config, testing_validator_config, ClientConfig, ClientGenesis, ValidatorFiles,
 };
 use rayon::prelude::*;
 use sensitive_url::SensitiveUrl;
@@ -107,71 +109,24 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> {
     let context = env.core_context();
 
     let main_future = async {
-        /*
-         * Deploy the deposit contract, spawn tasks to keep creating new blocks and deposit
-         * validators.
-         */
-        let anvil_eth1_instance = AnvilEth1Instance::new(DEFAULT_CHAIN_ID.into()).await?;
-        let deposit_contract = anvil_eth1_instance.deposit_contract;
-        let chain_id = anvil_eth1_instance.anvil.chain_id();
-        let anvil = anvil_eth1_instance.anvil;
-        let eth1_endpoint = SensitiveUrl::parse(anvil.endpoint().as_str())
-            .expect("Unable to parse anvil endpoint.");
-        let deposit_contract_address = deposit_contract.address();
-
-        // Start a timer that produces eth1 blocks on an interval.
-        tokio::spawn(async move {
-            let mut interval = tokio::time::interval(eth1_block_time);
-            loop {
-                interval.tick().await;
-                let _ = anvil.evm_mine().await;
-            }
-        });
-
-        // Submit deposits to the deposit contract.
-        tokio::spawn(async move {
-            for i in 0..total_validator_count {
-                println!("Submitting deposit for validator {}...", i);
-                let _ = deposit_contract
-                    .deposit_deterministic_async::<E>(i, deposit_amount)
-                    .await;
-            }
-        });
-
-        let mut beacon_config = testing_client_config();
-
-        beacon_config.genesis = ClientGenesis::DepositContract;
-        beacon_config.eth1.endpoint = Eth1Endpoint::NoAuth(eth1_endpoint);
-        beacon_config.eth1.deposit_contract_address = deposit_contract_address;
-        beacon_config.eth1.deposit_contract_deploy_block = 0;
-        beacon_config.eth1.lowest_cached_block_number = 0;
-        beacon_config.eth1.follow_distance = 1;
-        beacon_config.eth1.node_far_behind_seconds = 20;
-        beacon_config.dummy_eth1_backend = false;
-        beacon_config.sync_eth1_chain = true;
-        beacon_config.eth1.auto_update_interval_millis = eth1_block_time.as_millis() as u64;
-        beacon_config.eth1.chain_id = Eth1Id::from(chain_id);
-        beacon_config.network.target_peers = node_count + proposer_nodes - 1;
-
-        beacon_config.network.enr_address = (Some(Ipv4Addr::LOCALHOST), None);
-
-        if post_merge_sim {
-            let el_config = execution_layer::Config {
-                execution_endpoints: vec![SensitiveUrl::parse(&format!(
-                    "http://localhost:{}",
-                    EXECUTION_PORT
-                ))
-                .unwrap()],
-                ..Default::default()
-            };
-
-            beacon_config.execution_layer = Some(el_config);
-        }
-
         /*
          * Create a new `LocalNetwork` with one beacon node.
          */
-        let network = LocalNetwork::new(context.clone(), beacon_config.clone()).await?;
+        let max_retries = 3;
+        let (network, beacon_config) = with_retry(max_retries, || {
+            Box::pin(create_local_network(
+                LocalNetworkParams {
+                    eth1_block_time,
+                    total_validator_count,
+                    deposit_amount,
+                    node_count,
+                    proposer_nodes,
+                    post_merge_sim,
+                },
+                context.clone(),
+            ))
+        })
+        .await?;
 
         /*
          * One by one, add beacon nodes to the network.
@@ -341,3 +296,88 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> {
 
     Ok(())
 }
+
+struct LocalNetworkParams {
+    eth1_block_time: Duration,
+    total_validator_count: usize,
+    deposit_amount: u64,
+    node_count: usize,
+    proposer_nodes: usize,
+    post_merge_sim: bool,
+}
+
+async fn create_local_network<E: EthSpec>(
+    LocalNetworkParams {
+        eth1_block_time,
+        total_validator_count,
+        deposit_amount,
+        node_count,
+        proposer_nodes,
+        post_merge_sim,
+    }: LocalNetworkParams,
+    context: RuntimeContext<E>,
+) -> Result<(LocalNetwork<E>, ClientConfig), String> {
+    /*
+     * Deploy the deposit contract, spawn tasks to keep creating new blocks and deposit
+     * validators.
+     */
+    let anvil_eth1_instance = AnvilEth1Instance::new(DEFAULT_CHAIN_ID.into()).await?;
+    let deposit_contract = anvil_eth1_instance.deposit_contract;
+    let chain_id = anvil_eth1_instance.anvil.chain_id();
+    let anvil = anvil_eth1_instance.anvil;
+    let eth1_endpoint =
+        SensitiveUrl::parse(anvil.endpoint().as_str()).expect("Unable to parse anvil endpoint.");
+    let deposit_contract_address = deposit_contract.address();
+
+    // Start a timer that produces eth1 blocks on an interval.
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(eth1_block_time);
+        loop {
+            interval.tick().await;
+            let _ = anvil.evm_mine().await;
+        }
+    });
+
+    // Submit deposits to the deposit contract.
+    tokio::spawn(async move {
+        for i in 0..total_validator_count {
+            println!("Submitting deposit for validator {}...", i);
+            let _ = deposit_contract
+                .deposit_deterministic_async::<E>(i, deposit_amount)
+                .await;
+        }
+    });
+
+    let mut beacon_config = testing_client_config();
+
+    beacon_config.genesis = ClientGenesis::DepositContract;
+    beacon_config.eth1.endpoint = Eth1Endpoint::NoAuth(eth1_endpoint);
+    beacon_config.eth1.deposit_contract_address = deposit_contract_address;
+    beacon_config.eth1.deposit_contract_deploy_block = 0;
+    beacon_config.eth1.lowest_cached_block_number = 0;
+    beacon_config.eth1.follow_distance = 1;
+    beacon_config.eth1.node_far_behind_seconds = 20;
+    beacon_config.dummy_eth1_backend = false;
+    beacon_config.sync_eth1_chain = true;
+    beacon_config.eth1.auto_update_interval_millis = eth1_block_time.as_millis() as u64;
+    beacon_config.eth1.chain_id = Eth1Id::from(chain_id);
+    beacon_config.network.target_peers = node_count + proposer_nodes - 1;
+
+    beacon_config.network.enr_address = (Some(Ipv4Addr::LOCALHOST), None);
+
+    if post_merge_sim {
+        let el_config = execution_layer::Config {
+            execution_endpoints: vec![SensitiveUrl::parse(&format!(
+                "http://localhost:{}",
+                EXECUTION_PORT
+            ))
+            .unwrap()],
+            ..Default::default()
+        };
+
+        beacon_config.execution_layer = Some(el_config);
+    }
+
+    let network = LocalNetwork::new(context, beacon_config.clone()).await?;
+    Ok((network, beacon_config))
+}

testing/simulator/src/main.rs

@@ -21,6 +21,7 @@ mod cli;
 mod eth1_sim;
 mod local_network;
 mod no_eth1_sim;
+mod retry;
 mod sync_sim;
 
 use cli::cli_app;

testing/simulator/src/retry.rs

@@ -0,0 +1,63 @@
+use std::fmt::Debug;
+use std::future::Future;
+use std::pin::Pin;
+
+/// Executes the function with a specified number of retries if the function returns an error.
+/// Once it exceeds `max_retries` and still fails, the error is returned.
+pub async fn with_retry<T, E, F>(max_retries: usize, mut func: F) -> Result<T, E>
+where
+    F: FnMut() -> Pin<Box<dyn Future<Output = Result<T, E>>>>,
+    E: Debug,
+{
+    let mut retry_count = 0;
+    loop {
+        let result = Box::pin(func()).await;
+        if result.is_ok() || retry_count >= max_retries {
+            break result;
+        }
+        retry_count += 1;
+
+        if let Err(e) = result {
+            eprintln!(
+                "Operation failed with error {:?}, retrying {} of {}",
+                e, retry_count, max_retries
+            );
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::collections::VecDeque;
+
+    async fn my_async_func(is_ok: bool) -> Result<(), ()> {
+        if is_ok {
+            Ok(())
+        } else {
+            Err(())
+        }
+    }
+
+    #[tokio::test]
+    async fn test_with_retry_ok() {
+        let res = with_retry(3, || Box::pin(my_async_func(true))).await;
+        assert!(res.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_with_retry_2nd_ok() {
+        let mut mock_results = VecDeque::from([false, true]);
+        let res = with_retry(3, || {
+            Box::pin(my_async_func(mock_results.pop_front().unwrap()))
+        })
+        .await;
+        assert!(res.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_with_retry_fail() {
+        let res = with_retry(3, || Box::pin(my_async_func(false))).await;
+        assert!(res.is_err());
+    }
+}

watch/Cargo.toml

@@ -21,7 +21,7 @@ types = { path = "../consensus/types" }
 eth2 = { path = "../common/eth2" }
 beacon_node = { path = "../beacon_node"}
 tokio = { version = "1.14.0", features = ["time"] }
-axum = "0.5.15"
+axum = "0.6.18"
 hyper = "0.14.20"
 serde = "1.0.116"
 serde_json = "1.0.58"

watch/src/server/mod.rs

@@ -5,7 +5,6 @@ use crate::config::Config as FullConfig;
 use crate::database::{self, PgPool};
 use crate::suboptimal_attestations::{attestation_routes, blockprint_attestation_routes};
 use axum::{
-    handler::Handler,
     http::{StatusCode, Uri},
     routing::get,
     Extension, Json, Router,
@@ -104,7 +103,7 @@ pub fn start_server(
     }
 
     let app = routes
-        .fallback(route_not_found.into_service())
+        .fallback(route_not_found)
         .layer(Extension(pool))
         .layer(Extension(slots_per_epoch));