Backfill blocks only to the WSP by default (#4082)

## Limit Backfill Sync

This PR transitions Lighthouse from syncing all the way back to genesis to only syncing back to the weak subjectivity point (~ 5 months) when syncing via a checkpoint sync.

There are a number of important points to note with this PR:

- Firstly and most importantly, this PR fundamentally shifts the default security guarantees of checkpoint syncing in Lighthouse. Prior to this PR, Lighthouse could verify the checkpoint of any given chain by ensuring the chain eventually terminates at the corresponding genesis. This guarantee can still be employed via the new CLI flag --genesis-backfill which will prompt lighthouse to the old behaviour of downloading all blocks back to genesis. The new behaviour only checks the proposer signatures for the last 5 months of blocks but cannot guarantee the chain matches the genesis chain.
- I have not modified any of the peer scoring or RPC responses. Clients syncing from gensis, will downscore new Lighthouse peers that do not possess blocks prior to the WSP. This is by design, as Lighthouse nodes of this form, need a mechanism to sort through peers in order to find useful peers in order to complete their genesis sync. We therefore do not discriminate between empty/error responses for blocks prior or post the local WSP. If we request a block that a peer does not posses, then fundamentally that peer is less useful to us than other peers.
- This will make a radical shift in that the majority of nodes will no longer store the full history of the chain. In the future we could add a pruning mechanism to remove old blocks from the db also.


Co-authored-by: Paul Hauner <paul@paulhauner.com>

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -420,6 +420,8 @@ pub struct BeaconChain<T: BeaconChainTypes> {
     pub slasher: Option<Arc<Slasher<T::EthSpec>>>,
     /// Provides monitoring of a set of explicitly defined validators.
     pub validator_monitor: RwLock<ValidatorMonitor<T::EthSpec>>,
+    /// The slot at which blocks are downloaded back to.
+    pub genesis_backfill_slot: Slot,
 }
 
 type BeaconBlockAndState<T, Payload> = (BeaconBlock<T, Payload>, BeaconState<T>);

beacon_node/beacon_chain/src/builder.rs

@@ -772,6 +772,29 @@ where
         let canonical_head = CanonicalHead::new(fork_choice, Arc::new(head_snapshot));
         let shuffling_cache_size = self.chain_config.shuffling_cache_size;
 
+        // Calculate the weak subjectivity point in which to backfill blocks to.
+        let genesis_backfill_slot = if self.chain_config.genesis_backfill {
+            Slot::new(0)
+        } else {
+            let backfill_epoch_range = (self.spec.min_validator_withdrawability_delay
+                + self.spec.churn_limit_quotient)
+                .as_u64()
+                / 2;
+            match slot_clock.now() {
+                Some(current_slot) => {
+                    let genesis_backfill_epoch = current_slot
+                        .epoch(TEthSpec::slots_per_epoch())
+                        .saturating_sub(backfill_epoch_range);
+                    genesis_backfill_epoch.start_slot(TEthSpec::slots_per_epoch())
+                }
+                None => {
+                    // The slot clock cannot derive the current slot. We therefore assume we are
+                    // at or prior to genesis and backfill should sync all the way to genesis.
+                    Slot::new(0)
+                }
+            }
+        };
+
         let beacon_chain = BeaconChain {
             spec: self.spec,
             config: self.chain_config,
@@ -839,6 +862,7 @@ where
             graffiti: self.graffiti,
             slasher: self.slasher.clone(),
             validator_monitor: RwLock::new(validator_monitor),
+            genesis_backfill_slot,
         };
 
         let head = beacon_chain.head_snapshot();

beacon_node/beacon_chain/src/chain_config.rs

@@ -73,6 +73,9 @@ pub struct ChainConfig {
     pub optimistic_finalized_sync: bool,
     /// The size of the shuffling cache,
     pub shuffling_cache_size: usize,
+    /// If using a weak-subjectivity sync, whether we should download blocks all the way back to
+    /// genesis.
+    pub genesis_backfill: bool,
     /// Whether to send payload attributes every slot, regardless of connected proposers.
     ///
     /// This is useful for block builders and testing.
@@ -106,6 +109,7 @@ impl Default for ChainConfig {
             // This value isn't actually read except in tests.
             optimistic_finalized_sync: true,
             shuffling_cache_size: crate::shuffling_cache::DEFAULT_CACHE_SIZE,
+            genesis_backfill: false,
             always_prepare_payload: false,
             enable_backfill_rate_limiting: true,
         }

beacon_node/beacon_chain/src/historical_blocks.rs

@@ -189,13 +189,17 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             oldest_block_parent: expected_block_root,
             ..anchor_info
         };
-        let backfill_complete = new_anchor.block_backfill_complete();
+        let backfill_complete = new_anchor.block_backfill_complete(self.genesis_backfill_slot);
         self.store
             .compare_and_set_anchor_info_with_write(Some(anchor_info), Some(new_anchor))?;
 
         // If backfill has completed and the chain is configured to reconstruct historic states,
         // send a message to the background migrator instructing it to begin reconstruction.
-        if backfill_complete && self.config.reconstruct_historic_states {
+        // This can only happen if we have backfilled all the way to genesis.
+        if backfill_complete
+            && self.genesis_backfill_slot == Slot::new(0)
+            && self.config.reconstruct_historic_states
+        {
             self.store_migrator.process_reconstruction();
         }
 

beacon_node/client/src/builder.rs

@@ -250,6 +250,12 @@ where
                 genesis_state_bytes,
             } => {
                 info!(context.log(), "Starting checkpoint sync");
+                if config.chain.genesis_backfill {
+                    info!(
+                        context.log(),
+                        "Blocks will downloaded all the way back to genesis"
+                    );
+                }
 
                 let anchor_state = BeaconState::from_ssz_bytes(&anchor_state_bytes, &spec)
                     .map_err(|e| format!("Unable to parse weak subj state SSZ: {:?}", e))?;
@@ -271,6 +277,12 @@ where
                     "Starting checkpoint sync";
                     "remote_url" => %url,
                 );
+                if config.chain.genesis_backfill {
+                    info!(
+                        context.log(),
+                        "Blocks will be downloaded all the way back to genesis"
+                    );
+                }
 
                 let remote = BeaconNodeHttpClient::new(
                     url,

beacon_node/client/src/notifier.rs

@@ -142,7 +142,8 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
                             .get_anchor_info()
                             .map(|ai| ai.oldest_block_slot)
                         {
-                            sync_distance = current_anchor_slot;
+                            sync_distance = current_anchor_slot
+                                .saturating_sub(beacon_chain.genesis_backfill_slot);
                             speedo
                                 // For backfill sync use a fake slot which is the distance we've progressed from the starting `oldest_block_slot`.
                                 .observe(
@@ -207,14 +208,14 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
                         "Downloading historical blocks";
                         "distance" => distance,
                         "speed" => sync_speed_pretty(speed),
-                        "est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(original_anchor_slot.unwrap_or(current_slot))),
+                        "est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(original_anchor_slot.unwrap_or(current_slot).saturating_sub(beacon_chain.genesis_backfill_slot))),
                     );
                 } else {
                     info!(
                         log,
                         "Downloading historical blocks";
                         "distance" => distance,
-                        "est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(original_anchor_slot.unwrap_or(current_slot))),
+                        "est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(original_anchor_slot.unwrap_or(current_slot).saturating_sub(beacon_chain.genesis_backfill_slot))),
                     );
                 }
             } else if !is_backfilling && last_backfill_log_slot.is_some() {

beacon_node/lighthouse_network/src/types/sync_state.rs

@@ -13,7 +13,7 @@ pub enum SyncState {
     /// The node is undertaking a backfill sync. This occurs when a user has specified a trusted
     /// state. The node first syncs "forward" by downloading blocks up to the current head as
     /// specified by its peers. Once completed, the node enters this sync state and attempts to
-    /// download all required historical blocks to complete its chain.
+    /// download all required historical blocks.
     BackFillSyncing { completed: usize, remaining: usize },
     /// The node has completed syncing a finalized chain and is in the process of re-evaluating
     /// which sync state to progress to.

beacon_node/network/src/sync/backfill_sync/mod.rs

@@ -159,20 +159,20 @@ impl<T: BeaconChainTypes> BackFillSync<T> {
         // If, for some reason a backfill has already been completed (or we've used a trusted
         // genesis root) then backfill has been completed.
 
-        let (state, current_start) = if let Some(anchor_info) = beacon_chain.store.get_anchor_info()
-        {
-            if anchor_info.block_backfill_complete() {
-                (BackFillState::Completed, Epoch::new(0))
-            } else {
-                (
-                    BackFillState::Paused,
-                    anchor_info
-                        .oldest_block_slot
-                        .epoch(T::EthSpec::slots_per_epoch()),
-                )
+        let (state, current_start) = match beacon_chain.store.get_anchor_info() {
+            Some(anchor_info) => {
+                if anchor_info.block_backfill_complete(beacon_chain.genesis_backfill_slot) {
+                    (BackFillState::Completed, Epoch::new(0))
+                } else {
+                    (
+                        BackFillState::Paused,
+                        anchor_info
+                            .oldest_block_slot
+                            .epoch(T::EthSpec::slots_per_epoch()),
+                    )
+                }
             }
-        } else {
-            (BackFillState::NotRequired, Epoch::new(0))
+            None => (BackFillState::NotRequired, Epoch::new(0)),
         };
 
         let bfs = BackFillSync {
@@ -287,6 +287,7 @@ impl<T: BeaconChainTypes> BackFillSync<T> {
             remaining: self
                 .current_start
                 .start_slot(T::EthSpec::slots_per_epoch())
+                .saturating_sub(self.beacon_chain.genesis_backfill_slot)
                 .as_usize(),
         })
     }
@@ -1097,7 +1098,12 @@ impl<T: BeaconChainTypes> BackFillSync<T> {
         match self.batches.entry(batch_id) {
             Entry::Occupied(_) => {
                 // this batch doesn't need downloading, let this same function decide the next batch
-                if batch_id == 0 {
+                if batch_id
+                    == self
+                        .beacon_chain
+                        .genesis_backfill_slot
+                        .epoch(T::EthSpec::slots_per_epoch())
+                {
                     self.last_batch_downloaded = true;
                 }
 
@@ -1108,7 +1114,12 @@ impl<T: BeaconChainTypes> BackFillSync<T> {
             }
             Entry::Vacant(entry) => {
                 entry.insert(BatchInfo::new(&batch_id, BACKFILL_EPOCHS_PER_BATCH));
-                if batch_id == 0 {
+                if batch_id
+                    == self
+                        .beacon_chain
+                        .genesis_backfill_slot
+                        .epoch(T::EthSpec::slots_per_epoch())
+                {
                     self.last_batch_downloaded = true;
                 }
                 self.to_be_downloaded = self
@@ -1125,7 +1136,7 @@ impl<T: BeaconChainTypes> BackFillSync<T> {
     /// not required.
     fn reset_start_epoch(&mut self) -> Result<(), ResetEpochError> {
         if let Some(anchor_info) = self.beacon_chain.store.get_anchor_info() {
-            if anchor_info.block_backfill_complete() {
+            if anchor_info.block_backfill_complete(self.beacon_chain.genesis_backfill_slot) {
                 Err(ResetEpochError::SyncCompleted)
             } else {
                 self.current_start = anchor_info
@@ -1140,12 +1151,17 @@ impl<T: BeaconChainTypes> BackFillSync<T> {
 
     /// Checks with the beacon chain if backfill sync has completed.
     fn check_completed(&mut self) -> bool {
-        if self.current_start == 0 {
+        if self.current_start
+            == self
+                .beacon_chain
+                .genesis_backfill_slot
+                .epoch(T::EthSpec::slots_per_epoch())
+        {
             // Check that the beacon chain agrees
 
             if let Some(anchor_info) = self.beacon_chain.store.get_anchor_info() {
                 // Conditions that we have completed a backfill sync
-                if anchor_info.block_backfill_complete() {
+                if anchor_info.block_backfill_complete(self.beacon_chain.genesis_backfill_slot) {
                     return true;
                 } else {
                     error!(self.log, "Backfill out of sync with beacon chain");

beacon_node/src/cli.rs

@@ -247,6 +247,12 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .help("One or more comma-delimited trusted peer ids which always have the highest score according to the peer scoring system.")
                 .takes_value(true),
         )
+        .arg(
+            Arg::with_name("genesis-backfill")
+                .long("genesis-backfill")
+                .help("Attempts to download blocks all the way back to genesis when checkpoint syncing.")
+                .takes_value(false),
+        )
         .arg(
             Arg::with_name("enable-private-discovery")
                 .long("enable-private-discovery")
@@ -831,7 +837,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
         .arg(
             Arg::with_name("reconstruct-historic-states")
                 .long("reconstruct-historic-states")
-                .help("After a checkpoint sync, reconstruct historic states in the database.")
+                .help("After a checkpoint sync, reconstruct historic states in the database. This requires syncing all the way back to genesis.")
                 .takes_value(false)
         )
         .arg(

beacon_node/src/config.rs

@@ -508,6 +508,7 @@ pub fn get_config<E: EthSpec>(
 
     if cli_args.is_present("reconstruct-historic-states") {
         client_config.chain.reconstruct_historic_states = true;
+        client_config.chain.genesis_backfill = true;
     }
 
     let raw_graffiti = if let Some(graffiti) = cli_args.value_of("graffiti") {
@@ -780,6 +781,9 @@ pub fn get_config<E: EthSpec>(
     client_config.chain.optimistic_finalized_sync =
         !cli_args.is_present("disable-optimistic-finalized-sync");
 
+    if cli_args.is_present("genesis-backfill") {
+        client_config.chain.genesis_backfill = true;
+    }
     // Payload selection configs
     if cli_args.is_present("always-prefer-builder-payload") {
         client_config.always_prefer_builder_payload = true;

beacon_node/store/src/metadata.rs

@@ -99,8 +99,10 @@ pub struct AnchorInfo {
 
 impl AnchorInfo {
     /// Returns true if the block backfill has completed.
-    pub fn block_backfill_complete(&self) -> bool {
-        self.oldest_block_slot == 0
+    /// This is a comparison between the oldest block slot and the target backfill slot (which is
+    /// likely to be the closest WSP).
+    pub fn block_backfill_complete(&self, target_slot: Slot) -> bool {
+        self.oldest_block_slot <= target_slot
     }
 }
 

lighthouse/tests/beacon_node.rs

@@ -345,6 +345,23 @@ fn trusted_peers_flag() {
         });
 }
 
+#[test]
+fn genesis_backfill_flag() {
+    CommandLineTest::new()
+        .flag("genesis-backfill", None)
+        .run_with_zero_port()
+        .with_config(|config| assert_eq!(config.chain.genesis_backfill, true));
+}
+
+/// The genesis backfill flag should be enabled if historic states flag is set.
+#[test]
+fn genesis_backfill_with_historic_flag() {
+    CommandLineTest::new()
+        .flag("reconstruct-historic-states", None)
+        .run_with_zero_port()
+        .with_config(|config| assert_eq!(config.chain.genesis_backfill, true));
+}
+
 #[test]
 fn always_prefer_builder_payload_flag() {
     CommandLineTest::new()