Implement checkpoint sync (#2244)

## Issue Addressed

Closes #1891
Closes #1784

## Proposed Changes

Implement checkpoint sync for Lighthouse, enabling it to start from a weak subjectivity checkpoint.

## Additional Info

- [x] Return unavailable status for out-of-range blocks requested by peers (#2561)
- [x] Implement sync daemon for fetching historical blocks (#2561)
- [x] Verify chain hashes (either in `historical_blocks.rs` or the calling module)
- [x] Consistency check for initial block + state
- [x] Fetch the initial state and block from a beacon node HTTP endpoint
- [x] Don't crash fetching beacon states by slot from the API
- [x] Background service for state reconstruction, triggered by CLI flag or API call.

Considered out of scope for this PR:

- Drop the requirement to provide the `--checkpoint-block` (this would require some pretty heavy refactoring of block verification)


Co-authored-by: Diva M <divma@protonmail.com>

consensus/fork_choice/tests/tests.rs

@@ -983,7 +983,7 @@ fn weak_subjectivity_fail_on_startup() {
 
     let chain_config = ChainConfig {
         weak_subjectivity_checkpoint: Some(Checkpoint { epoch, root }),
-        import_max_skip_slots: None,
+        ..ChainConfig::default()
     };
 
     ForkChoiceTest::new_with_chain_config(chain_config);
@@ -996,7 +996,7 @@ fn weak_subjectivity_pass_on_startup() {
 
     let chain_config = ChainConfig {
         weak_subjectivity_checkpoint: Some(Checkpoint { epoch, root }),
-        import_max_skip_slots: None,
+        ..ChainConfig::default()
     };
 
     ForkChoiceTest::new_with_chain_config(chain_config)
@@ -1021,7 +1021,7 @@ fn weak_subjectivity_check_passes() {
 
     let chain_config = ChainConfig {
         weak_subjectivity_checkpoint: Some(checkpoint),
-        import_max_skip_slots: None,
+        ..ChainConfig::default()
     };
 
     ForkChoiceTest::new_with_chain_config(chain_config.clone())
@@ -1051,7 +1051,7 @@ fn weak_subjectivity_check_fails_early_epoch() {
 
     let chain_config = ChainConfig {
         weak_subjectivity_checkpoint: Some(checkpoint),
-        import_max_skip_slots: None,
+        ..ChainConfig::default()
     };
 
     ForkChoiceTest::new_with_chain_config(chain_config.clone())
@@ -1080,7 +1080,7 @@ fn weak_subjectivity_check_fails_late_epoch() {
 
     let chain_config = ChainConfig {
         weak_subjectivity_checkpoint: Some(checkpoint),
-        import_max_skip_slots: None,
+        ..ChainConfig::default()
     };
 
     ForkChoiceTest::new_with_chain_config(chain_config.clone())
@@ -1109,7 +1109,7 @@ fn weak_subjectivity_check_fails_incorrect_root() {
 
     let chain_config = ChainConfig {
         weak_subjectivity_checkpoint: Some(checkpoint),
-        import_max_skip_slots: None,
+        ..ChainConfig::default()
     };
 
     ForkChoiceTest::new_with_chain_config(chain_config.clone())
@@ -1145,7 +1145,7 @@ fn weak_subjectivity_check_epoch_boundary_is_skip_slot() {
 
     let chain_config = ChainConfig {
         weak_subjectivity_checkpoint: Some(checkpoint),
-        import_max_skip_slots: None,
+        ..ChainConfig::default()
     };
 
     // recreate the chain exactly
@@ -1186,7 +1186,7 @@ fn weak_subjectivity_check_epoch_boundary_is_skip_slot_failure() {
 
     let chain_config = ChainConfig {
         weak_subjectivity_checkpoint: Some(checkpoint),
-        import_max_skip_slots: None,
+        ..ChainConfig::default()
     };
 
     // recreate the chain exactly

consensus/state_processing/src/per_block_processing.rs

@@ -11,7 +11,7 @@ pub use self::verify_attester_slashing::{
 };
 pub use self::verify_proposer_slashing::verify_proposer_slashing;
 pub use altair::sync_committee::process_sync_aggregate;
-pub use block_signature_verifier::BlockSignatureVerifier;
+pub use block_signature_verifier::{BlockSignatureVerifier, ParallelSignatureSets};
 pub use is_valid_indexed_attestation::is_valid_indexed_attestation;
 pub use process_operations::process_operations;
 pub use verify_attestation::{

consensus/state_processing/src/per_block_processing/block_signature_verifier.rs

@@ -73,9 +73,20 @@ where
     decompressor: D,
     state: &'a BeaconState<T>,
     spec: &'a ChainSpec,
+    sets: ParallelSignatureSets<'a>,
+}
+
+#[derive(Default)]
+pub struct ParallelSignatureSets<'a> {
     sets: Vec<SignatureSet<'a>>,
 }
 
+impl<'a> From<Vec<SignatureSet<'a>>> for ParallelSignatureSets<'a> {
+    fn from(sets: Vec<SignatureSet<'a>>) -> Self {
+        Self { sets }
+    }
+}
+
 impl<'a, T, F, D> BlockSignatureVerifier<'a, T, F, D>
 where
     T: EthSpec,
@@ -95,7 +106,7 @@ where
             decompressor,
             state,
             spec,
-            sets: vec![],
+            sets: ParallelSignatureSets::default(),
         }
     }
 
@@ -119,36 +130,6 @@ where
         verifier.verify()
     }
 
-    /// Verify all* the signatures that have been included in `self`, returning `Ok(())` if the
-    /// signatures are all valid.
-    ///
-    /// ## Notes
-    ///
-    /// Signature validation will take place in accordance to the [Faster verification of multiple
-    /// BLS signatures](https://ethresear.ch/t/fast-verification-of-multiple-bls-signatures/5407)
-    /// optimization proposed by Vitalik Buterin.
-    ///
-    /// It is not possible to know exactly _which_ signature is invalid here, just that
-    /// _at least one_ was invalid.
-    ///
-    /// Uses `rayon` to do a map-reduce of Vitalik's method across multiple cores.
-    pub fn verify(self) -> Result<()> {
-        let num_sets = self.sets.len();
-        let num_chunks = std::cmp::max(1, num_sets / rayon::current_num_threads());
-        let result: bool = self
-            .sets
-            .into_par_iter()
-            .chunks(num_chunks)
-            .map(|chunk| verify_signature_sets(chunk.iter()))
-            .reduce(|| true, |current, this| current && this);
-
-        if result {
-            Ok(())
-        } else {
-            Err(Error::SignatureInvalid)
-        }
-    }
-
     /// Includes all signatures on the block (except the deposit signatures) for verification.
     pub fn include_all_signatures(
         &mut self,
@@ -210,6 +191,7 @@ where
     /// Includes all signatures in `self.block.body.proposer_slashings` for verification.
     pub fn include_proposer_slashings(&mut self, block: &'a SignedBeaconBlock<T>) -> Result<()> {
         self.sets
+            .sets
             .reserve(block.message().body().proposer_slashings().len() * 2);
 
         block
@@ -235,6 +217,7 @@ where
     /// Includes all signatures in `self.block.body.attester_slashings` for verification.
     pub fn include_attester_slashings(&mut self, block: &'a SignedBeaconBlock<T>) -> Result<()> {
         self.sets
+            .sets
             .reserve(block.message().body().attester_slashings().len() * 2);
 
         block
@@ -263,6 +246,7 @@ where
         block: &'a SignedBeaconBlock<T>,
     ) -> Result<Vec<IndexedAttestation<T>>> {
         self.sets
+            .sets
             .reserve(block.message().body().attestations().len());
 
         block
@@ -298,6 +282,7 @@ where
     /// Includes all signatures in `self.block.body.voluntary_exits` for verification.
     pub fn include_exits(&mut self, block: &'a SignedBeaconBlock<T>) -> Result<()> {
         self.sets
+            .sets
             .reserve(block.message().body().voluntary_exits().len());
 
         block
@@ -331,4 +316,46 @@ where
         }
         Ok(())
     }
+
+    /// Verify all the signatures that have been included in `self`, returning `true` if and only if
+    /// all the signatures are valid.
+    ///
+    /// See `ParallelSignatureSets::verify` for more info.
+    pub fn verify(self) -> Result<()> {
+        if self.sets.verify() {
+            Ok(())
+        } else {
+            Err(Error::SignatureInvalid)
+        }
+    }
+}
+
+impl<'a> ParallelSignatureSets<'a> {
+    pub fn push(&mut self, set: SignatureSet<'a>) {
+        self.sets.push(set);
+    }
+
+    /// Verify all the signatures that have been included in `self`, returning `true` if and only if
+    /// all the signatures are valid.
+    ///
+    /// ## Notes
+    ///
+    /// Signature validation will take place in accordance to the [Faster verification of multiple
+    /// BLS signatures](https://ethresear.ch/t/fast-verification-of-multiple-bls-signatures/5407)
+    /// optimization proposed by Vitalik Buterin.
+    ///
+    /// It is not possible to know exactly _which_ signature is invalid here, just that
+    /// _at least one_ was invalid.
+    ///
+    /// Uses `rayon` to do a map-reduce of Vitalik's method across multiple cores.
+    #[must_use]
+    pub fn verify(self) -> bool {
+        let num_sets = self.sets.len();
+        let num_chunks = std::cmp::max(1, num_sets / rayon::current_num_threads());
+        self.sets
+            .into_par_iter()
+            .chunks(num_chunks)
+            .map(|chunk| verify_signature_sets(chunk.iter()))
+            .reduce(|| true, |current, this| current && this)
+    }
 }

consensus/state_processing/src/per_block_processing/signature_sets.rs

@@ -77,6 +77,45 @@ pub fn block_proposal_signature_set<'a, T, F>(
     block_root: Option<Hash256>,
     spec: &'a ChainSpec,
 ) -> Result<SignatureSet<'a>>
+where
+    T: EthSpec,
+    F: Fn(usize) -> Option<Cow<'a, PublicKey>>,
+{
+    let block = signed_block.message();
+    let proposer_index = state.get_beacon_proposer_index(block.slot(), spec)? as u64;
+
+    if proposer_index != block.proposer_index() {
+        return Err(Error::IncorrectBlockProposer {
+            block: block.proposer_index(),
+            local_shuffling: proposer_index,
+        });
+    }
+
+    block_proposal_signature_set_from_parts(
+        signed_block,
+        block_root,
+        proposer_index,
+        &state.fork(),
+        state.genesis_validators_root(),
+        get_pubkey,
+        spec,
+    )
+}
+
+/// A signature set that is valid if a block was signed by the expected block producer.
+///
+/// Unlike `block_proposal_signature_set` this does **not** check that the proposer index is
+/// correct according to the shuffling. It should only be used if no suitable `BeaconState` is
+/// available.
+pub fn block_proposal_signature_set_from_parts<'a, T, F>(
+    signed_block: &'a SignedBeaconBlock<T>,
+    block_root: Option<Hash256>,
+    proposer_index: u64,
+    fork: &Fork,
+    genesis_validators_root: Hash256,
+    get_pubkey: F,
+    spec: &'a ChainSpec,
+) -> Result<SignatureSet<'a>>
 where
     T: EthSpec,
     F: Fn(usize) -> Option<Cow<'a, PublicKey>>,
@@ -87,20 +126,11 @@ where
         .map_err(Error::InconsistentBlockFork)?;
 
     let block = signed_block.message();
-    let proposer_index = state.get_beacon_proposer_index(block.slot(), spec)?;
-
-    if proposer_index as u64 != block.proposer_index() {
-        return Err(Error::IncorrectBlockProposer {
-            block: block.proposer_index(),
-            local_shuffling: proposer_index as u64,
-        });
-    }
-
     let domain = spec.get_domain(
         block.slot().epoch(T::slots_per_epoch()),
         Domain::BeaconProposer,
-        &state.fork(),
-        state.genesis_validators_root(),
+        fork,
+        genesis_validators_root,
     );
 
     let message = if let Some(root) = block_root {
@@ -115,7 +145,7 @@ where
 
     Ok(SignatureSet::single_pubkey(
         signed_block.signature(),
-        get_pubkey(proposer_index).ok_or_else(|| Error::ValidatorUnknown(proposer_index as u64))?,
+        get_pubkey(proposer_index as usize).ok_or(Error::ValidatorUnknown(proposer_index))?,
         message,
     ))
 }

consensus/types/Cargo.toml

@@ -51,6 +51,7 @@ serde_json = "1.0.58"
 criterion = "0.3.3"
 beacon_chain = { path = "../../beacon_node/beacon_chain" }
 eth2_interop_keypairs = { path = "../../common/eth2_interop_keypairs" }
+state_processing = { path = "../state_processing" }
 
 [features]
 default = ["sqlite", "legacy-arith"]

consensus/types/src/beacon_block.rs

@@ -197,6 +197,11 @@ impl<'a, T: EthSpec> BeaconBlockRef<'a, T> {
         }
     }
 
+    /// Returns the epoch corresponding to `self.slot()`.
+    pub fn epoch(&self) -> Epoch {
+        self.slot().epoch(T::slots_per_epoch())
+    }
+
     /// Returns a full `BeaconBlockHeader` of this block.
     pub fn block_header(&self) -> BeaconBlockHeader {
         BeaconBlockHeader {

consensus/types/src/beacon_state/tests.rs

@@ -1,15 +1,21 @@
 #![cfg(test)]
 use crate::test_utils::*;
+use crate::test_utils::{SeedableRng, XorShiftRng};
 use beacon_chain::store::config::StoreConfig;
-use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType};
+use beacon_chain::test_utils::{
+    interop_genesis_state, test_spec, BeaconChainHarness, EphemeralHarnessType,
+};
 use beacon_chain::types::{
     test_utils::TestRandom, BeaconState, BeaconStateAltair, BeaconStateBase, BeaconStateError,
     ChainSpec, CloneConfig, Domain, Epoch, EthSpec, FixedVector, Hash256, Keypair, MainnetEthSpec,
     MinimalEthSpec, RelativeEpoch, Slot,
 };
+use safe_arith::SafeArith;
 use ssz::{Decode, Encode};
+use state_processing::per_slot_processing;
 use std::ops::Mul;
 use swap_or_not_shuffle::compute_shuffled_index;
+use tree_hash::TreeHash;
 
 pub const MAX_VALIDATOR_COUNT: usize = 129;
 pub const SLOT_OFFSET: Slot = Slot::new(1);
@@ -489,9 +495,6 @@ fn decode_base_and_altair() {
 
 #[test]
 fn tree_hash_cache_linear_history() {
-    use crate::test_utils::{SeedableRng, XorShiftRng};
-    use tree_hash::TreeHash;
-
     let mut rng = XorShiftRng::from_seed([42; 16]);
 
     let mut state: BeaconState<MainnetEthSpec> =
@@ -545,3 +548,59 @@ fn tree_hash_cache_linear_history() {
     let root = state.update_tree_hash_cache().unwrap();
     assert_eq!(root.as_bytes(), &state.tree_hash_root()[..]);
 }
+
+// Check how the cache behaves when there's a distance larger than `SLOTS_PER_HISTORICAL_ROOT`
+// since its last update.
+#[test]
+fn tree_hash_cache_linear_history_long_skip() {
+    let validator_count = 128;
+    let keypairs = generate_deterministic_keypairs(validator_count);
+
+    let spec = &test_spec::<MinimalEthSpec>();
+
+    // This state has a cache that advances normally each slot.
+    let mut state: BeaconState<MinimalEthSpec> = interop_genesis_state(&keypairs, 0, spec).unwrap();
+
+    state.update_tree_hash_cache().unwrap();
+
+    // This state retains its original cache until it is updated after a long skip.
+    let mut original_cache_state = state.clone();
+    assert!(original_cache_state.tree_hash_cache().is_initialized());
+
+    // Advance the states to a slot beyond the historical state root limit, using the state root
+    // from the first state to avoid touching the original state's cache.
+    let start_slot = state.slot();
+    let target_slot = start_slot
+        .safe_add(MinimalEthSpec::slots_per_historical_root() as u64 + 1)
+        .unwrap();
+
+    let mut prev_state_root;
+    while state.slot() < target_slot {
+        prev_state_root = state.update_tree_hash_cache().unwrap();
+        per_slot_processing(&mut state, None, spec).unwrap();
+        per_slot_processing(&mut original_cache_state, Some(prev_state_root), spec).unwrap();
+    }
+
+    // The state with the original cache should still be initialized at the starting slot.
+    assert_eq!(
+        original_cache_state
+            .tree_hash_cache()
+            .initialized_slot()
+            .unwrap(),
+        start_slot
+    );
+
+    // Updating the tree hash cache should be successful despite the long skip.
+    assert_eq!(
+        original_cache_state.update_tree_hash_cache().unwrap(),
+        state.update_tree_hash_cache().unwrap()
+    );
+
+    assert_eq!(
+        original_cache_state
+            .tree_hash_cache()
+            .initialized_slot()
+            .unwrap(),
+        target_slot
+    );
+}

consensus/types/src/beacon_state/tree_hash_cache.rs

@@ -118,6 +118,13 @@ impl<T: EthSpec> BeaconTreeHashCache<T> {
     pub fn uninitialize(&mut self) {
         self.inner = None;
     }
+
+    /// Return the slot at which the cache was last updated.
+    ///
+    /// This should probably only be used during testing.
+    pub fn initialized_slot(&self) -> Option<Slot> {
+        Some(self.inner.as_ref()?.previous_state?.1)
+    }
 }
 
 #[derive(Debug, PartialEq, Clone)]
@@ -206,7 +213,8 @@ impl<T: EthSpec> BeaconTreeHashCacheInner<T> {
     /// Updates the cache and returns the tree hash root for the given `state`.
     ///
     /// The provided `state` should be a descendant of the last `state` given to this function, or
-    /// the `Self::new` function.
+    /// the `Self::new` function. If the state is more than `SLOTS_PER_HISTORICAL_ROOT` slots
+    /// after `self.previous_state` then the whole cache will be re-initialized.
     pub fn recalculate_tree_hash_root(&mut self, state: &BeaconState<T>) -> Result<Hash256, Error> {
         // If this cache has previously produced a root, ensure that it is in the state root
         // history of this state.
@@ -224,10 +232,15 @@ impl<T: EthSpec> BeaconTreeHashCacheInner<T> {
             }
 
             // If the state is newer, the previous root must be in the history of the given state.
-            if previous_slot < state.slot()
-                && *state.get_state_root(previous_slot)? != previous_root
-            {
-                return Err(Error::NonLinearTreeHashCacheHistory);
+            // If the previous slot is out of range of the `state_roots` array (indicating a long
+            // gap between the cache's last use and the current state) then we re-initialize.
+            match state.get_state_root(previous_slot) {
+                Ok(state_previous_root) if *state_previous_root == previous_root => {}
+                Ok(_) => return Err(Error::NonLinearTreeHashCacheHistory),
+                Err(Error::SlotOutOfBounds) => {
+                    *self = Self::new(state);
+                }
+                Err(e) => return Err(e),
             }
         }