Add proto_array fork choice (#804)

* Start implementing proto_array * Add progress * Add unfinished progress * Add further progress * Add progress * Add tree filtering * Add half-finished modifications * Add refactored version * Tidy, add incomplete LmdGhost impl * Move impls in LmdGhost trait def * Remove old reduced_tree fork choice * Combine two functions in to `compute_deltas` * Start testing * Add more compute_deltas tests * Add fork choice testing * Add more fork choice testing * Add more fork choice tests * Add more testing to proto-array * Remove old tests * Modify tests * Add more tests * Add more testing * Add comments and fixes * Re-organise crate * Tidy, finish pruning tests * Add ssz encoding, other pub fns * Rename lmd_ghost > proto_array_fork_choice * Integrate proto_array into lighthouse * Add first pass at fixing filter * Clean out old comments * Add more comments * Attempt to fix prune error * Adjust TODO * Fix test compile errors * Add extra justification change check * Update cargo.lock * Fix fork choice test compile errors * Most remove ffg_update_required * Fix bug with epoch of attestation votes * Start adding new test format * Make fork choice tests declarative * Create test def concept * Move test defs into crate * Add binary, re-org crate * Shuffle files * Start adding ffg tests * Add more fork choice tests * Add fork choice JSON dumping * Add more detail to best node error * Ensure fin+just checkpoints from from same block * Rename JustificationManager * Move checkpoint manager into own file * Tidy * Add targetted logging for sneaky sync bug * Fix justified balances bug * Add cache metrics * Add metrics for log levels * Fix bug in checkpoint manager * Fix compile error in fork choice tests * Ignore duplicate blocks in fork choice * Add block to fock choice before db * Rename on_new_block fn * Fix spec inconsistency in `CheckpointManager` * Remove BlockRootTree * Remove old reduced_tree code fragment * Add API endpoint for fork choice * Add more ffg tests * Remove block_root_tree reminents * Ensure effective balances are used * Remove old debugging code, fix API fault * Add check to ensure parent block is in fork choice * Update readme dates * Fix readme * Tidy checkpoint manager * Remove fork choice yaml files from repo * Remove fork choice yaml from repo * General tidy * Address majority of Michael's comments * Tidy bin/lib business * Remove dangling file * Undo changes for rpc/handler from master * Revert "Undo changes for rpc/handler from master" This reverts commit 876edff0e4a501aafbb47113454852826dcc24e8. Co-authored-by: Age Manning <Age@AgeManning.com>
2020-01-29 15:05:00 +11:00
parent cd401147ea
commit b771bbb60c
48 changed files with 3469 additions and 2332 deletions
@@ -33,7 +33,6 @@ exit-future = "0.1.4"
 futures = "0.1.29"
 reqwest = "0.9.22"
 url = "2.1.0"
-lmd_ghost = { path = "../../eth2/lmd_ghost" }
 eth1 = { path = "../eth1" }
 genesis = { path = "../genesis" }
 environment = { path = "../../lighthouse/environment" }
@@ -4,7 +4,6 @@ use crate::Client;
 use beacon_chain::{
    builder::{BeaconChainBuilder, Witness},
    eth1_chain::CachingEth1Backend,
-    lmd_ghost::ThreadSafeReducedTree,
    slot_clock::{SlotClock, SystemTimeSlotClock},
    store::{
        migrate::{BackgroundMigrator, Migrate, NullMigrator},
@@ -21,7 +20,6 @@ use genesis::{
    generate_deterministic_keypairs, interop_genesis_state, state_from_ssz_file, Eth1GenesisService,
 };
 use lighthouse_bootstrap::Bootstrapper;
-use lmd_ghost::LmdGhost;
 use network::{NetworkConfig, NetworkMessage, Service as NetworkService};
 use slog::info;
 use ssz::Decode;
@@ -67,23 +65,14 @@ pub struct ClientBuilder<T: BeaconChainTypes> {
    eth_spec_instance: T::EthSpec,
 }

-impl<TStore, TStoreMigrator, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
+impl<TStore, TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>
    ClientBuilder<
-        Witness<
-            TStore,
-            TStoreMigrator,
-            TSlotClock,
-            TLmdGhost,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-        >,
+        Witness<TStore, TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>,
    >
 where
    TStore: Store<TEthSpec> + 'static,
    TStoreMigrator: store::Migrate<TStore, TEthSpec>,
    TSlotClock: SlotClock + Clone + 'static,
-    TLmdGhost: LmdGhost<TStore, TEthSpec> + 'static,
    TEth1Backend: Eth1ChainBackend<TEthSpec, TStore> + 'static,
    TEthSpec: EthSpec + 'static,
    TEventHandler: EventHandler<TEthSpec> + 'static,
@@ -367,17 +356,8 @@ where
    /// If type inference errors are being raised, see the comment on the definition of `Self`.
    pub fn build(
        self,
-    ) -> Client<
-        Witness<
-            TStore,
-            TStoreMigrator,
-            TSlotClock,
-            TLmdGhost,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-        >,
-    > {
+    ) -> Client<Witness<TStore, TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>>
+    {
        Client {
            beacon_chain: self.beacon_chain,
            libp2p_network: self.libp2p_network,
@@ -390,15 +370,7 @@ where

 impl<TStore, TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>
    ClientBuilder<
-        Witness<
-            TStore,
-            TStoreMigrator,
-            TSlotClock,
-            ThreadSafeReducedTree<TStore, TEthSpec>,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-        >,
+        Witness<TStore, TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>,
    >
 where
    TStore: Store<TEthSpec> + 'static,
@@ -435,13 +407,12 @@ where
    }
 }

-impl<TStore, TStoreMigrator, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec>
+impl<TStore, TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec>
    ClientBuilder<
        Witness<
            TStore,
            TStoreMigrator,
            TSlotClock,
-            TLmdGhost,
            TEth1Backend,
            TEthSpec,
            WebSocketSender<TEthSpec>,
@@ -451,7 +422,6 @@ where
    TStore: Store<TEthSpec> + 'static,
    TStoreMigrator: store::Migrate<TStore, TEthSpec>,
    TSlotClock: SlotClock + 'static,
-    TLmdGhost: LmdGhost<TStore, TEthSpec> + 'static,
    TEth1Backend: Eth1ChainBackend<TEthSpec, TStore> + 'static,
    TEthSpec: EthSpec + 'static,
 {
@@ -485,13 +455,12 @@ where
    }
 }

-impl<TStoreMigrator, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
+impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>
    ClientBuilder<
        Witness<
            DiskStore<TEthSpec>,
            TStoreMigrator,
            TSlotClock,
-            TLmdGhost,
            TEth1Backend,
            TEthSpec,
            TEventHandler,
@@ -500,7 +469,6 @@ impl<TStoreMigrator, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandle
 where
    TSlotClock: SlotClock + 'static,
    TStoreMigrator: store::Migrate<DiskStore<TEthSpec>, TEthSpec> + 'static,
-    TLmdGhost: LmdGhost<DiskStore<TEthSpec>, TEthSpec> + 'static,
    TEth1Backend: Eth1ChainBackend<TEthSpec, DiskStore<TEthSpec>> + 'static,
    TEthSpec: EthSpec + 'static,
    TEventHandler: EventHandler<TEthSpec> + 'static,
@@ -535,13 +503,12 @@ where
    }
 }

-impl<TStoreMigrator, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
+impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>
    ClientBuilder<
        Witness<
            SimpleDiskStore<TEthSpec>,
            TStoreMigrator,
            TSlotClock,
-            TLmdGhost,
            TEth1Backend,
            TEthSpec,
            TEventHandler,
@@ -550,7 +517,6 @@ impl<TStoreMigrator, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandle
 where
    TSlotClock: SlotClock + 'static,
    TStoreMigrator: store::Migrate<SimpleDiskStore<TEthSpec>, TEthSpec> + 'static,
-    TLmdGhost: LmdGhost<SimpleDiskStore<TEthSpec>, TEthSpec> + 'static,
    TEth1Backend: Eth1ChainBackend<TEthSpec, SimpleDiskStore<TEthSpec>> + 'static,
    TEthSpec: EthSpec + 'static,
    TEventHandler: EventHandler<TEthSpec> + 'static,
@@ -564,13 +530,12 @@ where
    }
 }

-impl<TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
+impl<TSlotClock, TEth1Backend, TEthSpec, TEventHandler>
    ClientBuilder<
        Witness<
            MemoryStore<TEthSpec>,
            NullMigrator,
            TSlotClock,
-            TLmdGhost,
            TEth1Backend,
            TEthSpec,
            TEventHandler,
@@ -578,7 +543,6 @@ impl<TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
    >
 where
    TSlotClock: SlotClock + 'static,
-    TLmdGhost: LmdGhost<MemoryStore<TEthSpec>, TEthSpec> + 'static,
    TEth1Backend: Eth1ChainBackend<TEthSpec, MemoryStore<TEthSpec>> + 'static,
    TEthSpec: EthSpec + 'static,
    TEventHandler: EventHandler<TEthSpec> + 'static,
@@ -594,13 +558,12 @@ where
    }
 }

-impl<TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
+impl<TSlotClock, TEth1Backend, TEthSpec, TEventHandler>
    ClientBuilder<
        Witness<
            DiskStore<TEthSpec>,
            BackgroundMigrator<TEthSpec>,
            TSlotClock,
-            TLmdGhost,
            TEth1Backend,
            TEthSpec,
            TEventHandler,
@@ -608,7 +571,6 @@ impl<TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
    >
 where
    TSlotClock: SlotClock + 'static,
-    TLmdGhost: LmdGhost<DiskStore<TEthSpec>, TEthSpec> + 'static,
    TEth1Backend: Eth1ChainBackend<TEthSpec, DiskStore<TEthSpec>> + 'static,
    TEthSpec: EthSpec + 'static,
    TEventHandler: EventHandler<TEthSpec> + 'static,
@@ -622,13 +584,12 @@ where
    }
 }

-impl<TStore, TStoreMigrator, TSlotClock, TLmdGhost, TEthSpec, TEventHandler>
+impl<TStore, TStoreMigrator, TSlotClock, TEthSpec, TEventHandler>
    ClientBuilder<
        Witness<
            TStore,
            TStoreMigrator,
            TSlotClock,
-            TLmdGhost,
            CachingEth1Backend<TEthSpec, TStore>,
            TEthSpec,
            TEventHandler,
@@ -638,7 +599,6 @@ where
    TStore: Store<TEthSpec> + 'static,
    TStoreMigrator: store::Migrate<TStore, TEthSpec>,
    TSlotClock: SlotClock + 'static,
-    TLmdGhost: LmdGhost<TStore, TEthSpec> + 'static,
    TEthSpec: EthSpec + 'static,
    TEventHandler: EventHandler<TEthSpec> + 'static,
 {
@@ -724,22 +684,13 @@ where
    }
 }

-impl<TStore, TStoreMigrator, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
+impl<TStore, TStoreMigrator, TEth1Backend, TEthSpec, TEventHandler>
    ClientBuilder<
-        Witness<
-            TStore,
-            TStoreMigrator,
-            SystemTimeSlotClock,
-            TLmdGhost,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-        >,
+        Witness<TStore, TStoreMigrator, SystemTimeSlotClock, TEth1Backend, TEthSpec, TEventHandler>,
    >
 where
    TStore: Store<TEthSpec> + 'static,
    TStoreMigrator: store::Migrate<TStore, TEthSpec>,
-    TLmdGhost: LmdGhost<TStore, TEthSpec> + 'static,
    TEth1Backend: Eth1ChainBackend<TEthSpec, TStore> + 'static,
    TEthSpec: EthSpec + 'static,
    TEventHandler: EventHandler<TEthSpec> + 'static,