Adds simulator for syncing (#758)

* Add CLI for beacon_chain_sim

* Rename beacon-chain-sim to simulator

* Fix simulator workflow

* Push Cargo.lock

* WIP syncing simulator

* Add cli args

* Remove eth1 stuff and deposits

* Add syncing strategy simulations

* Successful one node sync

* Clean up

* Rename to avoid confusion

* add command line args

* fix cargo fmt issues

* Add additional syncing strategies

* Run all syncing strategies one after other; add comments

* Improve cli argument parsing

* Change `end_after_checks` default to true

* Small modifications to syncing-sim

* Add `strategy` cli argument

* Documented defaults in cli help

Co-authored-by: mkinney <mike.kinney@gmail.com>
Co-authored-by: Age Manning <Age@AgeManning.com>

.github/workflows/test-suite.yml

@@ -57,4 +57,4 @@ jobs:
     - name: Install ganache-cli
       run: sudo npm install -g ganache-cli
     - name: Run the beacon chain sim
-      run: cargo run --release --bin beacon_chain_sim
+      run: cargo run --release --bin simulator beacon-chain-sim

Cargo.lock

@@ -240,20 +240,6 @@ dependencies = [
  "websocket_server 0.1.0",
 ]
 
-[[package]]
-name = "beacon_chain_sim"
-version = "0.1.0"
-dependencies = [
- "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "eth1_test_rig 0.1.0",
- "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
- "node_test_rig 0.1.0",
- "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)",
- "types 0.1.0",
- "validator_client 0.1.0",
-]
-
 [[package]]
 name = "beacon_node"
 version = "0.1.0"
@@ -3744,6 +3730,21 @@ dependencies = [
  "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "simulator"
+version = "0.1.0"
+dependencies = [
+ "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "eth1_test_rig 0.1.0",
+ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
+ "node_test_rig 0.1.0",
+ "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)",
+ "types 0.1.0",
+ "validator_client 0.1.0",
+]
+
 [[package]]
 name = "slab"
 version = "0.3.0"

Cargo.toml

@@ -37,7 +37,7 @@ members = [
     "beacon_node/eth1",
     "beacon_node/beacon_chain",
     "beacon_node/websocket_server",
-    "tests/beacon_chain_sim",
+    "tests/simulator",
     "tests/ef_tests",
     "tests/eth1_test_rig",
     "tests/node_test_rig",

tests/simulator/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "beacon_chain_sim"
+name = "simulator"
 version = "0.1.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2018"
@@ -15,3 +15,4 @@ futures = "0.1.29"
 tokio = "0.1.22"
 eth1_test_rig = { path = "../eth1_test_rig" }
 env_logger = "0.7.1"
+clap = "2.33.0"

tests/simulator/src/checks.rs

@@ -41,7 +41,7 @@ pub fn verify_first_finalization<E: EthSpec>(
 }
 
 /// Delays for `epochs`, plus half a slot extra.
-fn epoch_delay(
+pub fn epoch_delay(
     epochs: Epoch,
     slot_duration: Duration,
     slots_per_epoch: u64,
@@ -60,7 +60,7 @@ fn slot_delay(slots: Slot, slot_duration: Duration) -> impl Future<Item = (), Er
 
 /// Verifies that all beacon nodes in the given network have a head state that has a finalized
 /// epoch of `epoch`.
-fn verify_all_finalized_at<E: EthSpec>(
+pub fn verify_all_finalized_at<E: EthSpec>(
     network: LocalNetwork<E>,
     epoch: Epoch,
 ) -> impl Future<Item = (), Error = String> {

tests/simulator/src/cli.rs

@@ -0,0 +1,73 @@
+use clap::{App, Arg, SubCommand};
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new("simulator")
+        .version(crate_version!())
+        .author("Sigma Prime <contact@sigmaprime.io>")
+        .about("Options for interacting with simulator")
+        .subcommand(
+            SubCommand::with_name("beacon-chain-sim")
+            .about(
+                "Lighthouse Beacon Chain Simulator creates `n` beacon node and validator clients, \
+                    each with `v` validators. A deposit contract is deployed at the start of the \
+                    simulation using a local `ganache-cli` instance (you must have `ganache-cli` \
+                    installed and avaliable on your path). All beacon nodes independently listen \
+                    for genesis from the deposit contract, then start operating. \
+
+                    As the simulation runs, there are checks made to ensure that all components \
+                    are running correctly. If any of these checks fail, the simulation will \
+                    exit immediately.",
+                    )
+                    .arg(Arg::with_name("nodes")
+                        .short("n")
+                        .long("nodes")
+                        .takes_value(true)
+                        .help("Number of beacon nodes (default 4)"))
+                    .arg(Arg::with_name("validators_per_node")
+                        .short("v")
+                        .long("validators_per_node")
+                        .takes_value(true)
+                        .help("Number of validators (default 20)"))
+                    .arg(Arg::with_name("speed_up_factor")
+                        .short("s")
+                        .long("speed_up_factor")
+                        .takes_value(true)
+                        .help("Speed up factor (default 4)"))
+                    .arg(Arg::with_name("end_after_checks")
+                        .short("e")
+                        .long("end_after_checks")
+                        .takes_value(false)
+                        .help("End after checks (default true)"))
+        )
+        .subcommand(
+            SubCommand::with_name("syncing-sim")
+                .about("Run the syncing simulation")
+                .arg(
+                    Arg::with_name("speedup")
+                        .short("s")
+                        .long("speedup")
+                        .takes_value(true)
+                        .help("Speed up factor for eth1 blocks and slot production (default 15)"),
+                )
+                .arg(
+                    Arg::with_name("initial_delay")
+                        .short("i")
+                        .long("initial_delay")
+                        .takes_value(true)
+                        .help("Epoch delay for new beacon node to start syncing (default 50)"),
+                )
+                .arg(
+                    Arg::with_name("sync_delay")
+                        .long("sync_delay")
+                        .takes_value(true)
+                        .help("Epoch delay for newly added beacon nodes get synced (default 10)"),
+                )
+                .arg(
+                    Arg::with_name("strategy")
+                        .long("strategy")
+                        .takes_value(true)
+                        .possible_values(&["one-node", "two-nodes", "mixed", "all"])
+                        .help("Sync strategy to run. (default all)"),
+                ),
+        )
+}

tests/simulator/src/local_network.rs

@@ -6,7 +6,7 @@ use node_test_rig::{
 use parking_lot::RwLock;
 use std::ops::Deref;
 use std::sync::Arc;
-use types::EthSpec;
+use types::{Epoch, EthSpec};
 
 const BOOTNODE_PORT: u16 = 42424;
 
@@ -82,7 +82,7 @@ impl<E: EthSpec> LocalNetwork<E> {
         mut beacon_config: ClientConfig,
     ) -> impl Future<Item = (), Error = String> {
         let self_1 = self.clone();
-
+        println!("Adding beacon node..");
         self.beacon_nodes
             .read()
             .first()
@@ -154,4 +154,16 @@ impl<E: EthSpec> LocalNetwork<E> {
             .map(|beacon_node| beacon_node.remote_node())
             .collect()
     }
+
+    /// Return current epoch of bootnode.
+    pub fn bootnode_epoch(&self) -> impl Future<Item = Epoch, Error = String> {
+        let nodes = self.remote_nodes().expect("Failed to get remote nodes");
+        let bootnode = nodes.first().expect("Should contain bootnode");
+        bootnode
+            .http
+            .beacon()
+            .get_head()
+            .map_err(|e| format!("Cannot get head: {:?}", e))
+            .map(|head| head.finalized_slot.epoch(E::slots_per_epoch()))
+    }
 }

tests/simulator/src/main.rs

@@ -6,21 +6,23 @@
 //! As the simulation runs, there are checks made to ensure that all components are running
 //! correctly. If any of these checks fail, the simulation will exit immediately.
 //!
-//! By default, the simulation will end as soon as all checks have finished. It may be configured
-//! to run indefinitely by setting `end_after_checks = false`.
-//!
 //! ## Future works
 //!
 //! Presently all the beacon nodes and validator clients all log to stdout. Additionally, the
 //! simulation uses `println` to communicate some info. It might be nice if the nodes logged to
 //! easy-to-find files and stdout only contained info from the simulation.
 //!
-//! It would also be nice to add a CLI using `clap` so that the variables in `main()` can be
+
-//! changed without a recompile.
+#[macro_use]
+extern crate clap;
 
 mod checks;
+mod cli;
 mod local_network;
+mod sync_sim;
 
+use clap::ArgMatches;
+use cli::cli_app;
 use env_logger::{Builder, Env};
 use eth1_test_rig::GanacheEth1Instance;
 use futures::{future, stream, Future, Stream};
@@ -29,6 +31,7 @@ use node_test_rig::{
     environment::EnvironmentBuilder, testing_client_config, ClientGenesis, ValidatorConfig,
 };
 use std::time::{Duration, Instant};
+use sync_sim::*;
 use tokio::timer::Interval;
 use types::MinimalEthSpec;
 
@@ -38,30 +41,161 @@ fn main() {
     // Debugging output for libp2p and external crates.
     Builder::from_env(Env::default()).init();
 
-    let nodes = 4;
+    let matches = cli_app().get_matches();
-    let validators_per_node = 20;
+    match matches.subcommand() {
+        ("beacon-chain-sim", Some(matches)) => match run_beacon_chain_sim(matches) {
+            Ok(()) => println!("Simulation exited successfully"),
+            Err(e) => {
+                eprintln!("Simulation exited with error: {}", e);
+                std::process::exit(1)
+            }
+        },
+        ("syncing-sim", Some(matches)) => match run_syncing_sim(matches) {
+            Ok(()) => println!("Simulation exited successfully"),
+            Err(e) => {
+                eprintln!("Simulation exited with error: {}", e);
+                std::process::exit(1)
+            }
+        },
+        _ => {
+            eprintln!("Invalid subcommand. Use --help to see available options");
+            std::process::exit(1)
+        }
+    }
+}
+
+fn run_beacon_chain_sim(matches: &ArgMatches) -> Result<(), String> {
+    let nodes = value_t!(matches, "nodes", usize).unwrap_or(4);
+    let validators_per_node = value_t!(matches, "validators_per_node", usize).unwrap_or(20);
+    let speed_up_factor = value_t!(matches, "nodes", u64).unwrap_or(4);
+    let mut end_after_checks = true;
+    if matches.is_present("end_after_checks") {
+        end_after_checks = false;
+    }
+
+    println!("Beacon Chain Simulator:");
+    println!(" nodes:{}", nodes);
+    println!(" validators_per_node:{}", validators_per_node);
+    println!(" end_after_checks:{}", end_after_checks);
+
     let log_level = "debug";
     let log_format = None;
-    let speed_up_factor = 4;
-    let end_after_checks = true;
 
-    match async_sim(
+    beacon_chain_sim(
         nodes,
         validators_per_node,
         speed_up_factor,
         log_level,
         log_format,
         end_after_checks,
-    ) {
+    )
-        Ok(()) => println!("Simulation exited successfully"),
-        Err(e) => {
-            eprintln!("Simulation exited with error: {}", e);
-            std::process::exit(1)
-        }
-    }
 }
 
-fn async_sim(
+fn run_syncing_sim(matches: &ArgMatches) -> Result<(), String> {
+    let initial_delay = value_t!(matches, "initial_delay", u64).unwrap_or(50);
+    let sync_delay = value_t!(matches, "sync_delay", u64).unwrap_or(10);
+    let speed_up_factor = value_t!(matches, "speedup", u64).unwrap_or(15);
+    let strategy = value_t!(matches, "strategy", String).unwrap_or("all".into());
+
+    println!("Syncing Simulator:");
+    println!(" initial_delay:{}", initial_delay);
+    println!(" sync delay:{}", sync_delay);
+    println!(" speed up factor:{}", speed_up_factor);
+    println!(" strategy:{}", strategy);
+
+    let log_level = "debug";
+    let log_format = None;
+
+    syncing_sim(
+        speed_up_factor,
+        initial_delay,
+        sync_delay,
+        strategy,
+        log_level,
+        log_format,
+    )
+}
+
+fn syncing_sim(
+    speed_up_factor: u64,
+    initial_delay: u64,
+    sync_delay: u64,
+    strategy: String,
+    log_level: &str,
+    log_format: Option<&str>,
+) -> Result<(), String> {
+    let mut env = EnvironmentBuilder::minimal()
+        .async_logger(log_level, log_format)?
+        .multi_threaded_tokio_runtime()?
+        .build()?;
+
+    let spec = &mut env.eth2_config.spec;
+    let end_after_checks = true;
+
+    spec.milliseconds_per_slot = spec.milliseconds_per_slot / speed_up_factor;
+    spec.min_genesis_time = 0;
+    spec.min_genesis_active_validator_count = 16;
+
+    let slot_duration = Duration::from_millis(spec.milliseconds_per_slot);
+
+    let context = env.core_context();
+    let beacon_config = testing_client_config();
+    let num_validators = 8;
+    let future = LocalNetwork::new(context, beacon_config.clone())
+        /*
+         * Add a validator client which handles all validators from the genesis state.
+         */
+        .and_then(move |network| {
+            network
+                .add_validator_client(ValidatorConfig::default(), 0, (0..num_validators).collect())
+                .map(|_| network)
+        })
+        /*
+         * Start the processes that will run checks on the network as it runs.
+         */
+        .and_then(move |network| {
+            // The `final_future` either completes immediately or never completes, depending on the value
+            // of `end_after_checks`.
+            let final_future: Box<dyn Future<Item = (), Error = String> + Send> =
+                if end_after_checks {
+                    Box::new(future::ok(()).map_err(|()| "".to_string()))
+                } else {
+                    Box::new(future::empty().map_err(|()| "".to_string()))
+                };
+
+            future::ok(())
+                // Check all syncing strategies one after other.
+                .join(pick_strategy(
+                    &strategy,
+                    network.clone(),
+                    beacon_config.clone(),
+                    slot_duration,
+                    initial_delay,
+                    sync_delay,
+                ))
+                .join(final_future)
+                .map(|_| network)
+        })
+        /*
+         * End the simulation by dropping the network. This will kill all running beacon nodes and
+         * validator clients.
+         */
+        .map(|network| {
+            println!(
+                "Simulation complete. Finished with {} beacon nodes and {} validator clients",
+                network.beacon_node_count(),
+                network.validator_client_count()
+            );
+
+            // Be explicit about dropping the network, as this kills all the nodes. This ensures
+            // all the checks have adequate time to pass.
+            drop(network)
+        });
+
+    env.runtime().block_on(future)
+}
+
+fn beacon_chain_sim(
     node_count: usize,
     validators_per_node: usize,
     speed_up_factor: u64,

tests/simulator/src/sync_sim.rs

@@ -0,0 +1,201 @@
+use crate::checks::{epoch_delay, verify_all_finalized_at};
+use crate::local_network::LocalNetwork;
+use futures::{Future, IntoFuture};
+use node_test_rig::ClientConfig;
+use std::time::Duration;
+use types::{Epoch, EthSpec};
+
+pub fn pick_strategy<E: EthSpec>(
+    strategy: &str,
+    network: LocalNetwork<E>,
+    beacon_config: ClientConfig,
+    slot_duration: Duration,
+    initial_delay: u64,
+    sync_delay: u64,
+) -> Box<dyn Future<Item = (), Error = String> + Send + 'static> {
+    match strategy {
+        "one-node" => Box::new(verify_one_node_sync(
+            network,
+            beacon_config,
+            slot_duration,
+            initial_delay,
+            sync_delay,
+        )),
+        "two-nodes" => Box::new(verify_two_nodes_sync(
+            network,
+            beacon_config,
+            slot_duration,
+            initial_delay,
+            sync_delay,
+        )),
+        "mixed" => Box::new(verify_in_between_sync(
+            network,
+            beacon_config,
+            slot_duration,
+            initial_delay,
+            sync_delay,
+        )),
+        "all" => Box::new(verify_syncing(
+            network,
+            beacon_config,
+            slot_duration,
+            initial_delay,
+            sync_delay,
+        )),
+        _ => Box::new(Err("Invalid strategy".into()).into_future()),
+    }
+}
+
+/// Verify one node added after `initial_delay` epochs is in sync
+/// after `sync_delay` epochs.
+pub fn verify_one_node_sync<E: EthSpec>(
+    network: LocalNetwork<E>,
+    beacon_config: ClientConfig,
+    slot_duration: Duration,
+    initial_delay: u64,
+    sync_delay: u64,
+) -> impl Future<Item = (), Error = String> {
+    // Delay for `initial_delay` epochs before adding another node to start syncing
+    epoch_delay(
+        Epoch::new(initial_delay),
+        slot_duration,
+        E::slots_per_epoch(),
+    )
+    .and_then(move |_| {
+        // Add a beacon node
+        network.add_beacon_node(beacon_config).map(|_| network)
+    })
+    .and_then(move |network| {
+        // Delay for `sync_delay` epochs before verifying synced state.
+        epoch_delay(Epoch::new(sync_delay), slot_duration, E::slots_per_epoch()).map(|_| network)
+    })
+    .and_then(move |network| network.bootnode_epoch().map(|e| (e, network)))
+    .and_then(move |(epoch, network)| {
+        verify_all_finalized_at(network, epoch).map_err(|e| format!("One node sync error: {}", e))
+    })
+}
+
+/// Verify two nodes added after `initial_delay` epochs are in sync
+/// after `sync_delay` epochs.
+pub fn verify_two_nodes_sync<E: EthSpec>(
+    network: LocalNetwork<E>,
+    beacon_config: ClientConfig,
+    slot_duration: Duration,
+    initial_delay: u64,
+    sync_delay: u64,
+) -> impl Future<Item = (), Error = String> {
+    // Delay for `initial_delay` epochs before adding another node to start syncing
+    epoch_delay(
+        Epoch::new(initial_delay),
+        slot_duration,
+        E::slots_per_epoch(),
+    )
+    .and_then(move |_| {
+        // Add beacon nodes
+        network
+            .add_beacon_node(beacon_config.clone())
+            .join(network.add_beacon_node(beacon_config.clone()))
+            .map(|_| network)
+    })
+    .and_then(move |network| {
+        // Delay for `sync_delay` epochs before verifying synced state.
+        epoch_delay(Epoch::new(sync_delay), slot_duration, E::slots_per_epoch()).map(|_| network)
+    })
+    .and_then(move |network| network.bootnode_epoch().map(|e| (e, network)))
+    .and_then(move |(epoch, network)| {
+        verify_all_finalized_at(network, epoch).map_err(|e| format!("Two node sync error: {}", e))
+    })
+}
+
+/// Add 2 syncing nodes after `initial_delay` epochs,
+/// Add another node after `sync_delay - 5` epochs and verify all are
+/// in sync after `sync_delay + 5` epochs.
+pub fn verify_in_between_sync<E: EthSpec>(
+    network: LocalNetwork<E>,
+    beacon_config: ClientConfig,
+    slot_duration: Duration,
+    initial_delay: u64,
+    sync_delay: u64,
+) -> impl Future<Item = (), Error = String> {
+    // Delay for `initial_delay` epochs before adding another node to start syncing
+    let config1 = beacon_config.clone();
+    epoch_delay(
+        Epoch::new(initial_delay),
+        slot_duration,
+        E::slots_per_epoch(),
+    )
+    .and_then(move |_| {
+        // Add a beacon node
+        network
+            .add_beacon_node(beacon_config.clone())
+            .join(network.add_beacon_node(beacon_config.clone()))
+            .map(|_| network)
+    })
+    .and_then(move |network| {
+        // Delay before adding additional syncing nodes.
+        epoch_delay(
+            Epoch::new(sync_delay - 5),
+            slot_duration,
+            E::slots_per_epoch(),
+        )
+        .map(|_| network)
+    })
+    .and_then(move |network| {
+        // Add a beacon node
+        network.add_beacon_node(config1.clone()).map(|_| network)
+    })
+    .and_then(move |network| {
+        // Delay for `sync_delay` epochs before verifying synced state.
+        epoch_delay(
+            Epoch::new(sync_delay + 5),
+            slot_duration,
+            E::slots_per_epoch(),
+        )
+        .map(|_| network)
+    })
+    .and_then(move |network| network.bootnode_epoch().map(|e| (e, network)))
+    .and_then(move |(epoch, network)| {
+        verify_all_finalized_at(network, epoch).map_err(|e| format!("In between sync error: {}", e))
+    })
+}
+
+/// Run syncing strategies one after other.
+pub fn verify_syncing<E: EthSpec>(
+    network: LocalNetwork<E>,
+    beacon_config: ClientConfig,
+    slot_duration: Duration,
+    initial_delay: u64,
+    sync_delay: u64,
+) -> impl Future<Item = (), Error = String> {
+    verify_one_node_sync(
+        network.clone(),
+        beacon_config.clone(),
+        slot_duration,
+        initial_delay,
+        sync_delay,
+    )
+    .map(|_| println!("Completed one node sync"))
+    .and_then(move |_| {
+        verify_two_nodes_sync(
+            network.clone(),
+            beacon_config.clone(),
+            slot_duration,
+            initial_delay,
+            sync_delay,
+        )
+        .map(|_| {
+            println!("Completed two node sync");
+            (network, beacon_config)
+        })
+    })
+    .and_then(move |(network, beacon_config)| {
+        verify_in_between_sync(
+            network,
+            beacon_config,
+            slot_duration,
+            initial_delay,
+            sync_delay,
+        )
+        .map(|_| println!("Completed in between sync"))
+    })
+}