Fix simulator failed runs (#2181)

## Issue Addressed

N/A

## Proposed Changes

Another attempt at fixing simulator issues for `eth1-sim`. 
The `LocalValidatorClient` here blocks till genesis has occurred. 
e4b62139d7/testing/simulator/src/local_network.rs (L145-L150)

Due to this, only the first validator(validator_0) starts before genesis. The remaining 3 vc's in the simulation start only after genesis. This was probably causing issues with missing the duties and eventually the proposal for slot 1.

This PR spawns each `LocalValidatorClient` in it's own tokio task to allow the remaining validators to start before genesis.

## Additional Info

Please provide any additional information. For example, future considerations
or information useful for reviewers.

testing/simulator/src/eth1_sim.rs

@@ -13,6 +13,7 @@ use rayon::prelude::*;
 use std::cmp::max;
 use std::net::{IpAddr, Ipv4Addr};
 use std::time::Duration;
+use tokio::time::sleep;
 use types::{Epoch, EthSpec, MainnetEthSpec};
 
 pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> {
@@ -123,7 +124,7 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> {
         /*
          * Create a new `LocalNetwork` with one beacon node.
          */
-        let network = LocalNetwork::new(context, beacon_config.clone()).await?;
+        let network = LocalNetwork::new(context.clone(), beacon_config.clone()).await?;
 
         /*
          * One by one, add beacon nodes to the network.
@@ -139,12 +140,26 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> {
         /*
          * One by one, add validators to the network.
          */
+
+        let executor = context.executor.clone();
         for (i, files) in validator_files.into_iter().enumerate() {
-            network
-                .add_validator_client(testing_validator_config(), i, files, i % 2 == 0)
-                .await?;
+            let network_1 = network.clone();
+            executor.spawn(
+                async move {
+                    println!("Adding validator client {}", i);
+                    network_1
+                        .add_validator_client(testing_validator_config(), i, files, i % 2 == 0)
+                        .await
+                        .expect("should add validator");
+                },
+                "vc",
+            );
         }
 
+        let duration_to_genesis = network.duration_to_genesis().await;
+        println!("Duration to genesis: {}", duration_to_genesis.as_secs());
+        sleep(duration_to_genesis).await;
+
         /*
          * Start the checks that ensure the network performs as expected.
          *

testing/simulator/src/local_network.rs

@@ -4,8 +4,11 @@ use node_test_rig::{
     ClientConfig, LocalBeaconNode, LocalValidatorClient, ValidatorConfig, ValidatorFiles,
 };
 use parking_lot::RwLock;
-use std::ops::Deref;
-use std::sync::Arc;
+use std::{
+    ops::Deref,
+    time::{SystemTime, UNIX_EPOCH},
+};
+use std::{sync::Arc, time::Duration};
 use types::{Epoch, EthSpec};
 
 const BOOTNODE_PORT: u16 = 42424;
@@ -122,8 +125,9 @@ impl<E: EthSpec> LocalNetwork<E> {
         validator_files: ValidatorFiles,
         invalid_first_beacon_node: bool, //to test beacon node fallbacks
     ) -> Result<(), String> {
-        let index = self.validator_clients.read().len();
-        let context = self.context.service_context(format!("validator_{}", index));
+        let context = self
+            .context
+            .service_context(format!("validator_{}", beacon_node));
         let self_1 = self.clone();
         let socket_addr = {
             let read_lock = self.beacon_nodes.read();
@@ -172,4 +176,19 @@ impl<E: EthSpec> LocalNetwork<E> {
             .map_err(|e| format!("Cannot get head: {:?}", e))
             .map(|body| body.unwrap().data.finalized.epoch)
     }
+
+    pub async fn duration_to_genesis(&self) -> Duration {
+        let nodes = self.remote_nodes().expect("Failed to get remote nodes");
+        let bootnode = nodes.first().expect("Should contain bootnode");
+        let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
+        let genesis_time = Duration::from_secs(
+            bootnode
+                .get_beacon_genesis()
+                .await
+                .unwrap()
+                .data
+                .genesis_time,
+        );
+        genesis_time - now
+    }
 }