Re-work deposit generation for memory efficiency

Helps ensure that variables are dropped after they're finished being
used.

beacon_node/beacon_chain/test_harness/src/beacon_chain_harness.rs

@@ -18,7 +18,7 @@ use types::*;
 mod generate_deposits;
 mod load_deposits_from_file;
 
-pub use generate_deposits::generate_deposits_with_deterministic_keypairs;
+pub use generate_deposits::{generate_deposits_from_keypairs, generate_deterministic_keypairs};
 pub use load_deposits_from_file::load_deposits_from_file;
 
 /// The beacon chain harness simulates a single beacon node with `validator_count` validators connected
@@ -62,7 +62,9 @@ impl BeaconChainHarness {
                 &deposits_path.as_path(),
             )
         } else {
-            generate_deposits_with_deterministic_keypairs(validator_count, genesis_time, &spec)
+            let keypairs = generate_deterministic_keypairs(validator_count);
+            let deposits = generate_deposits_from_keypairs(&keypairs, genesis_time, &spec);
+            (keypairs, deposits)
         };
 
         // Create the Beacon Chain

beacon_node/beacon_chain/test_harness/src/beacon_chain_harness/generate_deposits.rs

@@ -4,19 +4,15 @@ use log::debug;
 use rayon::prelude::*;
 use types::*;
 
-/// Generates `validator_count` deposits using keypairs where the secret key is the index of the
+/// Generates `validator_count` keypairs where the secret key is the index of the
 /// validator.
 ///
 /// For example, the first validator has a secret key of `int_to_bytes48(1)`, the second has
 /// `int_to_bytes48(2)` and so on. (We skip `0` as it generates a weird looking public key and is
 /// probably invalid).
-pub fn generate_deposits_with_deterministic_keypairs(
+pub fn generate_deterministic_keypairs(validator_count: usize) -> Vec<Keypair> {
-    validator_count: usize,
-    genesis_time: u64,
-    spec: &ChainSpec,
-) -> (Vec<Keypair>, Vec<Deposit>) {
     debug!(
-        "Generating {} random validator keypairs...",
+        "Generating {} deterministic validator keypairs...",
         validator_count
     );
 
@@ -31,9 +27,18 @@ pub fn generate_deposits_with_deterministic_keypairs(
         })
         .collect();
 
+    keypairs
+}
+
+/// Generates a `Deposit` for each keypairs
+pub fn generate_deposits_from_keypairs(
+    keypairs: &[Keypair],
+    genesis_time: u64,
+    spec: &ChainSpec,
+) -> Vec<Deposit> {
     debug!(
         "Generating {} validator deposits from random keypairs...",
-        validator_count
+        keypairs.len()
     );
 
     let initial_validator_deposits =
@@ -60,5 +65,5 @@ pub fn generate_deposits_with_deterministic_keypairs(
             })
             .collect();
 
-    (keypairs, initial_validator_deposits)
+    initial_validator_deposits
 }

beacon_node/beacon_chain/test_harness/src/prepare.rs

@@ -1,4 +1,6 @@
-use crate::beacon_chain_harness::generate_deposits_with_deterministic_keypairs;
+use crate::beacon_chain_harness::{
+    generate_deposits_from_keypairs, generate_deterministic_keypairs,
+};
 use clap::{value_t, ArgMatches};
 use log::debug;
 use serde_yaml;
@@ -18,17 +20,27 @@ pub fn prepare(matches: &ArgMatches, spec: &ChainSpec) {
         .value_of("output_dir")
         .expect("Output dir has a default value.");
 
-    let (keypairs, deposits) =
-        generate_deposits_with_deterministic_keypairs(validator_count, genesis_time, &spec);
-
     debug!("Created keypairs and deposits, writing to file...");
 
     fs::create_dir_all(Path::new(output_dir)).unwrap();
 
+    // Ensure that keypairs is dropped before writing deposits, this provides a big memory saving
+    // for large validator_counts.
+    let deposits = {
+        let keypairs = generate_deterministic_keypairs(validator_count);
+        write_keypairs(output_dir, &keypairs);
+        generate_deposits_from_keypairs(&keypairs, genesis_time, &spec)
+    };
+    write_deposits(output_dir, &deposits);
+}
+
+fn write_keypairs(output_dir: &str, keypairs: &[Keypair]) {
     let keypairs_path = Path::new(output_dir).join(KEYPAIRS_FILE);
     let keypairs_file = File::create(keypairs_path).unwrap();
     serde_yaml::to_writer(keypairs_file, &keypairs).unwrap();
+}
 
+fn write_deposits(output_dir: &str, deposits: &[Deposit]) {
     let deposits_path = Path::new(output_dir).join(DEPOSITS_FILE);
     let deposits_file = File::create(deposits_path).unwrap();
     serde_yaml::to_writer(deposits_file, &deposits).unwrap();