Spadina support (v2) (#1670)

## Issue Addressed

Resolves #1651

## Description

This supercedes #1658. Great work was done by @pawanjay176, I just needed to make a change whilst he is away.

See #1658 for a description, prior reviews and approval by @michaelsproul.

## Additional info

Ignores a rustsec advisory. This is tracked in #1669.


Co-authored-by: pawan <pawandhananjay@gmail.com>

Makefile

@@ -137,7 +137,11 @@ arbitrary-fuzz:
 # Runs cargo audit (Audit Cargo.lock files for crates with security vulnerabilities reported to the RustSec Advisory Database)
 audit:
 	cargo install --force cargo-audit
-	cargo audit
+	# TODO: we should address this --ignore.
+	#
+	# Tracking issue:
+	# https://github.com/sigp/lighthouse/issues/1669
+	cargo audit --ignore RUSTSEC-2020-0043
 
 # Runs `cargo udeps` to check for unused dependencies
 udeps:

common/eth2_config/src/lib.rs

@@ -52,7 +52,6 @@ pub struct Eth2NetArchiveAndDirectory<'a> {
     pub name: &'a str,
     pub unique_id: &'a str,
     pub archive_name: &'a str,
-    pub commit: &'a str,
     pub genesis_is_known: bool,
 }
 
@@ -76,26 +75,21 @@ impl<'a> Eth2NetArchiveAndDirectory<'a> {
 
 #[macro_export]
 macro_rules! unique_id {
-    ($name: tt, $commit: tt, $genesis_is_known: tt) => {
+    ($name: tt) => {
-        concat!("testnet_", $name, "_", $commit, "_", $genesis_is_known);
+        concat!("testnet_", $name);
-    };
-
-    ($name: tt, $commit: tt) => {
-        concat!("testnet_", $name, "_", $commit, ".zip");
     };
 }
 
 macro_rules! define_net {
-    ($title: ident, $macro_title: tt, $name: tt, $commit: tt, $genesis_is_known: tt) => {
+    ($title: ident, $macro_title: tt, $name: tt, $genesis_is_known: tt) => {
         #[macro_use]
         pub mod $title {
             use super::*;
 
             pub const ETH2_NET_DIR: Eth2NetArchiveAndDirectory = Eth2NetArchiveAndDirectory {
                 name: $name,
-                unique_id: unique_id!($name, $commit, $genesis_is_known),
+                unique_id: unique_id!($name),
-                archive_name: unique_id!($name, $commit),
+                archive_name: concat!(unique_id!($name), ".zip"),
-                commit: $commit,
                 genesis_is_known: $genesis_is_known,
             };
 
@@ -104,33 +98,18 @@ macro_rules! define_net {
             #[macro_export]
             macro_rules! $macro_title {
                 ($base_dir: tt, $filename: tt) => {
-                    include_bytes!(concat!(
+                    include_bytes!(concat!($base_dir, unique_id!($name), "/", $filename))
-                        $base_dir,
-                        unique_id!($name, $commit, $genesis_is_known),
-                        "/",
-                        $filename
-                    ))
                 };
             }
         }
     };
 }
 
-define_net!(
+define_net!(altona, include_altona_file, "altona", true);
-    altona,
-    include_altona_file,
-    "altona",
-    "a94e00c1a03df851f960fcf44a79f2a6b1d29af1",
-    true
-);
 
-define_net!(
+define_net!(medalla, include_medalla_file, "medalla", true);
-    medalla,
+
-    include_medalla_file,
+define_net!(spadina, include_spadina_file, "spadina", false);
-    "medalla",
-    "09bbf2c9d108944ac934f94ec6a1d0684ca062a5",
-    true
-);
 
 #[cfg(test)]
 mod tests {

common/eth2_testnet_config/build.rs

@@ -1,67 +1,55 @@
 //! Downloads a testnet configuration from Github.
 
-use eth2_config::{altona, medalla, Eth2NetArchiveAndDirectory};
+use eth2_config::{altona, medalla, spadina, Eth2NetArchiveAndDirectory};
 use std::fs;
 use std::fs::File;
 use std::io;
 use zip::ZipArchive;
 
-const ETH2_NET_DIRS: &[Eth2NetArchiveAndDirectory<'static>] =
+const ETH2_NET_DIRS: &[Eth2NetArchiveAndDirectory<'static>] = &[
-    &[altona::ETH2_NET_DIR, medalla::ETH2_NET_DIR];
+    altona::ETH2_NET_DIR,
+    medalla::ETH2_NET_DIR,
+    spadina::ETH2_NET_DIR,
+];
 
 fn main() {
     for testnet in ETH2_NET_DIRS {
-        let testnet_dir = testnet.dir();
+        match uncompress(testnet) {
-        let archive_fullpath = testnet.archive_fullpath();
+            Ok(()) => (),
-        //no need to do anything if archives have already been uncompressed before
+            Err(e) => panic!("Failed to uncompress testnet zip file: {}", e),
-        if !testnet_dir.exists() {
-            if archive_fullpath.exists() {
-                //uncompress archive and continue
-                let archive_file = match File::open(&archive_fullpath) {
-                    Ok(f) => f,
-                    Err(e) => panic!("Problem opening archive file: {}", e),
-                };
-
-                match uncompress(archive_file) {
-                    Ok(_) => (),
-                    Err(e) => panic!(e),
-                };
-            } else {
-                panic!(
-                    "Couldn't find testnet archive at this location: {:?}",
-                    archive_fullpath
-                );
-            }
         }
     }
 }
 
-fn uncompress(archive_file: File) -> Result<(), String> {
+/// Uncompress the testnet configs archive into a testnet configs folder.
+fn uncompress(testnet: &Eth2NetArchiveAndDirectory<'static>) -> Result<(), String> {
+    let archive_file = File::open(&testnet.archive_fullpath())
+        .map_err(|e| format!("Failed to open archive file: {:?}", e))?;
+
     let mut archive =
         ZipArchive::new(archive_file).map_err(|e| format!("Error with zip file: {}", e))?;
+
+    // Create testnet dir
+    fs::create_dir_all(testnet.dir())
+        .map_err(|e| format!("Failed to create testnet directory: {:?}", e))?;
+
+    // Create empty genesis.ssz if genesis is unknown
+    if !testnet.genesis_is_known {
+        File::create(testnet.dir().join("genesis.ssz"))
+            .map_err(|e| format!("Failed to create genesis.ssz: {}", e))?;
+    }
+
     for i in 0..archive.len() {
         let mut file = archive
             .by_index(i)
             .map_err(|e| format!("Error retrieving file {} inside zip: {}", i, e))?;
 
-        let outpath = file.sanitized_name();
+        let path = testnet.dir().join(file.name());
 
-        if file.name().ends_with('/') {
+        let mut outfile = File::create(&path)
-            fs::create_dir_all(&outpath)
+            .map_err(|e| format!("Error while creating file {:?}: {}", path, e))?;
-                .map_err(|e| format!("Error creating testnet directories: {}", e))?;
+        io::copy(&mut file, &mut outfile)
-        } else {
+            .map_err(|e| format!("Error writing file {:?}: {}", path, e))?;
-            if let Some(p) = outpath.parent() {
-                if !p.exists() {
-                    fs::create_dir_all(&p)
-                        .map_err(|e| format!("Error creating testnet directories: {}", e))?;
-                }
-            }
-
-            let mut outfile = File::create(&outpath)
-                .map_err(|e| format!("Error while creating file {:?}: {}", outpath, e))?;
-            io::copy(&mut file, &mut outfile)
-                .map_err(|e| format!("Error writing file {:?}: {}", outpath, e))?;
-        }
     }
 
     Ok(())

common/eth2_testnet_config/src/lib.rs

@@ -7,7 +7,7 @@
 //!
 //! https://github.com/sigp/lighthouse/pull/605
 //!
-use eth2_config::{include_altona_file, include_medalla_file, unique_id};
+use eth2_config::{include_altona_file, include_medalla_file, include_spadina_file, unique_id};
 
 use enr::{CombinedKey, Enr};
 use ssz::{Decode, Encode};
@@ -53,8 +53,9 @@ macro_rules! define_net {
 
 const ALTONA: HardcodedNet = define_net!(altona, include_altona_file);
 const MEDALLA: HardcodedNet = define_net!(medalla, include_medalla_file);
+const SPADINA: HardcodedNet = define_net!(spadina, include_spadina_file);
 
-const HARDCODED_NETS: &[HardcodedNet] = &[ALTONA, MEDALLA];
+const HARDCODED_NETS: &[HardcodedNet] = &[ALTONA, MEDALLA, SPADINA];
 pub const DEFAULT_HARDCODED_TESTNET: &str = "medalla";
 
 /// Specifies an Eth2 testnet.

lighthouse/src/main.rs

@@ -112,7 +112,7 @@ fn main() {
                 .long("testnet")
                 .value_name("testnet")
                 .help("Name of network lighthouse will connect to")
-                .possible_values(&["medalla", "altona"])
+                .possible_values(&["medalla", "altona", "spadina"])
                 .conflicts_with("testnet-dir")
                 .takes_value(true)
                 .global(true)