Improve ergonomics of adding a new network config (#2489)

## Issue Addressed

NA

## Proposed Changes

This PR adds some more fancy macro magic to make it easier to add a new built-in (aka "baked-in") testnet config to the `lighthouse` binary.

Previously, a user needed to modify several files and repeat themselves several times. Now, they only need to add a single definition in the `eth2_config` crate. No repetition 🎉

Cargo.lock

@@ -1846,6 +1846,7 @@ dependencies = [
 name = "eth2_config"
 version = "0.2.0"
 dependencies = [
+ "paste",
  "serde",
  "serde_derive",
  "types",
@@ -4638,6 +4639,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "paste"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "acbf547ad0c65e31259204bd90935776d1c693cec2f4ff7abb7a1bbbd40dfe58"
+
 [[package]]
 name = "pbkdf2"
 version = "0.4.0"

common/eth2_config/Cargo.toml

@@ -8,3 +8,4 @@ edition = "2018"
 serde = "1.0.116"
 serde_derive = "1.0.116"
 types = { path = "../../consensus/types" }
+paste = "1.0.5"

common/eth2_config/src/lib.rs

@@ -1,7 +1,16 @@
+//! This crate primarily exists to serve the `common/eth2_network_configs` crate, by providing the
+//! canonical list of built-in-networks and some tooling to help include those configurations in the
+//! `lighthouse` binary.
+//!
+//! It also provides some additional structs which are useful to other components of `lighthouse`
+//! (e.g., `Eth2Config`).
+
 use std::env;
 use std::path::PathBuf;
 use types::{ChainSpec, EthSpecId};
 
+pub use paste::paste;
+
 // A macro is used to define this constant so it can be used with `include_bytes!`.
 #[macro_export]
 macro_rules! predefined_networks_dir {
@@ -73,40 +82,152 @@ impl<'a> Eth2NetArchiveAndDirectory<'a> {
     }
 }
 
-macro_rules! define_net {
+/// Indicates that the `genesis.ssz.zip` file is present on the filesystem. This means that the
-    ($title: ident, $macro_title: tt, $name: tt, $genesis_is_known: tt) => {
+/// deposit ceremony has concluded and the final genesis `BeaconState` is known.
-        #[macro_use]
+const GENESIS_STATE_IS_KNOWN: bool = true;
-        pub mod $title {
-            use super::*;
 
-            pub const ETH2_NET_DIR: Eth2NetArchiveAndDirectory = Eth2NetArchiveAndDirectory {
+#[derive(Copy, Clone, Debug, PartialEq)]
-                name: $name,
+pub struct HardcodedNet {
-                unique_id: $name,
+    pub name: &'static str,
-                genesis_is_known: $genesis_is_known,
+    pub genesis_is_known: bool,
-            };
+    pub config: &'static [u8],
+    pub deploy_block: &'static [u8],
+    pub boot_enr: &'static [u8],
+    pub genesis_state_bytes: &'static [u8],
+}
 
-            // A wrapper around `std::include_bytes` which includes a file from a specific network
+/// Defines an `Eth2NetArchiveAndDirectory` for some network.
-            // directory. Used by upstream crates to import files at compile time.
+///
-            #[macro_export]
+/// It also defines a `include_<title>_file!` macro which provides a wrapper around
-            macro_rules! $macro_title {
+/// `std::include_bytes`, allowing the inclusion of bytes from the specific testnet directory.
-                ($base_dir: tt, $filename: tt) => {
+macro_rules! define_archive {
-                    include_bytes!(concat!(
+    ($name_ident: ident, $name_str: tt, $genesis_is_known: ident) => {
-                        $base_dir,
+        paste! {
-                        "/",
+            #[macro_use]
-                        predefined_networks_dir!(),
+            pub mod $name_ident {
-                        "/",
+                use super::*;
-                        $name,
+
-                        "/",
+                pub const ETH2_NET_DIR: Eth2NetArchiveAndDirectory = Eth2NetArchiveAndDirectory {
-                        $filename
+                    name: $name_str,
-                    ))
+                    unique_id: $name_str,
+                    genesis_is_known: $genesis_is_known,
                 };
+
+                /// A wrapper around `std::include_bytes` which includes a file from a specific network
+                /// directory. Used by upstream crates to import files at compile time.
+                #[macro_export]
+                macro_rules! [<include_ $name_ident _file>] {
+                    ($this_crate: ident, $base_dir: tt, $filename: tt) => {
+                        include_bytes!(concat!(
+                            $base_dir,
+                            "/",
+                            $this_crate::predefined_networks_dir!(),
+                            "/",
+                            $name_str,
+                            "/",
+                            $filename
+                        ))
+                    };
+                }
             }
         }
     };
 }
 
-define_net!(pyrmont, include_pyrmont_file, "pyrmont", true);
+/// Creates a `HardcodedNet` definition for some network.
+#[macro_export]
+macro_rules! define_net {
+    ($this_crate: ident, $mod: ident, $include_file: tt) => {{
+        use $this_crate::$mod::ETH2_NET_DIR;
 
-define_net!(mainnet, include_mainnet_file, "mainnet", true);
+        $this_crate::HardcodedNet {
+            name: ETH2_NET_DIR.name,
+            genesis_is_known: ETH2_NET_DIR.genesis_is_known,
+            config: $this_crate::$include_file!($this_crate, "../", "config.yaml"),
+            deploy_block: $this_crate::$include_file!($this_crate, "../", "deploy_block.txt"),
+            boot_enr: $this_crate::$include_file!($this_crate, "../", "boot_enr.yaml"),
+            genesis_state_bytes: $this_crate::$include_file!($this_crate, "../", "genesis.ssz"),
+        }
+    }};
+}
 
-define_net!(prater, include_prater_file, "prater", true);
+/// Calls `define_net` on a list of networks, and then defines two more lists:
+///
+/// - `HARDCODED_NETS`: a list of all the networks defined by this macro.
+/// - `HARDCODED_NET_NAMES`: a list of the *names* of the networks defined by this macro.
+#[macro_export]
+macro_rules! define_nets {
+    ($this_crate: ident, $($name_ident: ident, $name_str: tt,)+) => {
+        $this_crate::paste! {
+            $(
+            const [<$name_ident:upper>]: $this_crate::HardcodedNet = $this_crate::define_net!($this_crate, $name_ident, [<include_ $name_ident _file>]);
+            )+
+            const HARDCODED_NETS: &[$this_crate::HardcodedNet] = &[$([<$name_ident:upper>],)+];
+            pub const HARDCODED_NET_NAMES: &[&'static str] = &[$($name_str,)+];
+        }
+    };
+}
+
+/// The canonical macro for defining built-in network configurations.
+///
+/// This macro will provide:
+///
+/// - An `Eth2NetArchiveAndDirectory` for each network.
+/// - `ETH2_NET_DIRS`: a list of all the above `Eth2NetArchiveAndDirectory`.
+/// - The `instantiate_hardcoded_nets` macro (see its documentation).
+///
+/// ## Design Justification
+///
+/// Ultimately, this macro serves as a single list of all the networks. The reason it is structured
+/// in such a complex web-of-macros way is because two requirements of built-in (hard-coded) networks:
+///
+/// 1. We must use `std::include_bytes!` to "bake" arbitrary bytes (genesis states, etc) into the binary.
+/// 2. We must use a `build.rs` script to decompress the genesis state from a zip file, before we
+///    can include those bytes.
+///
+/// Because of these two constraints, we must first define all of the networks and the paths to
+/// their files in this crate. Then, we must use another crate (`eth2_network_configs`) to run a
+/// `build.rs` which will unzip the genesis states. Then, that `eth2_network_configs` crate can
+/// perform the final step of using `std::include_bytes` to bake the files (bytes) into the binary.
+macro_rules! define_hardcoded_nets {
+    ($(($name_ident: ident, $name_str: tt, $genesis_is_known: ident)),+) => {
+        $(
+        define_archive!($name_ident, $name_str, $genesis_is_known);
+        )+
+
+        pub const ETH2_NET_DIRS: &[Eth2NetArchiveAndDirectory<'static>] = &[$($name_ident::ETH2_NET_DIR,)+];
+
+        /// This macro is designed to be called by an external crate. When called, it will
+        /// define in that external crate:
+        ///
+        /// - A `HardcodedNet` for each network.
+        /// - `HARDCODED_NETS`: a list of all the above `HardcodedNet`.
+        /// - `HARDCODED_NET_NAMES`: a list of all the names of the above `HardcodedNet` (as `&str`).
+        #[macro_export]
+        macro_rules! instantiate_hardcoded_nets {
+            ($this_crate: ident) => {
+                $this_crate::define_nets!($this_crate, $($name_ident, $name_str,)+);
+            }
+        }
+    };
+}
+
+// Add a new "built-in" network by adding it to the list below.
+//
+// The last entry must not end with a comma, otherwise compilation will fail.
+//
+// This is the canonical place for defining the built-in network configurations that are present in
+// the `common/eth2_network_config/built_in_network_configs` directory.
+//
+// Each net is defined as a three-tuple:
+//
+// 0. The name of the testnet as an "ident" (i.e. something that can be a Rust variable name).
+// 1. The human-friendly name of the testnet (i.e. usually with "-" instead of "_").
+// 2. A bool indicating if the genesis state is known and present as a `genesis.ssz.zip`.
+//
+// The directory containing the testnet files should match the human-friendly name (element 1).
+define_hardcoded_nets!(
+    (mainnet, "mainnet", GENESIS_STATE_IS_KNOWN),
+    (pyrmont, "pyrmont", GENESIS_STATE_IS_KNOWN),
+    (prater, "prater", GENESIS_STATE_IS_KNOWN)
+);

common/eth2_network_config/build.rs

@@ -1,15 +1,9 @@
 //! Extracts zipped genesis states on first run.
-use eth2_config::{mainnet, prater, pyrmont, Eth2NetArchiveAndDirectory, GENESIS_FILE_NAME};
+use eth2_config::{Eth2NetArchiveAndDirectory, ETH2_NET_DIRS, GENESIS_FILE_NAME};
 use std::fs::File;
 use std::io;
 use zip::ZipArchive;
 
-const ETH2_NET_DIRS: &[Eth2NetArchiveAndDirectory<'static>] = &[
-    mainnet::ETH2_NET_DIR,
-    pyrmont::ETH2_NET_DIR,
-    prater::ETH2_NET_DIR,
-];
-
 fn main() {
     for network in ETH2_NET_DIRS {
         match uncompress_state(network) {

common/eth2_network_config/src/lib.rs

@@ -1,6 +1,18 @@
-use eth2_config::{predefined_networks_dir, *};
+//! Provides the `Eth2NetworkConfig` struct which defines the configuration of an eth2 network or
+//! test-network (aka "testnet").
+//!
+//! Whilst the `Eth2NetworkConfig` struct can be used to read a specification from a directory at
+//! runtime, this crate also includes some pre-defined network configurations "built-in" to the
+//! binary itself (the most notable of these being the "mainnet" configuration). When a network is
+//! "built-in", the  genesis state and configuration files is included in the final binary via the
+//! `std::include_bytes` macro. This provides convenience to the user, the binary is self-sufficient
+//! and does not require the configuration to be read from the filesystem at runtime.
+//!
+//! To add a new built-in testnet, add it to the `define_hardcoded_nets` invocation in the `eth2_config`
+//! crate.
 
 use enr::{CombinedKey, Enr};
+use eth2_config::{instantiate_hardcoded_nets, HardcodedNet};
 use std::fs::{create_dir_all, File};
 use std::io::{Read, Write};
 use std::path::PathBuf;
@@ -11,36 +23,13 @@ pub const BOOT_ENR_FILE: &str = "boot_enr.yaml";
 pub const GENESIS_STATE_FILE: &str = "genesis.ssz";
 pub const BASE_CONFIG_FILE: &str = "config.yaml";
 
-#[derive(Copy, Clone, Debug, PartialEq)]
+// Creates definitions for:
-pub struct HardcodedNet {
+//
-    pub name: &'static str,
+// - Each of the `HardcodedNet` values (e.g., `MAINNET`, `PYRMONT`, etc).
-    pub genesis_is_known: bool,
+// - `HARDCODED_NETS: &[HardcodedNet]`
-    pub config: &'static [u8],
+// - `HARDCODED_NET_NAMES: &[&'static str]`
-    pub deploy_block: &'static [u8],
+instantiate_hardcoded_nets!(eth2_config);
-    pub boot_enr: &'static [u8],
-    pub genesis_state_bytes: &'static [u8],
-}
 
-macro_rules! define_net {
-    ($mod: ident, $include_file: tt) => {{
-        use eth2_config::$mod::ETH2_NET_DIR;
-
-        HardcodedNet {
-            name: ETH2_NET_DIR.name,
-            genesis_is_known: ETH2_NET_DIR.genesis_is_known,
-            config: $include_file!("../", "config.yaml"),
-            deploy_block: $include_file!("../", "deploy_block.txt"),
-            boot_enr: $include_file!("../", "boot_enr.yaml"),
-            genesis_state_bytes: $include_file!("../", "genesis.ssz"),
-        }
-    }};
-}
-
-const PYRMONT: HardcodedNet = define_net!(pyrmont, include_pyrmont_file);
-const MAINNET: HardcodedNet = define_net!(mainnet, include_mainnet_file);
-const PRATER: HardcodedNet = define_net!(prater, include_prater_file);
-
-const HARDCODED_NETS: &[HardcodedNet] = &[PYRMONT, MAINNET, PRATER];
 pub const DEFAULT_HARDCODED_NETWORK: &str = "mainnet";
 
 /// Specifies an Eth2 network.
@@ -241,6 +230,19 @@ mod tests {
 
     type E = MainnetEthSpec;
 
+    #[test]
+    fn default_network_exists() {
+        assert!(HARDCODED_NET_NAMES.contains(&DEFAULT_HARDCODED_NETWORK));
+    }
+
+    #[test]
+    fn hardcoded_testnet_names() {
+        assert_eq!(HARDCODED_NET_NAMES.len(), HARDCODED_NETS.len());
+        for (name, net) in HARDCODED_NET_NAMES.iter().zip(HARDCODED_NETS.iter()) {
+            assert_eq!(name, &net.name);
+        }
+    }
+
     #[test]
     fn mainnet_config_eq_chain_spec() {
         let config = Eth2NetworkConfig::from_hardcoded_net(&MAINNET).unwrap();

lighthouse/src/main.rs

@@ -8,7 +8,7 @@ use clap_utils::flags::DISABLE_MALLOC_TUNING_FLAG;
 use env_logger::{Builder, Env};
 use environment::EnvironmentBuilder;
 use eth2_hashing::have_sha_extensions;
-use eth2_network_config::{Eth2NetworkConfig, DEFAULT_HARDCODED_NETWORK};
+use eth2_network_config::{Eth2NetworkConfig, DEFAULT_HARDCODED_NETWORK, HARDCODED_NET_NAMES};
 use lighthouse_version::VERSION;
 use malloc_utils::configure_memory_allocator;
 use slog::{crit, info, warn};
@@ -127,7 +127,7 @@ fn main() {
                 .long("network")
                 .value_name("network")
                 .help("Name of the Eth2 chain Lighthouse will sync and follow.")
-                .possible_values(&["pyrmont", "mainnet", "prater"])
+                .possible_values(HARDCODED_NET_NAMES)
                 .conflicts_with("testnet-dir")
                 .takes_value(true)
                 .global(true)