Improve ergonomics of adding a new network config (#2489)
## Issue Addressed
NA
## Proposed Changes
This PR adds some more fancy macro magic to make it easier to add a new built-in (aka "baked-in") testnet config to the `lighthouse` binary.
Previously, a user needed to modify several files and repeat themselves several times. Now, they only need to add a single definition in the `eth2_config` crate. No repetition 🎉
This commit is contained in:
Paul Hauner
parent
b4dd98b3c6
commit
1031f79aca
7
Cargo.lock
generated
7
Cargo.lock
generated
@ -1846,6 +1846,7 @@ dependencies = [
|
||||
name = "eth2_config"
|
||||
version = "0.2.0"
|
||||
dependencies = [
|
||||
"paste",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"types",
|
||||
@ -4638,6 +4639,12 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "paste"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "acbf547ad0c65e31259204bd90935776d1c693cec2f4ff7abb7a1bbbd40dfe58"
|
||||
|
||||
[[package]]
|
||||
name = "pbkdf2"
|
||||
version = "0.4.0"
|
||||
|
||||
@ -8,3 +8,4 @@ edition = "2018"
|
||||
serde = "1.0.116"
|
||||
serde_derive = "1.0.116"
|
||||
types = { path = "../../consensus/types" }
|
||||
paste = "1.0.5"
|
||||
|
||||
@ -1,7 +1,16 @@
|
||||
//! This crate primarily exists to serve the `common/eth2_network_configs` crate, by providing the
|
||||
//! canonical list of built-in-networks and some tooling to help include those configurations in the
|
||||
//! `lighthouse` binary.
|
||||
//!
|
||||
//! It also provides some additional structs which are useful to other components of `lighthouse`
|
||||
//! (e.g., `Eth2Config`).
|
||||
|
||||
use std::env;
|
||||
use std::path::PathBuf;
|
||||
use types::{ChainSpec, EthSpecId};
|
||||
|
||||
pub use paste::paste;
|
||||
|
||||
// A macro is used to define this constant so it can be used with `include_bytes!`.
|
||||
#[macro_export]
|
||||
macro_rules! predefined_networks_dir {
|
||||
@ -73,40 +82,152 @@ impl<'a> Eth2NetArchiveAndDirectory<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
macro_rules! define_net {
|
||||
($title: ident, $macro_title: tt, $name: tt, $genesis_is_known: tt) => {
|
||||
#[macro_use]
|
||||
pub mod $title {
|
||||
use super::*;
|
||||
/// Indicates that the `genesis.ssz.zip` file is present on the filesystem. This means that the
|
||||
/// deposit ceremony has concluded and the final genesis `BeaconState` is known.
|
||||
const GENESIS_STATE_IS_KNOWN: bool = true;
|
||||
|
||||
pub const ETH2_NET_DIR: Eth2NetArchiveAndDirectory = Eth2NetArchiveAndDirectory {
|
||||
name: $name,
|
||||
unique_id: $name,
|
||||
genesis_is_known: $genesis_is_known,
|
||||
};
|
||||
#[derive(Copy, Clone, Debug, PartialEq)]
|
||||
pub struct HardcodedNet {
|
||||
pub name: &'static str,
|
||||
pub genesis_is_known: bool,
|
||||
pub config: &'static [u8],
|
||||
pub deploy_block: &'static [u8],
|
||||
pub boot_enr: &'static [u8],
|
||||
pub genesis_state_bytes: &'static [u8],
|
||||
}
|
||||
|
||||
// A wrapper around `std::include_bytes` which includes a file from a specific network
|
||||
// directory. Used by upstream crates to import files at compile time.
|
||||
#[macro_export]
|
||||
macro_rules! $macro_title {
|
||||
($base_dir: tt, $filename: tt) => {
|
||||
include_bytes!(concat!(
|
||||
$base_dir,
|
||||
"/",
|
||||
predefined_networks_dir!(),
|
||||
"/",
|
||||
$name,
|
||||
"/",
|
||||
$filename
|
||||
))
|
||||
/// Defines an `Eth2NetArchiveAndDirectory` for some network.
|
||||
///
|
||||
/// It also defines a `include_<title>_file!` macro which provides a wrapper around
|
||||
/// `std::include_bytes`, allowing the inclusion of bytes from the specific testnet directory.
|
||||
macro_rules! define_archive {
|
||||
($name_ident: ident, $name_str: tt, $genesis_is_known: ident) => {
|
||||
paste! {
|
||||
#[macro_use]
|
||||
pub mod $name_ident {
|
||||
use super::*;
|
||||
|
||||
pub const ETH2_NET_DIR: Eth2NetArchiveAndDirectory = Eth2NetArchiveAndDirectory {
|
||||
name: $name_str,
|
||||
unique_id: $name_str,
|
||||
genesis_is_known: $genesis_is_known,
|
||||
};
|
||||
|
||||
/// A wrapper around `std::include_bytes` which includes a file from a specific network
|
||||
/// directory. Used by upstream crates to import files at compile time.
|
||||
#[macro_export]
|
||||
macro_rules! [<include_ $name_ident _file>] {
|
||||
($this_crate: ident, $base_dir: tt, $filename: tt) => {
|
||||
include_bytes!(concat!(
|
||||
$base_dir,
|
||||
"/",
|
||||
$this_crate::predefined_networks_dir!(),
|
||||
"/",
|
||||
$name_str,
|
||||
"/",
|
||||
$filename
|
||||
))
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
define_net!(pyrmont, include_pyrmont_file, "pyrmont", true);
|
||||
/// Creates a `HardcodedNet` definition for some network.
|
||||
#[macro_export]
|
||||
macro_rules! define_net {
|
||||
($this_crate: ident, $mod: ident, $include_file: tt) => {{
|
||||
use $this_crate::$mod::ETH2_NET_DIR;
|
||||
|
||||
define_net!(mainnet, include_mainnet_file, "mainnet", true);
|
||||
$this_crate::HardcodedNet {
|
||||
name: ETH2_NET_DIR.name,
|
||||
genesis_is_known: ETH2_NET_DIR.genesis_is_known,
|
||||
config: $this_crate::$include_file!($this_crate, "../", "config.yaml"),
|
||||
deploy_block: $this_crate::$include_file!($this_crate, "../", "deploy_block.txt"),
|
||||
boot_enr: $this_crate::$include_file!($this_crate, "../", "boot_enr.yaml"),
|
||||
genesis_state_bytes: $this_crate::$include_file!($this_crate, "../", "genesis.ssz"),
|
||||
}
|
||||
}};
|
||||
}
|
||||
|
||||
define_net!(prater, include_prater_file, "prater", true);
|
||||
/// Calls `define_net` on a list of networks, and then defines two more lists:
|
||||
///
|
||||
/// - `HARDCODED_NETS`: a list of all the networks defined by this macro.
|
||||
/// - `HARDCODED_NET_NAMES`: a list of the *names* of the networks defined by this macro.
|
||||
#[macro_export]
|
||||
macro_rules! define_nets {
|
||||
($this_crate: ident, $($name_ident: ident, $name_str: tt,)+) => {
|
||||
$this_crate::paste! {
|
||||
$(
|
||||
const [<$name_ident:upper>]: $this_crate::HardcodedNet = $this_crate::define_net!($this_crate, $name_ident, [<include_ $name_ident _file>]);
|
||||
)+
|
||||
const HARDCODED_NETS: &[$this_crate::HardcodedNet] = &[$([<$name_ident:upper>],)+];
|
||||
pub const HARDCODED_NET_NAMES: &[&'static str] = &[$($name_str,)+];
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// The canonical macro for defining built-in network configurations.
|
||||
///
|
||||
/// This macro will provide:
|
||||
///
|
||||
/// - An `Eth2NetArchiveAndDirectory` for each network.
|
||||
/// - `ETH2_NET_DIRS`: a list of all the above `Eth2NetArchiveAndDirectory`.
|
||||
/// - The `instantiate_hardcoded_nets` macro (see its documentation).
|
||||
///
|
||||
/// ## Design Justification
|
||||
///
|
||||
/// Ultimately, this macro serves as a single list of all the networks. The reason it is structured
|
||||
/// in such a complex web-of-macros way is because two requirements of built-in (hard-coded) networks:
|
||||
///
|
||||
/// 1. We must use `std::include_bytes!` to "bake" arbitrary bytes (genesis states, etc) into the binary.
|
||||
/// 2. We must use a `build.rs` script to decompress the genesis state from a zip file, before we
|
||||
/// can include those bytes.
|
||||
///
|
||||
/// Because of these two constraints, we must first define all of the networks and the paths to
|
||||
/// their files in this crate. Then, we must use another crate (`eth2_network_configs`) to run a
|
||||
/// `build.rs` which will unzip the genesis states. Then, that `eth2_network_configs` crate can
|
||||
/// perform the final step of using `std::include_bytes` to bake the files (bytes) into the binary.
|
||||
macro_rules! define_hardcoded_nets {
|
||||
($(($name_ident: ident, $name_str: tt, $genesis_is_known: ident)),+) => {
|
||||
$(
|
||||
define_archive!($name_ident, $name_str, $genesis_is_known);
|
||||
)+
|
||||
|
||||
pub const ETH2_NET_DIRS: &[Eth2NetArchiveAndDirectory<'static>] = &[$($name_ident::ETH2_NET_DIR,)+];
|
||||
|
||||
/// This macro is designed to be called by an external crate. When called, it will
|
||||
/// define in that external crate:
|
||||
///
|
||||
/// - A `HardcodedNet` for each network.
|
||||
/// - `HARDCODED_NETS`: a list of all the above `HardcodedNet`.
|
||||
/// - `HARDCODED_NET_NAMES`: a list of all the names of the above `HardcodedNet` (as `&str`).
|
||||
#[macro_export]
|
||||
macro_rules! instantiate_hardcoded_nets {
|
||||
($this_crate: ident) => {
|
||||
$this_crate::define_nets!($this_crate, $($name_ident, $name_str,)+);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// Add a new "built-in" network by adding it to the list below.
|
||||
//
|
||||
// The last entry must not end with a comma, otherwise compilation will fail.
|
||||
//
|
||||
// This is the canonical place for defining the built-in network configurations that are present in
|
||||
// the `common/eth2_network_config/built_in_network_configs` directory.
|
||||
//
|
||||
// Each net is defined as a three-tuple:
|
||||
//
|
||||
// 0. The name of the testnet as an "ident" (i.e. something that can be a Rust variable name).
|
||||
// 1. The human-friendly name of the testnet (i.e. usually with "-" instead of "_").
|
||||
// 2. A bool indicating if the genesis state is known and present as a `genesis.ssz.zip`.
|
||||
//
|
||||
// The directory containing the testnet files should match the human-friendly name (element 1).
|
||||
define_hardcoded_nets!(
|
||||
(mainnet, "mainnet", GENESIS_STATE_IS_KNOWN),
|
||||
(pyrmont, "pyrmont", GENESIS_STATE_IS_KNOWN),
|
||||
(prater, "prater", GENESIS_STATE_IS_KNOWN)
|
||||
);
|
||||
|
||||
@ -1,15 +1,9 @@
|
||||
//! Extracts zipped genesis states on first run.
|
||||
use eth2_config::{mainnet, prater, pyrmont, Eth2NetArchiveAndDirectory, GENESIS_FILE_NAME};
|
||||
use eth2_config::{Eth2NetArchiveAndDirectory, ETH2_NET_DIRS, GENESIS_FILE_NAME};
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
use zip::ZipArchive;
|
||||
|
||||
const ETH2_NET_DIRS: &[Eth2NetArchiveAndDirectory<'static>] = &[
|
||||
mainnet::ETH2_NET_DIR,
|
||||
pyrmont::ETH2_NET_DIR,
|
||||
prater::ETH2_NET_DIR,
|
||||
];
|
||||
|
||||
fn main() {
|
||||
for network in ETH2_NET_DIRS {
|
||||
match uncompress_state(network) {
|
||||
|
||||
@ -1,6 +1,18 @@
|
||||
use eth2_config::{predefined_networks_dir, *};
|
||||
//! Provides the `Eth2NetworkConfig` struct which defines the configuration of an eth2 network or
|
||||
//! test-network (aka "testnet").
|
||||
//!
|
||||
//! Whilst the `Eth2NetworkConfig` struct can be used to read a specification from a directory at
|
||||
//! runtime, this crate also includes some pre-defined network configurations "built-in" to the
|
||||
//! binary itself (the most notable of these being the "mainnet" configuration). When a network is
|
||||
//! "built-in", the genesis state and configuration files is included in the final binary via the
|
||||
//! `std::include_bytes` macro. This provides convenience to the user, the binary is self-sufficient
|
||||
//! and does not require the configuration to be read from the filesystem at runtime.
|
||||
//!
|
||||
//! To add a new built-in testnet, add it to the `define_hardcoded_nets` invocation in the `eth2_config`
|
||||
//! crate.
|
||||
|
||||
use enr::{CombinedKey, Enr};
|
||||
use eth2_config::{instantiate_hardcoded_nets, HardcodedNet};
|
||||
use std::fs::{create_dir_all, File};
|
||||
use std::io::{Read, Write};
|
||||
use std::path::PathBuf;
|
||||
@ -11,36 +23,13 @@ pub const BOOT_ENR_FILE: &str = "boot_enr.yaml";
|
||||
pub const GENESIS_STATE_FILE: &str = "genesis.ssz";
|
||||
pub const BASE_CONFIG_FILE: &str = "config.yaml";
|
||||
|
||||
#[derive(Copy, Clone, Debug, PartialEq)]
|
||||
pub struct HardcodedNet {
|
||||
pub name: &'static str,
|
||||
pub genesis_is_known: bool,
|
||||
pub config: &'static [u8],
|
||||
pub deploy_block: &'static [u8],
|
||||
pub boot_enr: &'static [u8],
|
||||
pub genesis_state_bytes: &'static [u8],
|
||||
}
|
||||
// Creates definitions for:
|
||||
//
|
||||
// - Each of the `HardcodedNet` values (e.g., `MAINNET`, `PYRMONT`, etc).
|
||||
// - `HARDCODED_NETS: &[HardcodedNet]`
|
||||
// - `HARDCODED_NET_NAMES: &[&'static str]`
|
||||
instantiate_hardcoded_nets!(eth2_config);
|
||||
|
||||
macro_rules! define_net {
|
||||
($mod: ident, $include_file: tt) => {{
|
||||
use eth2_config::$mod::ETH2_NET_DIR;
|
||||
|
||||
HardcodedNet {
|
||||
name: ETH2_NET_DIR.name,
|
||||
genesis_is_known: ETH2_NET_DIR.genesis_is_known,
|
||||
config: $include_file!("../", "config.yaml"),
|
||||
deploy_block: $include_file!("../", "deploy_block.txt"),
|
||||
boot_enr: $include_file!("../", "boot_enr.yaml"),
|
||||
genesis_state_bytes: $include_file!("../", "genesis.ssz"),
|
||||
}
|
||||
}};
|
||||
}
|
||||
|
||||
const PYRMONT: HardcodedNet = define_net!(pyrmont, include_pyrmont_file);
|
||||
const MAINNET: HardcodedNet = define_net!(mainnet, include_mainnet_file);
|
||||
const PRATER: HardcodedNet = define_net!(prater, include_prater_file);
|
||||
|
||||
const HARDCODED_NETS: &[HardcodedNet] = &[PYRMONT, MAINNET, PRATER];
|
||||
pub const DEFAULT_HARDCODED_NETWORK: &str = "mainnet";
|
||||
|
||||
/// Specifies an Eth2 network.
|
||||
@ -241,6 +230,19 @@ mod tests {
|
||||
|
||||
type E = MainnetEthSpec;
|
||||
|
||||
#[test]
|
||||
fn default_network_exists() {
|
||||
assert!(HARDCODED_NET_NAMES.contains(&DEFAULT_HARDCODED_NETWORK));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hardcoded_testnet_names() {
|
||||
assert_eq!(HARDCODED_NET_NAMES.len(), HARDCODED_NETS.len());
|
||||
for (name, net) in HARDCODED_NET_NAMES.iter().zip(HARDCODED_NETS.iter()) {
|
||||
assert_eq!(name, &net.name);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mainnet_config_eq_chain_spec() {
|
||||
let config = Eth2NetworkConfig::from_hardcoded_net(&MAINNET).unwrap();
|
||||
|
||||
@ -8,7 +8,7 @@ use clap_utils::flags::DISABLE_MALLOC_TUNING_FLAG;
|
||||
use env_logger::{Builder, Env};
|
||||
use environment::EnvironmentBuilder;
|
||||
use eth2_hashing::have_sha_extensions;
|
||||
use eth2_network_config::{Eth2NetworkConfig, DEFAULT_HARDCODED_NETWORK};
|
||||
use eth2_network_config::{Eth2NetworkConfig, DEFAULT_HARDCODED_NETWORK, HARDCODED_NET_NAMES};
|
||||
use lighthouse_version::VERSION;
|
||||
use malloc_utils::configure_memory_allocator;
|
||||
use slog::{crit, info, warn};
|
||||
@ -127,7 +127,7 @@ fn main() {
|
||||
.long("network")
|
||||
.value_name("network")
|
||||
.help("Name of the Eth2 chain Lighthouse will sync and follow.")
|
||||
.possible_values(&["pyrmont", "mainnet", "prater"])
|
||||
.possible_values(HARDCODED_NET_NAMES)
|
||||
.conflicts_with("testnet-dir")
|
||||
.takes_value(true)
|
||||
.global(true)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user