From d8fc5f31d8b842788b77662d305c66b5ad900038 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 8 Jun 2019 13:17:03 -0400 Subject: [PATCH] Add Eth2Config to runtime --- beacon_node/Cargo.toml | 1 + beacon_node/client/src/client_config.rs | 7 +- beacon_node/client/src/eth2_config.rs | 48 +++++++++ beacon_node/client/src/lib.rs | 25 +++-- beacon_node/src/main.rs | 66 +++++++++--- beacon_node/src/run.rs | 100 ++++++++++-------- .../block_processing_builder.rs | 2 +- eth2/types/src/chain_spec.rs | 10 +- eth2/types/src/slot_epoch.rs | 1 + eth2/types/src/slot_epoch_macros.rs | 2 +- .../builders/testing_beacon_state_builder.rs | 21 +--- eth2/types/src/test_utils/mod.rs | 2 +- eth2/types/src/test_utils/serde_utils.rs | 12 ++- 13 files changed, 196 insertions(+), 101 deletions(-) create mode 100644 beacon_node/client/src/eth2_config.rs diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 47abb6c6d..77c9844aa 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -12,6 +12,7 @@ store = { path = "./store" } client = { path = "client" } version = { path = "version" } clap = "2.32.0" +serde = "1.0" slog = { version = "^2.2.3" , features = ["max_level_trace", "release_max_level_debug"] } slog-term = "^2.4.0" slog-async = "^2.3.0" diff --git a/beacon_node/client/src/client_config.rs b/beacon_node/client/src/client_config.rs index 1f875f02d..b9caff9f8 100644 --- a/beacon_node/client/src/client_config.rs +++ b/beacon_node/client/src/client_config.rs @@ -4,26 +4,22 @@ use network::NetworkConfig; use serde_derive::{Deserialize, Serialize}; use std::fs; use std::path::PathBuf; -use types::ChainSpec; /// The core configuration of a Lighthouse beacon node. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ClientConfig { pub data_dir: String, - pub spec_constants: String, pub db_type: String, db_name: String, pub network: network::NetworkConfig, pub rpc: rpc::RPCConfig, - pub http: HttpServerConfig, //pub ipc_conf: - pub spec: ChainSpec, + pub http: HttpServerConfig, } impl Default for ClientConfig { fn default() -> Self { Self { data_dir: ".lighthouse".to_string(), - spec_constants: "testnet".to_string(), db_type: "disk".to_string(), db_name: "chain_db".to_string(), // Note: there are no default bootnodes specified. @@ -31,7 +27,6 @@ impl Default for ClientConfig { network: NetworkConfig::new(vec![]), rpc: rpc::RPCConfig::default(), http: HttpServerConfig::default(), - spec: ChainSpec::minimal(), } } } diff --git a/beacon_node/client/src/eth2_config.rs b/beacon_node/client/src/eth2_config.rs new file mode 100644 index 000000000..ca5db98ba --- /dev/null +++ b/beacon_node/client/src/eth2_config.rs @@ -0,0 +1,48 @@ +use clap::ArgMatches; +use serde_derive::{Deserialize, Serialize}; +use std::time::SystemTime; +use types::ChainSpec; + +/// The core configuration of a Lighthouse beacon node. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default)] +pub struct Eth2Config { + pub spec_constants: String, + pub spec: ChainSpec, +} + +impl Default for Eth2Config { + fn default() -> Self { + Self { + spec_constants: "minimal".to_string(), + spec: ChainSpec::minimal(), + } + } +} + +impl Eth2Config { + /// Apply the following arguments to `self`, replacing values if they are specified in `args`. + /// + /// Returns an error if arguments are obviously invalid. May succeed even if some values are + /// invalid. + pub fn apply_cli_args(&mut self, args: &ArgMatches) -> Result<(), &'static str> { + if args.is_present("recent_genesis") { + self.spec.genesis_time = recent_genesis_time() + } + + Ok(()) + } +} + +/// Returns the system time, mod 30 minutes. +/// +/// Used for easily creating testnets. +fn recent_genesis_time() -> u64 { + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(); + let secs_after_last_period = now.checked_rem(30 * 60).unwrap_or(0); + // genesis is now the last 30 minute block. + now - secs_after_last_period +} diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index c4e1c2558..67aff3342 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -3,6 +3,7 @@ extern crate slog; mod beacon_chain_types; mod client_config; pub mod error; +mod eth2_config; pub mod notifier; use beacon_chain::BeaconChain; @@ -22,12 +23,13 @@ pub use beacon_chain::BeaconChainTypes; pub use beacon_chain_types::ClientType; pub use beacon_chain_types::InitialiseBeaconChain; pub use client_config::ClientConfig; +pub use eth2_config::Eth2Config; /// Main beacon node client service. This provides the connection and initialisation of the clients /// sub-services in multiple threads. pub struct Client { /// Configuration for the lighthouse client. - _config: ClientConfig, + _client_config: ClientConfig, /// The beacon chain for the running client. beacon_chain: Arc>, /// Reference to the network service. @@ -50,19 +52,20 @@ where { /// Generate an instance of the client. Spawn and link all internal sub-processes. pub fn new( - config: ClientConfig, + client_config: ClientConfig, + eth2_config: Eth2Config, store: T::Store, log: slog::Logger, executor: &TaskExecutor, ) -> error::Result { let metrics_registry = Registry::new(); let store = Arc::new(store); - let seconds_per_slot = config.spec.seconds_per_slot; + let seconds_per_slot = eth2_config.spec.seconds_per_slot; // Load a `BeaconChain` from the store, or create a new one if it does not exist. let beacon_chain = Arc::new(T::initialise_beacon_chain( store, - config.spec.clone(), + eth2_config.spec.clone(), log.clone(), )); // Registry all beacon chain metrics with the global registry. @@ -102,7 +105,7 @@ where // Start the network service, libp2p and syncing threads // TODO: Add beacon_chain reference to network parameters - let network_config = &config.network; + let network_config = &client_config.network; let network_logger = log.new(o!("Service" => "Network")); let (network, network_send) = NetworkService::new( beacon_chain.clone(), @@ -112,9 +115,9 @@ where )?; // spawn the RPC server - let rpc_exit_signal = if config.rpc.enabled { + let rpc_exit_signal = if client_config.rpc.enabled { Some(rpc::start_server( - &config.rpc, + &client_config.rpc, executor, network_send.clone(), beacon_chain.clone(), @@ -127,13 +130,13 @@ where // Start the `http_server` service. // // Note: presently we are ignoring the config and _always_ starting a HTTP server. - let http_exit_signal = if config.http.enabled { + let http_exit_signal = if client_config.http.enabled { Some(http_server::start_service( - &config.http, + &client_config.http, executor, network_send, beacon_chain.clone(), - config.db_path().expect("unable to read datadir"), + client_config.db_path().expect("unable to read datadir"), metrics_registry, &log, )) @@ -168,7 +171,7 @@ where } Ok(Client { - _config: config, + _client_config: client_config, beacon_chain, http_exit_signal, rpc_exit_signal, diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index 8d2316d99..fe29ecd41 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -3,16 +3,17 @@ extern crate slog; mod run; use clap::{App, Arg}; -use client::ClientConfig; +use client::{ClientConfig, Eth2Config}; use slog::{crit, o, Drain}; use std::fs; use std::fs::File; use std::io::prelude::*; -pub const SAMPLE_CONFIG_FILENAME: &str = "beacon_node_config.sample.toml"; -pub const CONFIG_FILENAME: &str = "beacon_node_config.toml"; pub const DEFAULT_DATA_DIR: &str = ".lighthouse"; +pub const CLIENT_CONFIG_FILENAME: &str = "client_config.toml"; +pub const ETH2_CONFIG_FILENAME: &str = "eth2_config.toml"; + fn main() { let decorator = slog_term::TermDecorator::new().build(); let drain = slog_term::CompactFormat::new(decorator).build().fuse(); @@ -100,9 +101,17 @@ fn main() { .possible_values(&["disk", "memory"]) .default_value("memory"), ) + .arg( + Arg::with_name("recent_genesis") + .long("recent_genesis") + .help("When present, genesis will be within 30 minutes prior. Only for testing"), + ) .get_matches(); - let mut config = match load_config(matches.value_of("data_dir")) { + let mut client_config = match load_config::( + matches.value_of("data_dir"), + CLIENT_CONFIG_FILENAME, + ) { Ok(c) => c, Err(e) => { crit!(logger, "Failed to load/generate a ChainConfig"; "error" => format!("{:?}", e)); @@ -110,15 +119,38 @@ fn main() { } }; - match config.apply_cli_args(&matches) { + if let Some(data_dir) = matches.value_of("data_dir") { + client_config.data_dir = data_dir.to_string(); + } + + match client_config.apply_cli_args(&matches) { Ok(()) => (), Err(s) => { - crit!(logger, "Failed to parse CLI arguments"; "error" => s); + crit!(logger, "Failed to parse ClientConfig CLI arguments"; "error" => s); return; } }; - match run::run_beacon_node(config, &logger) { + let mut eth2_config = match load_config::( + matches.value_of("data_dir"), + ETH2_CONFIG_FILENAME, + ) { + Ok(c) => c, + Err(e) => { + crit!(logger, "Failed to load/generate an Eth2Config"; "error" => format!("{:?}", e)); + return; + } + }; + + match eth2_config.apply_cli_args(&matches) { + Ok(()) => (), + Err(s) => { + crit!(logger, "Failed to parse Eth2Config CLI arguments"; "error" => s); + return; + } + }; + + match run::run_beacon_node(client_config, eth2_config, &logger) { Ok(_) => {} Err(e) => crit!(logger, "Beacon node failed to start"; "reason" => format!("{:}", e)), } @@ -126,7 +158,10 @@ fn main() { /// Loads a `ClientConfig` from file. If unable to load from file, generates a default /// configuration and saves that as a sample file. -fn load_config(data_dir: Option<&str>) -> Result { +fn load_config(data_dir: Option<&str>, config_filename: &str) -> Result +where + T: Default + serde::de::DeserializeOwned + serde::Serialize, +{ let data_dir = data_dir.unwrap_or_else(|| DEFAULT_DATA_DIR); let path = dirs::home_dir() @@ -134,29 +169,28 @@ fn load_config(data_dir: Option<&str>) -> Result { .join(&data_dir); fs::create_dir_all(&path).map_err(|_| "Unable to open data_dir")?; - if let Ok(mut file) = File::open(path.join(CONFIG_FILENAME)) { + if let Ok(mut file) = File::open(path.join(config_filename)) { let mut contents = String::new(); file.read_to_string(&mut contents).map_err(|e| { format!( "Unable to read existing {}. Error: {:?}", - CONFIG_FILENAME, e + config_filename, e ) })?; - toml::from_str(&contents).map_err(|_| format!("Unable to parse {}", CONFIG_FILENAME)) + toml::from_str(&contents).map_err(|e| format!("Unable to parse {}: {:?}", config_filename, e)) } else { - let mut config = ClientConfig::default(); - config.data_dir = data_dir.to_string(); + let config = T::default(); - if let Ok(mut file) = File::create(path.join(SAMPLE_CONFIG_FILENAME)) { + if let Ok(mut file) = File::create(path.join(config_filename)) { let toml_encoded = toml::to_string(&config).map_err(|e| { format!( "Failed to write configuration to {}. Error: {:?}", - SAMPLE_CONFIG_FILENAME, e + config_filename, e ) })?; file.write_all(toml_encoded.as_bytes()) - .expect(&format!("Unable to write to {}", SAMPLE_CONFIG_FILENAME)); + .expect(&format!("Unable to write to {}", config_filename)); } Ok(config) diff --git a/beacon_node/src/run.rs b/beacon_node/src/run.rs index 4e2ea0876..834f9a428 100644 --- a/beacon_node/src/run.rs +++ b/beacon_node/src/run.rs @@ -1,9 +1,10 @@ use client::{ - error, notifier, BeaconChainTypes, Client, ClientConfig, ClientType, InitialiseBeaconChain, + error, notifier, BeaconChainTypes, Client, ClientConfig, ClientType, Eth2Config, + InitialiseBeaconChain, }; use futures::sync::oneshot; use futures::Future; -use slog::{warn, error, info}; +use slog::{error, info, warn}; use std::cell::RefCell; use std::path::Path; use std::path::PathBuf; @@ -14,7 +15,11 @@ use tokio::runtime::TaskExecutor; use tokio_timer::clock::Clock; use types::{MainnetEthSpec, MinimalEthSpec}; -pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Result<()> { +pub fn run_beacon_node( + client_config: ClientConfig, + eth2_config: Eth2Config, + log: &slog::Logger, +) -> error::Result<()> { let runtime = Builder::new() .name_prefix("main-") .clock(Clock::system()) @@ -23,29 +28,54 @@ pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Resul let executor = runtime.executor(); - let db_path: PathBuf = config + let db_path: PathBuf = client_config .db_path() .ok_or_else::(|| "Unable to access database path".into())?; - let db_type = &config.db_type; - let spec_constants = config.spec_constants.clone(); + let db_type = &client_config.db_type; + let spec_constants = eth2_config.spec_constants.clone(); - let other_config = config.clone(); + let other_client_config = client_config.clone(); + + warn!( + log, + "This software is EXPERIMENTAL and provides no guarantees or warranties." + ); let result = match (db_type.as_str(), spec_constants.as_str()) { - ("disk", "minimal") => { - run::>(&db_path, config, executor, runtime, log) - } - ("memory", "minimal") => { - run::>(&db_path, config, executor, runtime, log) - } - ("disk", "mainnet") => { - run::>(&db_path, config, executor, runtime, log) - } - ("memory", "mainnet") => { - run::>(&db_path, config, executor, runtime, log) - } + ("disk", "minimal") => run::>( + &db_path, + client_config, + eth2_config, + executor, + runtime, + log, + ), + ("memory", "minimal") => run::>( + &db_path, + client_config, + eth2_config, + executor, + runtime, + log, + ), + ("disk", "mainnet") => run::>( + &db_path, + client_config, + eth2_config, + executor, + runtime, + log, + ), + ("memory", "mainnet") => run::>( + &db_path, + client_config, + eth2_config, + executor, + runtime, + log, + ), (db_type, spec) => { - error!(log, "Unknown runtime configuration"; "spec" => spec, "db_type" => db_type); + error!(log, "Unknown runtime configuration"; "spec_constants" => spec, "db_type" => db_type); Err("Unknown specification and/or db_type.".into()) } }; @@ -54,28 +84,11 @@ pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Resul info!( log, "Started beacon node"; - "p2p_listen_addresses" => format!("{:?}", &other_config.network.listen_addresses()), - "data_dir" => format!("{:?}", other_config.data_dir()), - "spec_constants" => &other_config.spec_constants, - "db_type" => &other_config.db_type, + "p2p_listen_addresses" => format!("{:?}", &other_client_config.network.listen_addresses()), + "data_dir" => format!("{:?}", other_client_config.data_dir()), + "spec_constants" => &spec_constants, + "db_type" => &other_client_config.db_type, ); - - // `SHUFFLE_ROUND_COUNT == 10` in minimal, this is not considered safe. - if spec_constants.as_str() == "minimal" { - warn!( - log, - "The minimal specification does not use cryptographically secure committee selection." - ) - } - - // Mainnet is not really complete, it still generates determinitic, unsafe initial - // validators. - if spec_constants.as_str() == "mainnet" { - warn!( - log, - "The mainnet specification uses unsafe validator keypairs." - ) - } } result @@ -83,7 +96,8 @@ pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Resul pub fn run( db_path: &Path, - config: ClientConfig, + client_config: ClientConfig, + eth2_config: Eth2Config, executor: TaskExecutor, mut runtime: Runtime, log: &slog::Logger, @@ -94,7 +108,7 @@ where { let store = T::Store::open_database(&db_path)?; - let client: Client = Client::new(config, store, log.clone(), &executor)?; + let client: Client = Client::new(client_config, eth2_config, store, log.clone(), &executor)?; // run service until ctrl-c let (ctrlc_send, ctrlc_oneshot) = oneshot::channel(); diff --git a/eth2/state_processing/src/per_block_processing/block_processing_builder.rs b/eth2/state_processing/src/per_block_processing/block_processing_builder.rs index 5a9d264ce..3675ef99e 100644 --- a/eth2/state_processing/src/per_block_processing/block_processing_builder.rs +++ b/eth2/state_processing/src/per_block_processing/block_processing_builder.rs @@ -23,7 +23,7 @@ impl BlockProcessingBuilder { } pub fn set_slot(&mut self, slot: Slot, spec: &ChainSpec) { - self.state_builder.teleport_to_slot(slot, &spec); + self.state_builder.teleport_to_slot(slot); } pub fn build_caches(&mut self, spec: &ChainSpec) { diff --git a/eth2/types/src/chain_spec.rs b/eth2/types/src/chain_spec.rs index 0a2602ea8..b16947328 100644 --- a/eth2/types/src/chain_spec.rs +++ b/eth2/types/src/chain_spec.rs @@ -1,7 +1,7 @@ use crate::*; use int_to_bytes::int_to_bytes4; use serde_derive::{Deserialize, Serialize}; -use test_utils::u8_from_hex_str; +use test_utils::{u8_from_hex_str, u8_to_hex_str}; /// Each of the BLS signature domains. /// @@ -48,17 +48,19 @@ pub struct ChainSpec { * Initial Values */ pub genesis_slot: Slot, + // Skipped because serde TOML can't handle u64::max_value, the typical value for this field. + #[serde(skip_serializing)] pub far_future_epoch: Epoch, pub zero_hash: Hash256, - #[serde(deserialize_with = "u8_from_hex_str")] + #[serde(deserialize_with = "u8_from_hex_str", serialize_with = "u8_to_hex_str")] pub bls_withdrawal_prefix_byte: u8, /* * Time parameters */ + pub genesis_time: u64, pub seconds_per_slot: u64, pub min_attestation_inclusion_delay: u64, - //pub slots_per_epoch: u64, pub min_seed_lookahead: Epoch, pub activation_exit_delay: u64, pub slots_per_eth1_voting_period: u64, @@ -172,9 +174,9 @@ impl ChainSpec { /* * Time parameters */ + genesis_time: u32::max_value() as u64, seconds_per_slot: 6, min_attestation_inclusion_delay: 4, - // slots_per_epoch: 64, min_seed_lookahead: Epoch::new(1), activation_exit_delay: 4, slots_per_eth1_voting_period: 1_024, diff --git a/eth2/types/src/slot_epoch.rs b/eth2/types/src/slot_epoch.rs index 82cee0d75..fbeb479d7 100644 --- a/eth2/types/src/slot_epoch.rs +++ b/eth2/types/src/slot_epoch.rs @@ -23,6 +23,7 @@ use std::iter::Iterator; use std::ops::{Add, AddAssign, Div, DivAssign, Mul, MulAssign, Rem, Sub, SubAssign}; #[derive(Eq, Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[serde(transparent)] pub struct Slot(u64); #[derive(Eq, Debug, Clone, Copy, Default, Serialize, Deserialize)] diff --git a/eth2/types/src/slot_epoch_macros.rs b/eth2/types/src/slot_epoch_macros.rs index 1e24f8e99..5e02e40c1 100644 --- a/eth2/types/src/slot_epoch_macros.rs +++ b/eth2/types/src/slot_epoch_macros.rs @@ -184,7 +184,7 @@ macro_rules! impl_display { key: slog::Key, serializer: &mut slog::Serializer, ) -> slog::Result { - self.0.serialize(record, key, serializer) + slog::Value::serialize(&self.0, record, key, serializer) } } }; diff --git a/eth2/types/src/test_utils/builders/testing_beacon_state_builder.rs b/eth2/types/src/test_utils/builders/testing_beacon_state_builder.rs index f9a4ea00c..4f0927508 100644 --- a/eth2/types/src/test_utils/builders/testing_beacon_state_builder.rs +++ b/eth2/types/src/test_utils/builders/testing_beacon_state_builder.rs @@ -6,7 +6,6 @@ use dirs; use log::debug; use rayon::prelude::*; use std::path::{Path, PathBuf}; -use std::time::SystemTime; pub const KEYPAIRS_FILE: &str = "keypairs.raw_keypairs"; @@ -123,20 +122,8 @@ impl TestingBeaconStateBuilder { }) .collect(); - // TODO: Testing only. Burn with fire later. - // set genesis to the last 30 minute block. - // this is used for testing only. Allows multiple nodes to connect within a 30min window - // and agree on a genesis - let now = SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap() - .as_secs(); - let secs_after_last_period = now.checked_rem(30 * 60).unwrap_or(0); - // genesis is now the last 30 minute block. - let genesis_time = now - secs_after_last_period; - let mut state = BeaconState::genesis( - genesis_time, + spec.genesis_time, Eth1Data { deposit_root: Hash256::zero(), deposit_count: 0, @@ -172,8 +159,8 @@ impl TestingBeaconStateBuilder { } /// Sets the `BeaconState` to be in a slot, calling `teleport_to_epoch` to update the epoch. - pub fn teleport_to_slot(&mut self, slot: Slot, spec: &ChainSpec) { - self.teleport_to_epoch(slot.epoch(T::slots_per_epoch()), spec); + pub fn teleport_to_slot(&mut self, slot: Slot) { + self.teleport_to_epoch(slot.epoch(T::slots_per_epoch())); self.state.slot = slot; } @@ -181,7 +168,7 @@ impl TestingBeaconStateBuilder { /// /// Sets all justification/finalization parameters to be be as "perfect" as possible (i.e., /// highest justified and finalized slots, full justification bitfield, etc). - fn teleport_to_epoch(&mut self, epoch: Epoch, spec: &ChainSpec) { + fn teleport_to_epoch(&mut self, epoch: Epoch) { let state = &mut self.state; let slot = epoch.start_slot(T::slots_per_epoch()); diff --git a/eth2/types/src/test_utils/mod.rs b/eth2/types/src/test_utils/mod.rs index ee8327be8..b5ec7a027 100644 --- a/eth2/types/src/test_utils/mod.rs +++ b/eth2/types/src/test_utils/mod.rs @@ -14,5 +14,5 @@ pub use rand::{ RngCore, {prng::XorShiftRng, SeedableRng}, }; -pub use serde_utils::{fork_from_hex_str, graffiti_from_hex_str, u8_from_hex_str}; +pub use serde_utils::{fork_from_hex_str, graffiti_from_hex_str, u8_from_hex_str, u8_to_hex_str}; pub use test_random::TestRandom; diff --git a/eth2/types/src/test_utils/serde_utils.rs b/eth2/types/src/test_utils/serde_utils.rs index 5c0238c0b..be8f002ad 100644 --- a/eth2/types/src/test_utils/serde_utils.rs +++ b/eth2/types/src/test_utils/serde_utils.rs @@ -1,5 +1,5 @@ use serde::de::Error; -use serde::{Deserialize, Deserializer}; +use serde::{Deserialize, Deserializer, Serializer}; pub const FORK_BYTES_LEN: usize = 4; pub const GRAFFITI_BYTES_LEN: usize = 32; @@ -13,6 +13,16 @@ where u8::from_str_radix(&s.as_str()[2..], 16).map_err(D::Error::custom) } +pub fn u8_to_hex_str(byte: &u8, serializer: S) -> Result +where + S: Serializer, +{ + let mut hex: String = "0x".to_string(); + hex.push_str(&hex::encode(&[*byte])); + + serializer.serialize_str(&hex) +} + pub fn fork_from_hex_str<'de, D>(deserializer: D) -> Result<[u8; FORK_BYTES_LEN], D::Error> where D: Deserializer<'de>,