Add whiteblock script, CLI options to support it

This commit is contained in:
Paul Hauner 2019-09-10 12:13:54 -04:00
parent 5de80f2799
commit 8c5a8034b6
No known key found for this signature in database
GPG Key ID: 303E4494BB28068C
16 changed files with 342 additions and 28 deletions

View File

@ -6,6 +6,7 @@ edition = "2018"
[dependencies]
clap = "2.32.0"
hex = "0.3"
#SigP repository
libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "d4851ea3b564266aeb9d83d10148b972721999db" }
enr = { git = "https://github.com/SigP/rust-libp2p/", rev = "d4851ea3b564266aeb9d83d10148b972721999db", features = ["serde"] }

View File

@ -40,6 +40,12 @@ pub struct Config {
/// Target number of connected peers.
pub max_peers: usize,
/// A secp256k1 secret key, as bytes in ASCII-encoded hex.
///
/// With or without `0x` prefix.
#[serde(skip)]
pub secret_key_hex: Option<String>,
/// Gossipsub configuration parameters.
#[serde(skip)]
pub gs_config: GossipsubConfig,
@ -70,6 +76,7 @@ impl Default for Config {
discovery_address: "127.0.0.1".parse().expect("valid ip address"),
discovery_port: 9000,
max_peers: 10,
secret_key_hex: None,
// Note: The topics by default are sent as plain strings. Hashes are an optional
// parameter.
gs_config: GossipsubConfigBuilder::new()
@ -158,6 +165,10 @@ impl Config {
.map_err(|_| format!("Invalid discovery port: {}", disc_port_str))?;
}
if let Some(p2p_priv_key) = args.value_of("p2p-priv-key") {
self.secret_key_hex = Some(p2p_priv_key.to_string());
}
Ok(())
}
}

View File

@ -42,16 +42,22 @@ impl Service {
pub fn new(config: NetworkConfig, log: slog::Logger) -> error::Result<Self> {
trace!(log, "Libp2p Service starting");
let local_keypair = if let Some(hex_bytes) = &config.secret_key_hex {
keypair_from_hex(hex_bytes)?
} else {
load_private_key(&config, &log)
};
// load the private key from CLI flag, disk or generate a new one
let local_private_key = load_private_key(&config, &log);
let local_peer_id = PeerId::from(local_private_key.public());
// let local_private_key = load_private_key(&config, &log);
let local_peer_id = PeerId::from(local_keypair.public());
info!(log, "Libp2p Service"; "peer_id" => format!("{:?}", local_peer_id));
let mut swarm = {
// Set up the transport - tcp/ws with secio and mplex/yamux
let transport = build_transport(local_private_key.clone());
let transport = build_transport(local_keypair.clone());
// Lighthouse network behaviour
let behaviour = Behaviour::new(&local_private_key, &config, &log)?;
let behaviour = Behaviour::new(&local_keypair, &config, &log)?;
Swarm::new(transport, behaviour, local_peer_id.clone())
};
@ -246,6 +252,27 @@ pub enum Libp2pEvent {
},
}
fn keypair_from_hex(hex_bytes: &str) -> error::Result<Keypair> {
let hex_bytes = if hex_bytes.starts_with("0x") {
hex_bytes[2..].to_string()
} else {
hex_bytes.to_string()
};
hex::decode(&hex_bytes)
.map_err(|e| format!("Failed to parse p2p secret key bytes: {:?}", e).into())
.and_then(keypair_from_bytes)
}
fn keypair_from_bytes(mut bytes: Vec<u8>) -> error::Result<Keypair> {
libp2p::core::identity::secp256k1::SecretKey::from_bytes(&mut bytes)
.map(|secret| {
let keypair: libp2p::core::identity::secp256k1::Keypair = secret.into();
Keypair::Secp256k1(keypair)
})
.map_err(|e| format!("Unable to parse p2p secret key: {:?}", e).into())
}
/// Loads a private key from disk. If this fails, a new key is
/// generated and is then saved to disk.
///

View File

@ -13,7 +13,7 @@ pub const CLIENT_CONFIG_FILENAME: &str = "beacon-node.toml";
pub const ETH2_CONFIG_FILENAME: &str = "eth2-spec.toml";
type Result<T> = std::result::Result<T, String>;
type Config = (ClientConfig, Eth2Config);
type Config = (ClientConfig, Eth2Config, Logger);
/// Gets the fully-initialized global client and eth2 configuration objects.
///
@ -22,8 +22,10 @@ type Config = (ClientConfig, Eth2Config);
/// The output of this function depends primarily upon the given `cli_args`, however it's behaviour
/// may be influenced by other external services like the contents of the file system or the
/// response of some remote server.
pub fn get_configs(cli_args: &ArgMatches, log: &Logger) -> Result<Config> {
let mut builder = ConfigBuilder::new(cli_args, log)?;
pub fn get_configs(cli_args: &ArgMatches, core_log: Logger) -> Result<Config> {
let log = core_log.clone();
let mut builder = ConfigBuilder::new(cli_args, core_log)?;
if let Some(server) = cli_args.value_of("eth1-server") {
builder.set_eth1_backend_method(Eth1BackendMethod::Web3 {
@ -35,7 +37,7 @@ pub fn get_configs(cli_args: &ArgMatches, log: &Logger) -> Result<Config> {
match cli_args.subcommand() {
("testnet", Some(sub_cmd_args)) => {
process_testnet_subcommand(&mut builder, sub_cmd_args, log)?
process_testnet_subcommand(&mut builder, sub_cmd_args, &log)?
}
// No sub-command assumes a resume operation.
_ => {
@ -216,15 +218,15 @@ fn process_testnet_subcommand(
}
/// Allows for building a set of configurations based upon `clap` arguments.
struct ConfigBuilder<'a> {
log: &'a Logger,
struct ConfigBuilder {
log: Logger,
eth2_config: Eth2Config,
client_config: ClientConfig,
}
impl<'a> ConfigBuilder<'a> {
impl ConfigBuilder {
/// Create a new builder with default settings.
pub fn new(cli_args: &'a ArgMatches, log: &'a Logger) -> Result<Self> {
pub fn new(cli_args: &ArgMatches, log: Logger) -> Result<Self> {
// Read the `--datadir` flag.
//
// If it's not present, try and find the home directory (`~`) and push the default data
@ -539,8 +541,7 @@ impl<'a> ConfigBuilder<'a> {
/// cli_args).
pub fn build(mut self, cli_args: &ArgMatches) -> Result<Config> {
self.eth2_config.apply_cli_args(cli_args)?;
self.client_config
.apply_cli_args(cli_args, &mut self.log.clone())?;
self.client_config.apply_cli_args(cli_args, &mut self.log)?;
if let Some(bump) = cli_args.value_of("port-bump") {
let bump = bump
@ -561,7 +562,7 @@ impl<'a> ConfigBuilder<'a> {
return Err("Specification constant mismatch".into());
}
Ok((self.client_config, self.eth2_config))
Ok((self.client_config, self.eth2_config, self.log))
}
}

View File

@ -116,6 +116,13 @@ fn main() {
.help("One or more comma-delimited multiaddrs to manually connect to a libp2p peer without an ENR.")
.takes_value(true),
)
.arg(
Arg::with_name("p2p-priv-key")
.long("p2p-priv-key")
.value_name("HEX")
.help("A secp256k1 secret key, represented as ASCII-encoded hex bytes (with or without 0x prefix).")
.takes_value(true),
)
/*
* gRPC parameters.
*/
@ -355,13 +362,15 @@ fn main() {
"Ethereum 2.0 is pre-release. This software is experimental."
);
let log_clone = log.clone();
// Load the process-wide configuration.
//
// May load this from disk or create a new configuration, depending on the CLI flags supplied.
let (client_config, eth2_config) = match get_configs(&matches, &log) {
let (client_config, eth2_config, log) = match get_configs(&matches, log) {
Ok(configs) => configs,
Err(e) => {
crit!(log, "Failed to load configuration"; "error" => e);
crit!(log_clone, "Failed to load configuration. Exiting"; "error" => e);
return;
}
};

View File

@ -1,7 +1,8 @@
use crate::*;
use eth2_interop_keypairs::keypair;
use eth2_interop_keypairs::{keypair, keypairs_from_yaml_file};
use log::debug;
use rayon::prelude::*;
use std::path::PathBuf;
/// Generates `validator_count` keypairs where the secret key is derived solely from the index of
/// the validator.
@ -32,3 +33,14 @@ pub fn generate_deterministic_keypair(validator_index: usize) -> Keypair {
sk: SecretKey::from_raw(raw.sk),
}
}
/// Loads a list of keypairs from file.
pub fn load_keypairs_from_yaml(path: PathBuf) -> Result<Vec<Keypair>, String> {
Ok(keypairs_from_yaml_file(path)?
.into_iter()
.map(|raw| Keypair {
pk: PublicKey::from_raw(raw.pk),
sk: SecretKey::from_raw(raw.sk),
})
.collect())
}

View File

@ -8,6 +8,7 @@ mod test_random;
pub use builders::*;
pub use generate_deterministic_keypairs::generate_deterministic_keypair;
pub use generate_deterministic_keypairs::generate_deterministic_keypairs;
pub use generate_deterministic_keypairs::load_keypairs_from_yaml;
pub use keypairs_file::KeypairsFile;
pub use rand::{
RngCore,

View File

@ -10,10 +10,11 @@ edition = "2018"
lazy_static = "1.4"
num-bigint = "0.2"
eth2_hashing = "0.1"
hex = "0.3"
milagro_bls = { git = "https://github.com/michaelsproul/milagro_bls", branch = "little-endian-v0.10" }
serde_yaml = "0.8"
serde = "1.0"
serde_derive = "1.0"
[dev-dependencies]
base64 = "0.10"
serde = "1.0"
serde_derive = "1.0"
serde_yaml = "0.8"

View File

@ -0,0 +1,20 @@
- {privkey: '0x25295f0d1d592a90b333e26e85149708208e9f8e8bc18f6c77bd62f8ad7a6866',
pubkey: '0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c'}
- {privkey: '0x51d0b65185db6989ab0b560d6deed19c7ead0e24b9b6372cbecb1f26bdfad000',
pubkey: '0xb89bebc699769726a318c8e9971bd3171297c61aea4a6578a7a4f94b547dcba5bac16a89108b6b6a1fe3695d1a874a0b'}
- {privkey: '0x315ed405fafe339603932eebe8dbfd650ce5dafa561f6928664c75db85f97857',
pubkey: '0xa3a32b0f8b4ddb83f1a0a853d81dd725dfe577d4f4c3db8ece52ce2b026eca84815c1a7e8e92a4de3d755733bf7e4a9b'}
- {privkey: '0x25b1166a43c109cb330af8945d364722757c65ed2bfed5444b5a2f057f82d391',
pubkey: '0x88c141df77cd9d8d7a71a75c826c41a9c9f03c6ee1b180f3e7852f6a280099ded351b58d66e653af8e42816a4d8f532e'}
- {privkey: '0x3f5615898238c4c4f906b507ee917e9ea1bb69b93f1dbd11a34d229c3b06784b',
pubkey: '0x81283b7a20e1ca460ebd9bbd77005d557370cabb1f9a44f530c4c4c66230f675f8df8b4c2818851aa7d77a80ca5a4a5e'}
- {privkey: '0x055794614bc85ed5436c1f5cab586aab6ca84835788621091f4f3b813761e7a8',
pubkey: '0xab0bdda0f85f842f431beaccf1250bf1fd7ba51b4100fd64364b6401fda85bb0069b3e715b58819684e7fc0b10a72a34'}
- {privkey: '0x1023c68852075965e0f7352dee3f76a84a83e7582c181c10179936c6d6348893',
pubkey: '0x9977f1c8b731a8d5558146bfb86caea26434f3c5878b589bf280a42c9159e700e9df0e4086296c20b011d2e78c27d373'}
- {privkey: '0x3a941600dc41e5d20e818473b817a28507c23cdfdb4b659c15461ee5c71e41f5',
pubkey: '0xa8d4c7c27795a725961317ef5953a7032ed6d83739db8b0e8a72353d1b8b4439427f7efa2c89caa03cc9f28f8cbab8ac'}
- {privkey: '0x066e3bdc0415530e5c7fed6382d5c822c192b620203cf669903e1810a8c67d06',
pubkey: '0xa6d310dbbfab9a22450f59993f87a4ce5db6223f3b5f1f30d2c4ec718922d400e0b3c7741de8e59960f72411a0ee10a7'}
- {privkey: '0x2b3b88a041168a1c4cd04bdd8de7964fd35238f95442dc678514f9dadb81ec34',
pubkey: '0x9893413c00283a3f9ed9fd9845dda1cea38228d22567f9541dccc357e54a2d6a6e204103c92564cbc05f4905ac7c493a'}

View File

@ -22,8 +22,13 @@ extern crate lazy_static;
use eth2_hashing::hash;
use milagro_bls::{Keypair, PublicKey, SecretKey};
use num_bigint::BigUint;
use serde_derive::{Deserialize, Serialize};
use std::convert::TryInto;
use std::fs::File;
use std::path::PathBuf;
pub const PRIVATE_KEY_BYTES: usize = 48;
pub const PUBLIC_KEY_BYTES: usize = 48;
pub const HASH_BYTES: usize = 32;
lazy_static! {
@ -63,3 +68,65 @@ pub fn keypair(validator_index: usize) -> Keypair {
sk,
}
}
#[derive(Serialize, Deserialize)]
struct YamlKeypair {
/// Big-endian.
privkey: String,
/// Big-endian.
pubkey: String,
}
impl TryInto<Keypair> for YamlKeypair {
type Error = String;
fn try_into(self) -> Result<Keypair, Self::Error> {
let privkey = string_to_bytes(&self.privkey)?;
let pubkey = string_to_bytes(&self.pubkey)?;
if (privkey.len() > PRIVATE_KEY_BYTES) || (pubkey.len() > PUBLIC_KEY_BYTES) {
return Err("Public or private key is too long".into());
}
let sk = {
let mut bytes = vec![0; PRIVATE_KEY_BYTES - privkey.len()];
bytes.extend_from_slice(&privkey);
SecretKey::from_bytes(&bytes)
.map_err(|e| format!("Failed to decode bytes into secret key: {:?}", e))?
};
let pk = {
let mut bytes = vec![0; PUBLIC_KEY_BYTES - pubkey.len()];
bytes.extend_from_slice(&pubkey);
PublicKey::from_bytes(&bytes)
.map_err(|e| format!("Failed to decode bytes into public key: {:?}", e))?
};
Ok(Keypair { pk, sk })
}
}
fn string_to_bytes(string: &str) -> Result<Vec<u8>, String> {
let string = if string.starts_with("0x") {
&string[2..]
} else {
string
};
hex::decode(string).map_err(|e| format!("Unable to decode public or private key: {}", e))
}
/// Loads keypairs from a YAML encoded file.
///
/// Uses this as reference:
/// https://github.com/ethereum/eth2.0-pm/blob/9a9dbcd95e2b8e10287797bd768014ab3d842e99/interop/mocked_start/keygen_10_validators.yaml
pub fn keypairs_from_yaml_file(path: PathBuf) -> Result<Vec<Keypair>, String> {
let file =
File::open(path.clone()).map_err(|e| format!("Unable to open YAML key file: {}", e))?;
serde_yaml::from_reader::<_, Vec<YamlKeypair>>(file)
.map_err(|e| format!("Could not parse YAML: {:?}", e))?
.into_iter()
.map(TryInto::try_into)
.collect::<Result<Vec<_>, String>>()
}

View File

@ -0,0 +1,23 @@
#![cfg(test)]
use eth2_interop_keypairs::{keypair as reference_keypair, keypairs_from_yaml_file};
use std::path::PathBuf;
fn yaml_path() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("specs")
.join("keygen_10_validators.yaml")
}
#[test]
fn load_from_yaml() {
let keypairs = keypairs_from_yaml_file(yaml_path()).expect("should read keypairs from file");
keypairs.into_iter().enumerate().for_each(|(i, keypair)| {
assert_eq!(
keypair,
reference_keypair(i),
"Decoded key {} does not match generated key",
i
)
});
}

96
scripts/whiteblock_start.sh Executable file
View File

@ -0,0 +1,96 @@
#!/bin/bash
<<COMMENT
Used by Whiteblock for simulated network testing.
Based upon:
https://github.com/whiteblock/dockerfiles/blob/a31b412d32d0384de12aa8392e43bac32837b6bc/ethereum/interop-example/launch/start.sh
Here's an example script used for testing:
./whiteblock_start.sh \
--identity=55c7fc76505ddeb6cf750b1f9f43d6d12c1a53b77ada018a390d7592a7f36dbck \
--peers=/ip4/192.168.0.1/tcp/9000 \
--validator-keys=/tmp/keygen_10_validators.yaml \
--gen-state=/tmp/genesis.ssz \
--port=9008
The example script was run in the target/release directory of lighthouse.
The following change was made to this script:
YAML_KEY_FILE="/tmp/keygen_10_validators.yaml"
COMMENT
# Flags
IDENTITY=""
PEERS=""
YAML_KEY_FILE="/tmp/keygen_10_validators.yaml"
GEN_STATE=""
PORT="8000"
# Constants
BEACON_LOG_FILE="/tmp/beacon.log"
VALIDATOR_LOG_FILE="/tmp/validator.log"
usage() {
echo "--identity=<hex prepresentation of the priv key for libp2p>"
echo "--peers=<peer>"
echo "--validator-keys=<path to /launch/keys.yaml>"
echo "--gen-state=<path to /launch/state.ssz>"
echo "--port=<port>"
}
while [ "$1" != "" ];
do
PARAM=`echo $1 | awk -F= '{print $1}'`
VALUE=`echo $1 | sed 's/^[^=]*=//g'`
case $PARAM in
--identity)
IDENTITY=$VALUE
;;
--peers)
PEERS+=",$VALUE"
;;
--validator-keys)
VALIDATOR_KEYS=$VALUE
;;
--gen-state)
GEN_STATE=$VALUE
;;
--port)
PORT=$VALUE
;;
--help)
usage
exit
;;
*)
echo "ERROR: unknown parameter \"$PARAM\""
usage
exit 1
;;
esac
shift
done
./beacon_node \
--p2p-priv-key $IDENTITY \
--logfile $BEACON_LOG_FILE \
--libp2p-addresses $PEERS \
--port $PORT \
testnet \
--force \
file \
ssz \
$GEN_STATE \
& \
./validator_client \
--logfile $VALIDATOR_LOG_FILE \
testnet \
--bootstrap \
interop-yaml \
$YAML_KEY_FILE \
trap 'trap - SIGTERM && kill 0' SIGINT SIGTERM EXIT

View File

@ -19,6 +19,7 @@ eth2_config = { path = "../eth2/utils/eth2_config" }
tree_hash = "0.1"
clap = "2.32.0"
lighthouse_bootstrap = { path = "../eth2/utils/lighthouse_bootstrap" }
eth2_interop_keypairs = { path = "../eth2/utils/eth2_interop_keypairs" }
grpcio = { version = "0.4", default-features = false, features = ["protobuf-codec"] }
protos = { path = "../protos" }
slot_clock = { path = "../eth2/utils/slot_clock" }

View File

@ -8,7 +8,10 @@ use std::io::{Error, ErrorKind};
use std::ops::Range;
use std::path::PathBuf;
use std::sync::Mutex;
use types::{test_utils::generate_deterministic_keypair, EthSpec, MainnetEthSpec};
use types::{
test_utils::{generate_deterministic_keypair, load_keypairs_from_yaml},
EthSpec, MainnetEthSpec,
};
pub const DEFAULT_SERVER: &str = "localhost";
pub const DEFAULT_SERVER_GRPC_PORT: &str = "5051";
@ -20,6 +23,8 @@ pub enum KeySource {
Disk,
/// Generate the keypairs (insecure, generates predictable keys).
TestingKeypairRange(Range<usize>),
/// Load testing keypairs from YAML
YamlKeypairs(PathBuf),
}
impl Default for KeySource {
@ -230,6 +235,14 @@ impl Config {
warn!(log, "Using insecure private keys");
self.fetch_testing_keypairs(range.clone())?
}
KeySource::YamlKeypairs(path) => {
warn!(
log,
"Private keys are stored insecurely (plain text). Testing use only."
);
load_keypairs_from_yaml(path.to_path_buf())?
}
};
// Check if it's an empty vector, and return none.

View File

@ -16,6 +16,7 @@ use eth2_config::Eth2Config;
use lighthouse_bootstrap::Bootstrapper;
use protos::services_grpc::ValidatorServiceClient;
use slog::{crit, error, info, o, Drain, Level, Logger};
use std::path::PathBuf;
use types::{InteropEthSpec, Keypair, MainnetEthSpec, MinimalEthSpec};
pub const DEFAULT_SPEC: &str = "minimal";
@ -131,6 +132,14 @@ fn main() {
.required(true)
.help("The number of validators."))
)
.subcommand(SubCommand::with_name("interop-yaml")
.about("Loads plain-text secret keys from YAML files. Expects the interop format defined
in the ethereum/eth2.0-pm repo.")
.arg(Arg::with_name("path")
.value_name("PATH")
.required(true)
.help("Path to a YAML file."))
)
)
.get_matches();
@ -143,8 +152,8 @@ fn main() {
Some("crit") => drain.filter_level(Level::Critical),
_ => unreachable!("guarded by clap"),
};
let log = slog::Logger::root(drain.fuse(), o!());
let (client_config, eth2_config) = match get_configs(&matches, &log) {
let mut log = slog::Logger::root(drain.fuse(), o!());
let (client_config, eth2_config) = match get_configs(&matches, &mut log) {
Ok(tuple) => tuple,
Err(e) => {
crit!(
@ -195,9 +204,14 @@ fn main() {
/// Parses the CLI arguments and attempts to load the client and eth2 configuration.
///
/// This is not a pure function, it reads from disk and may contact network servers.
pub fn get_configs(cli_args: &ArgMatches, log: &Logger) -> Result<(ClientConfig, Eth2Config)> {
pub fn get_configs(
cli_args: &ArgMatches,
mut log: &mut Logger,
) -> Result<(ClientConfig, Eth2Config)> {
let mut client_config = ClientConfig::default();
client_config.apply_cli_args(&cli_args, &mut log)?;
if let Some(server) = cli_args.value_of("server") {
client_config.server = server.to_string();
}
@ -215,14 +229,14 @@ pub fn get_configs(cli_args: &ArgMatches, log: &Logger) -> Result<(ClientConfig,
}
info!(
log,
*log,
"Beacon node connection info";
"grpc_port" => client_config.server_grpc_port,
"http_port" => client_config.server_http_port,
"server" => &client_config.server,
);
match cli_args.subcommand() {
let (client_config, eth2_config) = match cli_args.subcommand() {
("testnet", Some(sub_cli_args)) => {
if cli_args.is_present("eth2-config") && sub_cli_args.is_present("bootstrap") {
return Err(
@ -234,7 +248,9 @@ pub fn get_configs(cli_args: &ArgMatches, log: &Logger) -> Result<(ClientConfig,
process_testnet_subcommand(sub_cli_args, client_config, log)
}
_ => return Err("You must use the testnet command. See '--help'.".into()),
}
}?;
Ok((client_config, eth2_config))
}
/// Parses the `testnet` CLI subcommand.
@ -296,6 +312,21 @@ fn process_testnet_subcommand(
KeySource::TestingKeypairRange(first..first + count)
}
("interop-yaml", Some(sub_cli_args)) => {
let path = sub_cli_args
.value_of("path")
.ok_or_else(|| "No yaml path supplied")?
.parse::<PathBuf>()
.map_err(|e| format!("Unable to parse yaml path: {:?}", e))?;
info!(
log,
"Loading keypairs from interop YAML format";
"path" => format!("{:?}", path),
);
KeySource::YamlKeypairs(path)
}
_ => KeySource::Disk,
};