cerc-io/lighthouse
16
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'interop' into api-alignment

Browse Source
This commit is contained in:
Luke Anderson 2019-09-12 01:32:29 +10:00
commit d9b7e684c9
No known key found for this signature in database
GPG Key ID: 44408169EC61E228
34 changed files with 881 additions and 310 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.toml
View File

@ -34,7 +34,7 @@ members = [
"beacon_node/version",
"beacon_node/beacon_chain",
"tests/ef_tests",
"tests/cli_util",
"tests/lcli",
"protos",
"validator_client",
"account_manager",

218
beacon_node/beacon_chain/src/beacon_chain.rs
View File

@ -8,8 +8,8 @@ use crate::persisted_beacon_chain::{PersistedBeaconChain, BEACON_CHAIN_DB_KEY};
use lmd_ghost::LmdGhost;
use operation_pool::DepositInsertStatus;
use operation_pool::{OperationPool, PersistedOperationPool};
use parking_lot::{RwLock, RwLockReadGuard};
use slog::{error, info, warn, Logger};
use parking_lot::RwLock;
use slog::{error, info, trace, warn, Logger};
use slot_clock::SlotClock;
use ssz::Encode;
use state_processing::per_block_processing::{
@ -37,6 +37,12 @@ use types::*;
// |-------must be this long------|
pub const GRAFFITI: &str = "sigp/lighthouse-0.0.0-prerelease";
/// If true, everytime a block is processed the pre-state, post-state and block are written to SSZ
/// files in the temp directory.
///
/// Only useful for testing.
const WRITE_BLOCK_PROCESSING_SSZ: bool = true;
#[derive(Debug, PartialEq)]
pub enum BlockProcessingOutcome {
/// Block was valid and imported into the block graph.
@ -83,35 +89,6 @@ pub enum AttestationProcessingOutcome {
Invalid(AttestationValidationError),
}
/// Effectively a `Cow<BeaconState>`, however when it is `Borrowed` it holds a `RwLockReadGuard` (a
/// read-lock on some read/write-locked state).
///
/// Only has a small subset of the functionality of a `std::borrow::Cow`.
pub enum BeaconStateCow<'a, T: EthSpec> {
Borrowed(RwLockReadGuard<'a, CheckPoint<T>>),
Owned(BeaconState<T>),
}
impl<'a, T: EthSpec> BeaconStateCow<'a, T> {
pub fn maybe_as_mut_ref(&mut self) -> Option<&mut BeaconState<T>> {
match self {
BeaconStateCow::Borrowed(_) => None,
BeaconStateCow::Owned(ref mut state) => Some(state),
}
}
}
impl<'a, T: EthSpec> std::ops::Deref for BeaconStateCow<'a, T> {
type Target = BeaconState<T>;
fn deref(&self) -> &BeaconState<T> {
match self {
BeaconStateCow::Borrowed(checkpoint) => &checkpoint.beacon_state,
BeaconStateCow::Owned(state) => &state,
}
}
}
pub trait BeaconChainTypes: Send + Sync + 'static {
type Store: store::Store;
type SlotClock: slot_clock::SlotClock;
@ -332,13 +309,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
/// - As this iterator starts at the `head` of the chain (viz., the best block), the first slot
/// returned may be earlier than the wall-clock slot.
pub fn rev_iter_block_roots(&self) -> ReverseBlockRootIterator<T::EthSpec, T::Store> {
let state = &self.head().beacon_state;
let block_root = self.head().beacon_block_root;
let block_slot = state.slot;
let head = self.head();
let iter = BlockRootsIterator::owned(self.store.clone(), state.clone());
let iter = BlockRootsIterator::owned(self.store.clone(), head.beacon_state);
ReverseBlockRootIterator::new((block_root, block_slot), iter)
ReverseBlockRootIterator::new((head.beacon_block_root, head.beacon_block.slot), iter)
}
/// Iterates across all `(state_root, slot)` pairs from the head of the chain (inclusive) to
@ -351,13 +326,12 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
/// - As this iterator starts at the `head` of the chain (viz., the best block), the first slot
/// returned may be earlier than the wall-clock slot.
pub fn rev_iter_state_roots(&self) -> ReverseStateRootIterator<T::EthSpec, T::Store> {
let state = &self.head().beacon_state;
let state_root = self.head().beacon_state_root;
let state_slot = state.slot;
let head = self.head();
let slot = head.beacon_state.slot;
let iter = StateRootsIterator::owned(self.store.clone(), state.clone());
let iter = StateRootsIterator::owned(self.store.clone(), head.beacon_state);
ReverseStateRootIterator::new((state_root, state_slot), iter)
ReverseStateRootIterator::new((head.beacon_state_root, slot), iter)
}
/// Returns the block at the given root, if any.
@ -372,32 +346,25 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
Ok(self.store.get(block_root)?)
}
/// Returns a read-lock guarded `CheckPoint` struct for reading the head (as chosen by the
/// fork-choice rule).
/// Returns a `Checkpoint` representing the head block and state. Contains the "best block";
/// the head of the canonical `BeaconChain`.
///
/// It is important to note that the `beacon_state` returned may not match the present slot. It
/// is the state as it was when the head block was received, which could be some slots prior to
/// now.
pub fn head<'a>(&'a self) -> RwLockReadGuard<'a, CheckPoint<T::EthSpec>> {
self.canonical_head.read()
pub fn head(&self) -> CheckPoint<T::EthSpec> {
self.canonical_head.read().clone()
}
/// Returns the `BeaconState` at the given slot.
///
/// May return:
///
/// - A new state loaded from the database (for states prior to the head)
/// - A reference to the head state (note: this keeps a read lock on the head, try to use
/// sparingly).
/// - The head state, but with skipped slots (for states later than the head).
///
/// Returns `None` when the state is not found in the database or there is an error skipping
/// to a future state.
pub fn state_at_slot(&self, slot: Slot) -> Result<BeaconStateCow<T::EthSpec>, Error> {
let head_state = &self.head().beacon_state;
pub fn state_at_slot(&self, slot: Slot) -> Result<BeaconState<T::EthSpec>, Error> {
let head_state = self.head().beacon_state;
if slot == head_state.slot {
Ok(BeaconStateCow::Borrowed(self.head()))
Ok(head_state)
} else if slot > head_state.slot {
let head_state_slot = head_state.slot;
let mut state = head_state.clone();
@ -417,7 +384,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
}
};
}
Ok(BeaconStateCow::Owned(state))
Ok(state)
} else {
let state_root = self
.rev_iter_state_roots()
@ -425,11 +392,10 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
.map(|(root, _slot)| root)
.ok_or_else(|| Error::NoStateForSlot(slot))?;
Ok(BeaconStateCow::Owned(
self.store
.get(&state_root)?
.ok_or_else(|| Error::NoStateForSlot(slot))?,
))
Ok(self
.store
.get(&state_root)?
.ok_or_else(|| Error::NoStateForSlot(slot))?)
}
}
@ -441,7 +407,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
///
/// Returns `None` when there is an error skipping to a future state or the slot clock cannot
/// be read.
pub fn state_now(&self) -> Result<BeaconStateCow<T::EthSpec>, Error> {
pub fn wall_clock_state(&self) -> Result<BeaconState<T::EthSpec>, Error> {
self.state_at_slot(self.slot()?)
}
@ -493,14 +459,12 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
let head_state = &self.head().beacon_state;
let mut state = if epoch(slot) == epoch(head_state.slot) {
BeaconStateCow::Borrowed(self.head())
self.head().beacon_state.clone()
} else {
self.state_at_slot(slot)?
};
if let Some(state) = state.maybe_as_mut_ref() {
state.build_committee_cache(RelativeEpoch::Current, &self.spec)?;
}
state.build_committee_cache(RelativeEpoch::Current, &self.spec)?;
if epoch(state.slot) != epoch(slot) {
return Err(Error::InvariantViolated(format!(
@ -528,14 +492,12 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
let head_state = &self.head().beacon_state;
let mut state = if epoch == as_epoch(head_state.slot) {
BeaconStateCow::Borrowed(self.head())
self.head().beacon_state.clone()
} else {
self.state_at_slot(epoch.start_slot(T::EthSpec::slots_per_epoch()))?
};
if let Some(state) = state.maybe_as_mut_ref() {
state.build_committee_cache(RelativeEpoch::Current, &self.spec)?;
}
state.build_committee_cache(RelativeEpoch::Current, &self.spec)?;
if as_epoch(state.slot) != epoch {
return Err(Error::InvariantViolated(format!(
@ -563,11 +525,14 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
slot: Slot,
) -> Result<AttestationData, Error> {
let state = self.state_at_slot(slot)?;
let head = self.head();
let head_block_root = self.head().beacon_block_root;
let head_block_slot = self.head().beacon_block.slot;
self.produce_attestation_data_for_block(shard, head_block_root, head_block_slot, &*state)
self.produce_attestation_data_for_block(
shard,
head.beacon_block_root,
head.beacon_block.slot,
&state,
)
}
/// Produce an `AttestationData` that attests to the chain denoted by `block_root` and `state`.
@ -633,6 +598,14 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
metrics::inc_counter(&metrics::ATTESTATION_PRODUCTION_SUCCESSES);
metrics::stop_timer(timer);
trace!(
self.log,
"Produced beacon attestation data";
"beacon_block_root" => format!("{}", head_block_root),
"shard" => shard,
"slot" => state.slot
);
Ok(AttestationData {
beacon_block_root: head_block_root,
source: state.current_justified_checkpoint.clone(),
@ -745,7 +718,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
// has a higher slot than the attestation.
//
// Permitting this would allow for attesters to vote on _future_ slots.
if attestation_slot > state.slot {
if state.slot > attestation_slot {
Ok(AttestationProcessingOutcome::AttestsToFutureState {
state: state.slot,
attestation: attestation_slot,
@ -886,10 +859,8 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
/// Accept some exit and queue it for inclusion in an appropriate block.
pub fn process_voluntary_exit(&self, exit: VoluntaryExit) -> Result<(), ExitValidationError> {
match self.state_now() {
Ok(state) => self
.op_pool
.insert_voluntary_exit(exit, &*state, &self.spec),
match self.wall_clock_state() {
Ok(state) => self.op_pool.insert_voluntary_exit(exit, &state, &self.spec),
Err(e) => {
error!(
&self.log,
@ -904,8 +875,8 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
/// Accept some transfer and queue it for inclusion in an appropriate block.
pub fn process_transfer(&self, transfer: Transfer) -> Result<(), TransferValidationError> {
match self.state_now() {
Ok(state) => self.op_pool.insert_transfer(transfer, &*state, &self.spec),
match self.wall_clock_state() {
Ok(state) => self.op_pool.insert_transfer(transfer, &state, &self.spec),
Err(e) => {
error!(
&self.log,
@ -923,10 +894,10 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
&self,
proposer_slashing: ProposerSlashing,
) -> Result<(), ProposerSlashingValidationError> {
match self.state_now() {
match self.wall_clock_state() {
Ok(state) => {
self.op_pool
.insert_proposer_slashing(proposer_slashing, &*state, &self.spec)
.insert_proposer_slashing(proposer_slashing, &state, &self.spec)
}
Err(e) => {
error!(
@ -945,10 +916,10 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
&self,
attester_slashing: AttesterSlashing<T::EthSpec>,
) -> Result<(), AttesterSlashingValidationError> {
match self.state_now() {
match self.wall_clock_state() {
Ok(state) => {
self.op_pool
.insert_attester_slashing(attester_slashing, &*state, &self.spec)
.insert_attester_slashing(attester_slashing, &state, &self.spec)
}
Err(e) => {
error!(
@ -1264,6 +1235,14 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
metrics::inc_counter(&metrics::BLOCK_PRODUCTION_SUCCESSES);
metrics::stop_timer(timer);
trace!(
self.log,
"Produced beacon block";
"parent" => format!("{}", block.parent_root),
"attestations" => block.body.attestations.len(),
"slot" => block.slot
);
Ok((block, state))
}
@ -1301,15 +1280,20 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
warn!(
self.log,
"Beacon chain re-org";
"previous_head" => format!("{}", self.head().beacon_block_root),
"previous_slot" => previous_slot,
"new_head_parent" => format!("{}", beacon_block.parent_root),
"new_head" => format!("{}", beacon_block_root),
"new_slot" => new_slot
);
} else {
info!(
self.log,
"new head block";
"New head beacon block";
"justified_root" => format!("{}", beacon_state.current_justified_checkpoint.root),
"justified_epoch" => beacon_state.current_justified_checkpoint.epoch,
"finalized_root" => format!("{}", beacon_state.finalized_checkpoint.root),
"finalized_epoch" => beacon_state.finalized_checkpoint.epoch,
"root" => format!("{}", beacon_block_root),
"slot" => new_slot,
);
@ -1359,10 +1343,14 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
new_head.beacon_state.build_all_caches(&self.spec)?;
trace!(self.log, "Taking write lock on head");
// Update the checkpoint that stores the head of the chain at the time it received the
// block.
*self.canonical_head.write() = new_head;
trace!(self.log, "Dropping write lock on head");
// Save `self` to `self.store`.
self.persist()?;
@ -1463,41 +1451,45 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
}
fn write_state<T: EthSpec>(prefix: &str, state: &BeaconState<T>, log: &Logger) {
let root = Hash256::from_slice(&state.tree_hash_root());
let filename = format!("{}_slot_{}_root_{}.ssz", prefix, state.slot, root);
let mut path = std::env::temp_dir().join("lighthouse");
let _ = fs::create_dir_all(path.clone());
path = path.join(filename);
if WRITE_BLOCK_PROCESSING_SSZ {
let root = Hash256::from_slice(&state.tree_hash_root());
let filename = format!("{}_slot_{}_root_{}.ssz", prefix, state.slot, root);
let mut path = std::env::temp_dir().join("lighthouse");
let _ = fs::create_dir_all(path.clone());
path = path.join(filename);
match fs::File::create(path.clone()) {
Ok(mut file) => {
let _ = file.write_all(&state.as_ssz_bytes());
match fs::File::create(path.clone()) {
Ok(mut file) => {
let _ = file.write_all(&state.as_ssz_bytes());
}
Err(e) => error!(
log,
"Failed to log state";
"path" => format!("{:?}", path),
"error" => format!("{:?}", e)
),
}
Err(e) => error!(
log,
"Failed to log state";
"path" => format!("{:?}", path),
"error" => format!("{:?}", e)
),
}
}
fn write_block<T: EthSpec>(block: &BeaconBlock<T>, root: Hash256, log: &Logger) {
let filename = format!("block_slot_{}_root{}.ssz", block.slot, root);
let mut path = std::env::temp_dir().join("lighthouse");
let _ = fs::create_dir_all(path.clone());
path = path.join(filename);
if WRITE_BLOCK_PROCESSING_SSZ {
let filename = format!("block_slot_{}_root{}.ssz", block.slot, root);
let mut path = std::env::temp_dir().join("lighthouse");
let _ = fs::create_dir_all(path.clone());
path = path.join(filename);
match fs::File::create(path.clone()) {
Ok(mut file) => {
let _ = file.write_all(&block.as_ssz_bytes());
match fs::File::create(path.clone()) {
Ok(mut file) => {
let _ = file.write_all(&block.as_ssz_bytes());
}
Err(e) => error!(
log,
"Failed to log block";
"path" => format!("{:?}", path),
"error" => format!("{:?}", e)
),
}
Err(e) => error!(
log,
"Failed to log block";
"path" => format!("{:?}", path),
"error" => format!("{:?}", e)
),
}
}

5
beacon_node/eth2-libp2p/Cargo.toml
View File

@ -6,9 +6,10 @@ edition = "2018"
[dependencies]
clap = "2.32.0"
hex = "0.3"
#SigP repository
libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "76f7475e4b7063e663ad03c7524cf091f9961968" }
enr = { git = "https://github.com/SigP/rust-libp2p/", rev = "76f7475e4b7063e663ad03c7524cf091f9961968", features = ["serde"] }
libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "8ac9c744197faaadc0e2b64fed7470ac4e2a41ca" }
enr = { git = "https://github.com/SigP/rust-libp2p/", rev = "8ac9c744197faaadc0e2b64fed7470ac4e2a41ca", features = ["serde"] }
types = { path = "../../eth2/types" }
serde = "1.0"
serde_derive = "1.0"

11
beacon_node/eth2-libp2p/src/config.rs
View File

@ -40,6 +40,12 @@ pub struct Config {
/// Target number of connected peers.
pub max_peers: usize,
/// A secp256k1 secret key, as bytes in ASCII-encoded hex.
///
/// With or without `0x` prefix.
#[serde(skip)]
pub secret_key_hex: Option<String>,
/// Gossipsub configuration parameters.
#[serde(skip)]
pub gs_config: GossipsubConfig,
@ -70,6 +76,7 @@ impl Default for Config {
discovery_address: "127.0.0.1".parse().expect("valid ip address"),
discovery_port: 9000,
max_peers: 10,
secret_key_hex: None,
// Note: The topics by default are sent as plain strings. Hashes are an optional
// parameter.
gs_config: GossipsubConfigBuilder::new()
@ -158,6 +165,10 @@ impl Config {
.map_err(|_| format!("Invalid discovery port: {}", disc_port_str))?;
}
if let Some(p2p_priv_key) = args.value_of("p2p-priv-key") {
self.secret_key_hex = Some(p2p_priv_key.to_string());
}
Ok(())
}
}

35
beacon_node/eth2-libp2p/src/service.rs
View File

@ -42,16 +42,22 @@ impl Service {
pub fn new(config: NetworkConfig, log: slog::Logger) -> error::Result<Self> {
trace!(log, "Libp2p Service starting");
let local_keypair = if let Some(hex_bytes) = &config.secret_key_hex {
keypair_from_hex(hex_bytes)?
} else {
load_private_key(&config, &log)
};
// load the private key from CLI flag, disk or generate a new one
let local_private_key = load_private_key(&config, &log);
let local_peer_id = PeerId::from(local_private_key.public());
// let local_private_key = load_private_key(&config, &log);
let local_peer_id = PeerId::from(local_keypair.public());
info!(log, "Libp2p Service"; "peer_id" => format!("{:?}", local_peer_id));
let mut swarm = {
// Set up the transport - tcp/ws with secio and mplex/yamux
let transport = build_transport(local_private_key.clone());
let transport = build_transport(local_keypair.clone());
// Lighthouse network behaviour
let behaviour = Behaviour::new(&local_private_key, &config, &log)?;
let behaviour = Behaviour::new(&local_keypair, &config, &log)?;
Swarm::new(transport, behaviour, local_peer_id.clone())
};
@ -246,6 +252,27 @@ pub enum Libp2pEvent {
},
}
fn keypair_from_hex(hex_bytes: &str) -> error::Result<Keypair> {
let hex_bytes = if hex_bytes.starts_with("0x") {
hex_bytes[2..].to_string()
} else {
hex_bytes.to_string()
};
hex::decode(&hex_bytes)
.map_err(|e| format!("Failed to parse p2p secret key bytes: {:?}", e).into())
.and_then(keypair_from_bytes)
}
fn keypair_from_bytes(mut bytes: Vec<u8>) -> error::Result<Keypair> {
libp2p::core::identity::secp256k1::SecretKey::from_bytes(&mut bytes)
.map(|secret| {
let keypair: libp2p::core::identity::secp256k1::Keypair = secret.into();
Keypair::Secp256k1(keypair)
})
.map_err(|e| format!("Unable to parse p2p secret key: {:?}", e).into())
}
/// Loads a private key from disk. If this fails, a new key is
/// generated and is then saved to disk.
///

2
beacon_node/rpc/src/validator.rs
View File

@ -25,8 +25,8 @@ impl<T: BeaconChainTypes> ValidatorService for ValidatorServiceInstance<T> {
req: GetDutiesRequest,
sink: UnarySink<GetDutiesResponse>,
) {
let validators = req.get_validators();
trace!(self.log, "RPC request"; "endpoint" => "GetValidatorDuties", "epoch" => req.get_epoch());
let validators = req.get_validators();
let epoch = Epoch::from(req.get_epoch());
let slot = epoch.start_slot(T::EthSpec::slots_per_epoch());

23
beacon_node/src/config.rs
View File

@ -13,7 +13,7 @@ pub const CLIENT_CONFIG_FILENAME: &str = "beacon-node.toml";
pub const ETH2_CONFIG_FILENAME: &str = "eth2-spec.toml";
type Result<T> = std::result::Result<T, String>;
type Config = (ClientConfig, Eth2Config);
type Config = (ClientConfig, Eth2Config, Logger);
/// Gets the fully-initialized global client and eth2 configuration objects.
///
@ -22,8 +22,10 @@ type Config = (ClientConfig, Eth2Config);
/// The output of this function depends primarily upon the given `cli_args`, however it's behaviour
/// may be influenced by other external services like the contents of the file system or the
/// response of some remote server.
pub fn get_configs(cli_args: &ArgMatches, log: &Logger) -> Result<Config> {
let mut builder = ConfigBuilder::new(cli_args, log)?;
pub fn get_configs(cli_args: &ArgMatches, core_log: Logger) -> Result<Config> {
let log = core_log.clone();
let mut builder = ConfigBuilder::new(cli_args, core_log)?;
if let Some(server) = cli_args.value_of("eth1-server") {
builder.set_eth1_backend_method(Eth1BackendMethod::Web3 {
@ -35,7 +37,7 @@ pub fn get_configs(cli_args: &ArgMatches, log: &Logger) -> Result<Config> {
match cli_args.subcommand() {
("testnet", Some(sub_cmd_args)) => {
process_testnet_subcommand(&mut builder, sub_cmd_args, log)?
process_testnet_subcommand(&mut builder, sub_cmd_args, &log)?
}
// No sub-command assumes a resume operation.
_ => {
@ -216,15 +218,15 @@ fn process_testnet_subcommand(
}
/// Allows for building a set of configurations based upon `clap` arguments.
struct ConfigBuilder<'a> {
log: &'a Logger,
struct ConfigBuilder {
log: Logger,
eth2_config: Eth2Config,
client_config: ClientConfig,
}
impl<'a> ConfigBuilder<'a> {
impl ConfigBuilder {
/// Create a new builder with default settings.
pub fn new(cli_args: &'a ArgMatches, log: &'a Logger) -> Result<Self> {
pub fn new(cli_args: &ArgMatches, log: Logger) -> Result<Self> {
// Read the `--datadir` flag.
//
// If it's not present, try and find the home directory (`~`) and push the default data
@ -539,8 +541,7 @@ impl<'a> ConfigBuilder<'a> {
/// cli_args).
pub fn build(mut self, cli_args: &ArgMatches) -> Result<Config> {
self.eth2_config.apply_cli_args(cli_args)?;
self.client_config
.apply_cli_args(cli_args, &mut self.log.clone())?;
self.client_config.apply_cli_args(cli_args, &mut self.log)?;
if let Some(bump) = cli_args.value_of("port-bump") {
let bump = bump
@ -561,7 +562,7 @@ impl<'a> ConfigBuilder<'a> {
return Err("Specification constant mismatch".into());
}
Ok((self.client_config, self.eth2_config))
Ok((self.client_config, self.eth2_config, self.log))
}
}

13
beacon_node/src/main.rs
View File

@ -116,6 +116,13 @@ fn main() {
.help("One or more comma-delimited multiaddrs to manually connect to a libp2p peer without an ENR.")
.takes_value(true),
)
.arg(
Arg::with_name("p2p-priv-key")
.long("p2p-priv-key")
.value_name("HEX")
.help("A secp256k1 secret key, represented as ASCII-encoded hex bytes (with or without 0x prefix).")
.takes_value(true),
)
/*
* gRPC parameters.
*/
@ -355,13 +362,15 @@ fn main() {
"Ethereum 2.0 is pre-release. This software is experimental."
);
let log_clone = log.clone();
// Load the process-wide configuration.
//
// May load this from disk or create a new configuration, depending on the CLI flags supplied.
let (client_config, eth2_config) = match get_configs(&matches, &log) {
let (client_config, eth2_config, log) = match get_configs(&matches, log) {
Ok(configs) => configs,
Err(e) => {
crit!(log, "Failed to load configuration"; "error" => e);
crit!(log_clone, "Failed to load configuration. Exiting"; "error" => e);
return;
}
};

8
eth2/state_processing/src/per_block_processing/signature_sets.rs
View File

@ -42,8 +42,12 @@ pub fn block_proposal_signature_set<'a, T: EthSpec>(
block_signed_root: Option<Hash256>,
spec: &'a ChainSpec,
) -> Result<SignatureSet<'a>> {
let block_proposer = &state.validators
[state.get_beacon_proposer_index(block.slot, RelativeEpoch::Current, spec)?];
let proposer_index =
state.get_beacon_proposer_index(block.slot, RelativeEpoch::Current, spec)?;
let block_proposer = &state
.validators
.get(proposer_index)
.ok_or_else(|| Error::ValidatorUnknown(proposer_index as u64))?;
let domain = spec.get_domain(
block.slot.epoch(T::slots_per_epoch()),

16
eth2/state_processing/src/per_epoch_processing.rs
View File

@ -48,15 +48,10 @@ pub fn per_epoch_processing<T: EthSpec>(
process_justification_and_finalization(state, &validator_statuses.total_balances)?;
// Crosslinks.
let winning_root_for_shards = process_crosslinks(state, spec)?;
process_crosslinks(state, spec)?;
// Rewards and Penalties.
process_rewards_and_penalties(
state,
&mut validator_statuses,
&winning_root_for_shards,
spec,
)?;
process_rewards_and_penalties(state, &mut validator_statuses, spec)?;
// Registry Updates.
process_registry_updates(state, spec)?;
@ -160,9 +155,7 @@ pub fn process_justification_and_finalization<T: EthSpec>(
pub fn process_crosslinks<T: EthSpec>(
state: &mut BeaconState<T>,
spec: &ChainSpec,
) -> Result<WinningRootHashSet, Error> {
let mut winning_root_for_shards: WinningRootHashSet = HashMap::new();
) -> Result<(), Error> {
state.previous_crosslinks = state.current_crosslinks.clone();
for &relative_epoch in &[RelativeEpoch::Previous, RelativeEpoch::Current] {
@ -182,12 +175,11 @@ pub fn process_crosslinks<T: EthSpec>(
if 3 * winning_root.total_attesting_balance >= 2 * total_committee_balance {
state.current_crosslinks[shard as usize] = winning_root.crosslink.clone();
}
winning_root_for_shards.insert(shard, winning_root);
}
}
}
Ok(winning_root_for_shards)
Ok(())
}
/// Finish up an epoch update.

19
eth2/state_processing/src/per_epoch_processing/apply_rewards.rs
View File

@ -1,5 +1,5 @@
use super::validator_statuses::{TotalBalances, ValidatorStatus, ValidatorStatuses};
use super::{Error, WinningRootHashSet};
use super::Error;
use integer_sqrt::IntegerSquareRoot;
use types::*;
@ -36,7 +36,6 @@ impl std::ops::AddAssign for Delta {
pub fn process_rewards_and_penalties<T: EthSpec>(
state: &mut BeaconState<T>,
validator_statuses: &mut ValidatorStatuses,
winning_root_for_shards: &WinningRootHashSet,
spec: &ChainSpec,
) -> Result<(), Error> {
if state.current_epoch() == T::genesis_epoch() {
@ -53,15 +52,13 @@ pub fn process_rewards_and_penalties<T: EthSpec>(
let mut deltas = vec![Delta::default(); state.balances.len()];
get_attestation_deltas(&mut deltas, state, &validator_statuses, spec)?;
// Update statuses with the information from winning roots.
validator_statuses.process_winning_roots(state, spec)?;
get_crosslink_deltas(&mut deltas, state, &validator_statuses, spec)?;
get_proposer_deltas(
&mut deltas,
state,
validator_statuses,
winning_root_for_shards,
spec,
)?;
get_proposer_deltas(&mut deltas, state, validator_statuses, spec)?;
// Apply the deltas, over-flowing but not under-flowing (saturating at 0 instead).
for (i, delta) in deltas.iter().enumerate() {
@ -79,12 +76,8 @@ fn get_proposer_deltas<T: EthSpec>(
deltas: &mut Vec<Delta>,
state: &BeaconState<T>,
validator_statuses: &mut ValidatorStatuses,
winning_root_for_shards: &WinningRootHashSet,
spec: &ChainSpec,
) -> Result<(), Error> {
// Update statuses with the information from winning roots.
validator_statuses.process_winning_roots(state, winning_root_for_shards, spec)?;
for (index, validator) in validator_statuses.statuses.iter().enumerate() {
if validator.is_previous_epoch_attester {
let inclusion = validator

26
eth2/state_processing/src/per_epoch_processing/validator_statuses.rs
View File

@ -1,4 +1,4 @@
use super::WinningRootHashSet;
use super::{winning_root::winning_root, WinningRootHashSet};
use crate::common::get_attesting_indices;
use types::*;
@ -292,9 +292,29 @@ impl ValidatorStatuses {
pub fn process_winning_roots<T: EthSpec>(
&mut self,
state: &BeaconState<T>,
winning_roots: &WinningRootHashSet,
spec: &ChainSpec,
) -> Result<(), BeaconStateError> {
// We must re-calculate the winning roots here because it is possible that they have
// changed since the first time they were calculated.
//
// This is because we altered the state during the first time we calculated the winning
// roots.
let winning_root_for_shards = {
let mut winning_root_for_shards = WinningRootHashSet::new();
let relative_epoch = RelativeEpoch::Previous;
let epoch = relative_epoch.into_epoch(state.current_epoch());
for offset in 0..state.get_committee_count(relative_epoch)? {
let shard = (state.get_epoch_start_shard(relative_epoch)? + offset)
% T::ShardCount::to_u64();
if let Some(winning_root) = winning_root(state, shard, epoch, spec)? {
winning_root_for_shards.insert(shard, winning_root);
}
}
winning_root_for_shards
};
// Loop through each slot in the previous epoch.
for slot in state.previous_epoch().slot_iter(T::slots_per_epoch()) {
let crosslink_committees_at_slot = state.get_crosslink_committees_at_slot(slot)?;
@ -302,7 +322,7 @@ impl ValidatorStatuses {
// Loop through each committee in the slot.
for c in crosslink_committees_at_slot {
// If there was some winning crosslink root for the committee's shard.
if let Some(winning_root) = winning_roots.get(&c.shard) {
if let Some(winning_root) = winning_root_for_shards.get(&c.shard) {
let total_committee_balance = state.get_total_balance(&c.committee, spec)?;
for &validator_index in &winning_root.attesting_validator_indices {
// Take note of the balance information for the winning root, it will be

2
eth2/state_processing/src/per_epoch_processing/winning_root.rs
View File

@ -3,7 +3,7 @@ use std::collections::{HashMap, HashSet};
use tree_hash::TreeHash;
use types::*;
#[derive(Clone)]
#[derive(Clone, Debug)]
pub struct WinningRoot {
pub crosslink: Crosslink,
pub attesting_validator_indices: Vec<usize>,

14
eth2/types/src/test_utils/generate_deterministic_keypairs.rs
View File

@ -1,7 +1,8 @@
use crate::*;
use eth2_interop_keypairs::keypair;
use eth2_interop_keypairs::{keypair, keypairs_from_yaml_file};
use log::debug;
use rayon::prelude::*;
use std::path::PathBuf;
/// Generates `validator_count` keypairs where the secret key is derived solely from the index of
/// the validator.
@ -32,3 +33,14 @@ pub fn generate_deterministic_keypair(validator_index: usize) -> Keypair {
sk: SecretKey::from_raw(raw.sk),
}
}
/// Loads a list of keypairs from file.
pub fn load_keypairs_from_yaml(path: PathBuf) -> Result<Vec<Keypair>, String> {
Ok(keypairs_from_yaml_file(path)?
.into_iter()
.map(|raw| Keypair {
pk: PublicKey::from_raw(raw.pk),
sk: SecretKey::from_raw(raw.sk),
})
.collect())
}

1
eth2/types/src/test_utils/mod.rs
View File

@ -8,6 +8,7 @@ mod test_random;
pub use builders::*;
pub use generate_deterministic_keypairs::generate_deterministic_keypair;
pub use generate_deterministic_keypairs::generate_deterministic_keypairs;
pub use generate_deterministic_keypairs::load_keypairs_from_yaml;
pub use keypairs_file::KeypairsFile;
pub use rand::{
RngCore,

7
eth2/utils/eth2_interop_keypairs/Cargo.toml
View File

@ -10,10 +10,11 @@ edition = "2018"
lazy_static = "1.4"
num-bigint = "0.2"
eth2_hashing = "0.1"
hex = "0.3"
milagro_bls = { git = "https://github.com/michaelsproul/milagro_bls", branch = "little-endian-v0.10" }
serde_yaml = "0.8"
serde = "1.0"
serde_derive = "1.0"
[dev-dependencies]
base64 = "0.10"
serde = "1.0"
serde_derive = "1.0"
serde_yaml = "0.8"

20
eth2/utils/eth2_interop_keypairs/specs/keygen_10_validators.yaml Normal file
View File

@ -0,0 +1,20 @@
- {privkey: '0x25295f0d1d592a90b333e26e85149708208e9f8e8bc18f6c77bd62f8ad7a6866',
pubkey: '0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c'}
- {privkey: '0x51d0b65185db6989ab0b560d6deed19c7ead0e24b9b6372cbecb1f26bdfad000',
pubkey: '0xb89bebc699769726a318c8e9971bd3171297c61aea4a6578a7a4f94b547dcba5bac16a89108b6b6a1fe3695d1a874a0b'}
- {privkey: '0x315ed405fafe339603932eebe8dbfd650ce5dafa561f6928664c75db85f97857',
pubkey: '0xa3a32b0f8b4ddb83f1a0a853d81dd725dfe577d4f4c3db8ece52ce2b026eca84815c1a7e8e92a4de3d755733bf7e4a9b'}
- {privkey: '0x25b1166a43c109cb330af8945d364722757c65ed2bfed5444b5a2f057f82d391',
pubkey: '0x88c141df77cd9d8d7a71a75c826c41a9c9f03c6ee1b180f3e7852f6a280099ded351b58d66e653af8e42816a4d8f532e'}
- {privkey: '0x3f5615898238c4c4f906b507ee917e9ea1bb69b93f1dbd11a34d229c3b06784b',
pubkey: '0x81283b7a20e1ca460ebd9bbd77005d557370cabb1f9a44f530c4c4c66230f675f8df8b4c2818851aa7d77a80ca5a4a5e'}
- {privkey: '0x055794614bc85ed5436c1f5cab586aab6ca84835788621091f4f3b813761e7a8',
pubkey: '0xab0bdda0f85f842f431beaccf1250bf1fd7ba51b4100fd64364b6401fda85bb0069b3e715b58819684e7fc0b10a72a34'}
- {privkey: '0x1023c68852075965e0f7352dee3f76a84a83e7582c181c10179936c6d6348893',
pubkey: '0x9977f1c8b731a8d5558146bfb86caea26434f3c5878b589bf280a42c9159e700e9df0e4086296c20b011d2e78c27d373'}
- {privkey: '0x3a941600dc41e5d20e818473b817a28507c23cdfdb4b659c15461ee5c71e41f5',
pubkey: '0xa8d4c7c27795a725961317ef5953a7032ed6d83739db8b0e8a72353d1b8b4439427f7efa2c89caa03cc9f28f8cbab8ac'}
- {privkey: '0x066e3bdc0415530e5c7fed6382d5c822c192b620203cf669903e1810a8c67d06',
pubkey: '0xa6d310dbbfab9a22450f59993f87a4ce5db6223f3b5f1f30d2c4ec718922d400e0b3c7741de8e59960f72411a0ee10a7'}
- {privkey: '0x2b3b88a041168a1c4cd04bdd8de7964fd35238f95442dc678514f9dadb81ec34',
pubkey: '0x9893413c00283a3f9ed9fd9845dda1cea38228d22567f9541dccc357e54a2d6a6e204103c92564cbc05f4905ac7c493a'}

67
eth2/utils/eth2_interop_keypairs/src/lib.rs
View File

@ -22,8 +22,13 @@ extern crate lazy_static;
use eth2_hashing::hash;
use milagro_bls::{Keypair, PublicKey, SecretKey};
use num_bigint::BigUint;
use serde_derive::{Deserialize, Serialize};
use std::convert::TryInto;
use std::fs::File;
use std::path::PathBuf;
pub const PRIVATE_KEY_BYTES: usize = 48;
pub const PUBLIC_KEY_BYTES: usize = 48;
pub const HASH_BYTES: usize = 32;
lazy_static! {
@ -63,3 +68,65 @@ pub fn keypair(validator_index: usize) -> Keypair {
sk,
}
}
#[derive(Serialize, Deserialize)]
struct YamlKeypair {
/// Big-endian.
privkey: String,
/// Big-endian.
pubkey: String,
}
impl TryInto<Keypair> for YamlKeypair {
type Error = String;
fn try_into(self) -> Result<Keypair, Self::Error> {
let privkey = string_to_bytes(&self.privkey)?;
let pubkey = string_to_bytes(&self.pubkey)?;
if (privkey.len() > PRIVATE_KEY_BYTES) || (pubkey.len() > PUBLIC_KEY_BYTES) {
return Err("Public or private key is too long".into());
}
let sk = {
let mut bytes = vec![0; PRIVATE_KEY_BYTES - privkey.len()];
bytes.extend_from_slice(&privkey);
SecretKey::from_bytes(&bytes)
.map_err(|e| format!("Failed to decode bytes into secret key: {:?}", e))?
};
let pk = {
let mut bytes = vec![0; PUBLIC_KEY_BYTES - pubkey.len()];
bytes.extend_from_slice(&pubkey);
PublicKey::from_bytes(&bytes)
.map_err(|e| format!("Failed to decode bytes into public key: {:?}", e))?
};
Ok(Keypair { pk, sk })
}
}
fn string_to_bytes(string: &str) -> Result<Vec<u8>, String> {
let string = if string.starts_with("0x") {
&string[2..]
} else {
string
};
hex::decode(string).map_err(|e| format!("Unable to decode public or private key: {}", e))
}
/// Loads keypairs from a YAML encoded file.
///
/// Uses this as reference:
/// https://github.com/ethereum/eth2.0-pm/blob/9a9dbcd95e2b8e10287797bd768014ab3d842e99/interop/mocked_start/keygen_10_validators.yaml
pub fn keypairs_from_yaml_file(path: PathBuf) -> Result<Vec<Keypair>, String> {
let file =
File::open(path.clone()).map_err(|e| format!("Unable to open YAML key file: {}", e))?;
serde_yaml::from_reader::<_, Vec<YamlKeypair>>(file)
.map_err(|e| format!("Could not parse YAML: {:?}", e))?
.into_iter()
.map(TryInto::try_into)
.collect::<Result<Vec<_>, String>>()
}

23
eth2/utils/eth2_interop_keypairs/tests/from_file.rs Normal file
View File

@ -0,0 +1,23 @@
#![cfg(test)]
use eth2_interop_keypairs::{keypair as reference_keypair, keypairs_from_yaml_file};
use std::path::PathBuf;
fn yaml_path() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("specs")
.join("keygen_10_validators.yaml")
}
#[test]
fn load_from_yaml() {
let keypairs = keypairs_from_yaml_file(yaml_path()).expect("should read keypairs from file");
keypairs.into_iter().enumerate().for_each(|(i, keypair)| {
assert_eq!(
keypair,
reference_keypair(i),
"Decoded key {} does not match generated key",
i
)
});
}

0
eth2/utils/eth2_interop_keypairs/tests/test.rs → eth2/utils/eth2_interop_keypairs/tests/generation.rs
View File

7
eth2/utils/ssz_types/src/bitfield.rs
View File

@ -729,6 +729,13 @@ mod bitvector {
assert_eq!(bitfield.ssz_bytes_len(), bytes.len(), "i = {}", i);
}
}
#[test]
fn excess_bits_nimbus() {
let bad = vec![0b0001_1111];
assert!(BitVector4::from_ssz_bytes(&bad).is_err());
}
}
#[cfg(test)]

96
scripts/whiteblock_start.sh Executable file
View File

@ -0,0 +1,96 @@
#!/bin/bash
<<COMMENT
Used by Whiteblock for simulated network testing.
Based upon:
https://github.com/whiteblock/dockerfiles/blob/a31b412d32d0384de12aa8392e43bac32837b6bc/ethereum/interop-example/launch/start.sh
Here's an example script used for testing:
./whiteblock_start.sh \
--identity=55c7fc76505ddeb6cf750b1f9f43d6d12c1a53b77ada018a390d7592a7f36dbck \
--peers=/ip4/192.168.0.1/tcp/9000 \
--validator-keys=/tmp/keygen_10_validators.yaml \
--gen-state=/tmp/genesis.ssz \
--port=9008
The example script was run in the target/release directory of lighthouse.
The following change was made to this script:
YAML_KEY_FILE="/tmp/keygen_10_validators.yaml"
COMMENT
# Flags
IDENTITY=""
PEERS=""
YAML_KEY_FILE="/tmp/keygen_10_validators.yaml"
GEN_STATE=""
PORT="8000"
# Constants
BEACON_LOG_FILE="/tmp/beacon.log"
VALIDATOR_LOG_FILE="/tmp/validator.log"
usage() {
echo "--identity=<hex prepresentation of the priv key for libp2p>"
echo "--peers=<peer>"
echo "--validator-keys=<path to /launch/keys.yaml>"
echo "--gen-state=<path to /launch/state.ssz>"
echo "--port=<port>"
}
while [ "$1" != "" ];
do
PARAM=`echo $1 | awk -F= '{print $1}'`
VALUE=`echo $1 | sed 's/^[^=]*=//g'`
case $PARAM in
--identity)
IDENTITY=$VALUE
;;
--peers)
PEERS+=",$VALUE"
;;
--validator-keys)
VALIDATOR_KEYS=$VALUE
;;
--gen-state)
GEN_STATE=$VALUE
;;
--port)
PORT=$VALUE
;;
--help)
usage
exit
;;
*)
echo "ERROR: unknown parameter \"$PARAM\""
usage
exit 1
;;
esac
shift
done
./beacon_node \
--p2p-priv-key $IDENTITY \
--logfile $BEACON_LOG_FILE \
--libp2p-addresses $PEERS \
--port $PORT \
testnet \
--force \
file \
ssz \
$GEN_STATE \
& \
./validator_client \
--logfile $VALIDATOR_LOG_FILE \
testnet \
--bootstrap \
interop-yaml \
$YAML_KEY_FILE \
trap 'trap - SIGTERM && kill 0' SIGINT SIGTERM EXIT

118
tests/cli_util/src/main.rs
View File

@ -1,118 +0,0 @@
#[macro_use]
extern crate log;
use clap::{App, Arg, SubCommand};
use std::fs::File;
use std::path::PathBuf;
use std::time::{SystemTime, UNIX_EPOCH};
use types::{test_utils::TestingBeaconStateBuilder, EthSpec, MainnetEthSpec, MinimalEthSpec};
fn main() {
simple_logger::init().expect("logger should initialize");
let matches = App::new("Lighthouse Testing CLI Tool")
.version("0.1.0")
.author("Paul Hauner <paul@sigmaprime.io>")
.about("Performs various testing-related tasks.")
.subcommand(
SubCommand::with_name("genesis_yaml")
.about("Generates a genesis YAML file")
.version("0.1.0")
.author("Paul Hauner <paul@sigmaprime.io>")
.arg(
Arg::with_name("num_validators")
.short("n")
.value_name("INTEGER")
.takes_value(true)
.required(true)
.help("Number of initial validators."),
)
.arg(
Arg::with_name("genesis_time")
.short("g")
.value_name("INTEGER")
.takes_value(true)
.required(false)
.help("Eth2 genesis time (seconds since UNIX epoch)."),
)
.arg(
Arg::with_name("spec")
.short("s")
.value_name("STRING")
.takes_value(true)
.required(true)
.possible_values(&["minimal", "mainnet"])
.default_value("minimal")
.help("Eth2 genesis time (seconds since UNIX epoch)."),
)
.arg(
Arg::with_name("output_file")
.short("f")
.value_name("PATH")
.takes_value(true)
.default_value("./genesis_state.yaml")
.help("Output file for generated state."),
),
)
.get_matches();
if let Some(matches) = matches.subcommand_matches("genesis_yaml") {
let num_validators = matches
.value_of("num_validators")
.expect("slog requires num_validators")
.parse::<usize>()
.expect("num_validators must be a valid integer");
let genesis_time = if let Some(string) = matches.value_of("genesis_time") {
string
.parse::<u64>()
.expect("genesis_time must be a valid integer")
} else {
warn!("No genesis time supplied via CLI, using the current time.");
SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("should obtain time since unix epoch")
.as_secs()
};
let file = matches
.value_of("output_file")
.expect("slog requires output file")
.parse::<PathBuf>()
.expect("output_file must be a valid path");
info!(
"Creating genesis state with {} validators and genesis time {}.",
num_validators, genesis_time
);
match matches.value_of("spec").expect("spec is required by slog") {
"minimal" => genesis_yaml::<MinimalEthSpec>(num_validators, genesis_time, file),
"mainnet" => genesis_yaml::<MainnetEthSpec>(num_validators, genesis_time, file),
_ => unreachable!("guarded by slog possible_values"),
};
info!("Genesis state YAML file created. Exiting successfully.");
} else {
error!("No subcommand supplied.")
}
}
/// Creates a genesis state and writes it to a YAML file.
fn genesis_yaml<T: EthSpec>(validator_count: usize, genesis_time: u64, output: PathBuf) {
let spec = &T::default_spec();
let builder: TestingBeaconStateBuilder<T> =
TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(validator_count, spec);
let (mut state, _keypairs) = builder.build();
state.genesis_time = genesis_time;
info!("Generated state root: {:?}", state.canonical_root());
info!("Writing genesis state to {:?}", output);
let file = File::create(output.clone())
.unwrap_or_else(|e| panic!("unable to create file: {:?}. Error: {:?}", output, e));
serde_yaml::to_writer(file, &state).expect("should be able to serialize BeaconState");
}

0
tests/cli_util/.gitignore → tests/lcli/.gitignore vendored
View File

6
tests/cli_util/Cargo.toml → tests/lcli/Cargo.toml
View File

@ -1,5 +1,6 @@
[package]
name = "cli_util"
name = "lcli"
description = "Lighthouse CLI (modeled after zcli)"
version = "0.1.0"
authors = ["Paul Hauner <paul@paulhauner.com>"]
edition = "2018"
@ -8,8 +9,11 @@ edition = "2018"
[dependencies]
clap = "2.33"
hex = "0.3"
log = "0.4"
serde = "1.0"
serde_yaml = "0.8"
simple_logger = "1.0"
types = { path = "../../eth2/types" }
state_processing = { path = "../../eth2/state_processing" }
eth2_ssz = { path = "../../eth2/utils/ssz" }

181
tests/lcli/src/main.rs Normal file
View File

@ -0,0 +1,181 @@
#[macro_use]
extern crate log;
mod parse_hex;
mod transition_blocks;
use clap::{App, Arg, SubCommand};
use parse_hex::run_parse_hex;
use std::fs::File;
use std::path::PathBuf;
use std::time::{SystemTime, UNIX_EPOCH};
use transition_blocks::run_transition_blocks;
use types::{test_utils::TestingBeaconStateBuilder, EthSpec, MainnetEthSpec, MinimalEthSpec};
fn main() {
simple_logger::init().expect("logger should initialize");
let matches = App::new("Lighthouse CLI Tool")
.version("0.1.0")
.author("Paul Hauner <paul@sigmaprime.io>")
.about(
"Performs various testing-related tasks, modelled after zcli. \
by @protolambda.",
)
.subcommand(
SubCommand::with_name("genesis_yaml")
.about("Generates a genesis YAML file")
.version("0.1.0")
.author("Paul Hauner <paul@sigmaprime.io>")
.arg(
Arg::with_name("num_validators")
.short("n")
.value_name("INTEGER")
.takes_value(true)
.required(true)
.help("Number of initial validators."),
)
.arg(
Arg::with_name("genesis_time")
.short("g")
.value_name("INTEGER")
.takes_value(true)
.required(false)
.help("Eth2 genesis time (seconds since UNIX epoch)."),
)
.arg(
Arg::with_name("spec")
.short("s")
.value_name("STRING")
.takes_value(true)
.required(true)
.possible_values(&["minimal", "mainnet"])
.default_value("minimal")
.help("Eth2 genesis time (seconds since UNIX epoch)."),
)
.arg(
Arg::with_name("output_file")
.short("f")
.value_name("PATH")
.takes_value(true)
.default_value("./genesis_state.yaml")
.help("Output file for generated state."),
),
)
.subcommand(
SubCommand::with_name("transition-blocks")
.about("Performs a state transition given a pre-state and block")
.version("0.1.0")
.author("Paul Hauner <paul@sigmaprime.io>")
.arg(
Arg::with_name("pre-state")
.value_name("BEACON_STATE")
.takes_value(true)
.required(true)
.help("Path to a SSZ file of the pre-state."),
)
.arg(
Arg::with_name("block")
.value_name("BEACON_BLOCK")
.takes_value(true)
.required(true)
.help("Path to a SSZ file of the block to apply to pre-state."),
)
.arg(
Arg::with_name("output")
.value_name("SSZ_FILE")
.takes_value(true)
.required(true)
.default_value("./output.ssz")
.help("Path to output a SSZ file."),
),
)
.subcommand(
SubCommand::with_name("pretty-hex")
.about("Parses SSZ encoded as ASCII 0x-prefixed hex")
.version("0.1.0")
.author("Paul Hauner <paul@sigmaprime.io>")
.arg(
Arg::with_name("type")
.value_name("TYPE")
.takes_value(true)
.required(true)
.possible_values(&["block"])
.help("The schema of the supplied SSZ."),
)
.arg(
Arg::with_name("hex_ssz")
.value_name("HEX")
.takes_value(true)
.required(true)
.help("SSZ encoded as 0x-prefixed hex"),
),
)
.get_matches();
match matches.subcommand() {
("genesis_yaml", Some(matches)) => {
let num_validators = matches
.value_of("num_validators")
.expect("slog requires num_validators")
.parse::<usize>()
.expect("num_validators must be a valid integer");
let genesis_time = if let Some(string) = matches.value_of("genesis_time") {
string
.parse::<u64>()
.expect("genesis_time must be a valid integer")
} else {
warn!("No genesis time supplied via CLI, using the current time.");
SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("should obtain time since unix epoch")
.as_secs()
};
let file = matches
.value_of("output_file")
.expect("slog requires output file")
.parse::<PathBuf>()
.expect("output_file must be a valid path");
info!(
"Creating genesis state with {} validators and genesis time {}.",
num_validators, genesis_time
);
match matches.value_of("spec").expect("spec is required by slog") {
"minimal" => genesis_yaml::<MinimalEthSpec>(num_validators, genesis_time, file),
"mainnet" => genesis_yaml::<MainnetEthSpec>(num_validators, genesis_time, file),
_ => unreachable!("guarded by slog possible_values"),
};
info!("Genesis state YAML file created. Exiting successfully.");
}
("transition-blocks", Some(matches)) => run_transition_blocks(matches)
.unwrap_or_else(|e| error!("Failed to transition blocks: {}", e)),
("pretty-hex", Some(matches)) => {
run_parse_hex(matches).unwrap_or_else(|e| error!("Failed to pretty print hex: {}", e))
}
(other, _) => error!("Unknown subcommand {}. See --help.", other),
}
}
/// Creates a genesis state and writes it to a YAML file.
fn genesis_yaml<T: EthSpec>(validator_count: usize, genesis_time: u64, output: PathBuf) {
let spec = &T::default_spec();
let builder: TestingBeaconStateBuilder<T> =
TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(validator_count, spec);
let (mut state, _keypairs) = builder.build();
state.genesis_time = genesis_time;
info!("Generated state root: {:?}", state.canonical_root());
info!("Writing genesis state to {:?}", output);
let file = File::create(output.clone())
.unwrap_or_else(|e| panic!("unable to create file: {:?}. Error: {:?}", output, e));
serde_yaml::to_writer(file, &state).expect("should be able to serialize BeaconState");
}

43
tests/lcli/src/parse_hex.rs Normal file
View File

@ -0,0 +1,43 @@
use clap::ArgMatches;
use serde::Serialize;
use ssz::Decode;
use types::{BeaconBlock, BeaconState, MinimalEthSpec};
pub fn run_parse_hex(matches: &ArgMatches) -> Result<(), String> {
let type_str = matches
.value_of("type")
.ok_or_else(|| "No type supplied".to_string())?;
let mut hex: String = matches
.value_of("hex_ssz")
.ok_or_else(|| "No hex ssz supplied".to_string())?
.to_string();
if hex.starts_with("0x") {
hex = hex[2..].to_string();
}
let hex = hex::decode(&hex).map_err(|e| format!("Failed to parse hex: {:?}", e))?;
info!("Using minimal spec");
info!("Type: {:?}", type_str);
match type_str.as_ref() {
"block" => decode_and_print::<BeaconBlock<MinimalEthSpec>>(&hex)?,
"state" => decode_and_print::<BeaconState<MinimalEthSpec>>(&hex)?,
other => return Err(format!("Unknown type: {}", other)),
};
Ok(())
}
fn decode_and_print<T: Decode + Serialize>(bytes: &[u8]) -> Result<(), String> {
let item = T::from_ssz_bytes(&bytes).map_err(|e| format!("Ssz decode failed: {:?}", e))?;
println!(
"{}",
serde_yaml::to_string(&item)
.map_err(|e| format!("Unable to write object to YAML: {:?}", e))?
);
Ok(())
}

93
tests/lcli/src/transition_blocks.rs Normal file
View File

@ -0,0 +1,93 @@
use clap::ArgMatches;
use ssz::{Decode, Encode};
use state_processing::{per_block_processing, per_slot_processing, BlockSignatureStrategy};
use std::fs::File;
use std::io::prelude::*;
use std::path::PathBuf;
use types::{BeaconBlock, BeaconState, EthSpec, MinimalEthSpec};
pub fn run_transition_blocks(matches: &ArgMatches) -> Result<(), String> {
let pre_state_path = matches
.value_of("pre-state")
.ok_or_else(|| "No pre-state file supplied".to_string())?
.parse::<PathBuf>()
.map_err(|e| format!("Failed to parse pre-state path: {}", e))?;
let block_path = matches
.value_of("block")
.ok_or_else(|| "No block file supplied".to_string())?
.parse::<PathBuf>()
.map_err(|e| format!("Failed to parse block path: {}", e))?;
let output_path = matches
.value_of("output")
.ok_or_else(|| "No output file supplied".to_string())?
.parse::<PathBuf>()
.map_err(|e| format!("Failed to parse output path: {}", e))?;
info!("Using minimal spec");
info!("Pre-state path: {:?}", pre_state_path);
info!("Block path: {:?}", block_path);
let pre_state: BeaconState<MinimalEthSpec> = load_from_ssz(pre_state_path)?;
let block: BeaconBlock<MinimalEthSpec> = load_from_ssz(block_path)?;
let post_state = do_transition(pre_state, block)?;
let mut output_file = File::create(output_path.clone())
.map_err(|e| format!("Unable to create output file: {:?}", e))?;
output_file
.write_all(&post_state.as_ssz_bytes())
.map_err(|e| format!("Unable to write to output file: {:?}", e))?;
/*
println!(
"{}",
serde_yaml::to_string(&post_state).expect("Should serialize state")
);
*/
Ok(())
}
fn do_transition<T: EthSpec>(
mut pre_state: BeaconState<T>,
block: BeaconBlock<T>,
) -> Result<BeaconState<T>, String> {
let spec = &T::default_spec();
pre_state
.build_all_caches(spec)
.map_err(|e| format!("Unable to build caches: {:?}", e))?;
// Transition the parent state to the block slot.
for i in pre_state.slot.as_u64()..block.slot.as_u64() {
per_slot_processing(&mut pre_state, spec)
.map_err(|e| format!("Failed to advance slot on iteration {}: {:?}", i, e))?;
}
pre_state
.build_all_caches(spec)
.map_err(|e| format!("Unable to build caches: {:?}", e))?;
per_block_processing(
&mut pre_state,
&block,
None,
BlockSignatureStrategy::VerifyIndividual,
spec,
)
.map_err(|e| format!("State transition failed: {:?}", e))?;
Ok(pre_state)
}
fn load_from_ssz<T: Decode>(path: PathBuf) -> Result<T, String> {
let mut file =
File::open(path.clone()).map_err(|e| format!("Unable to open file {:?}: {:?}", path, e))?;
let mut bytes = vec![];
file.read_to_end(&mut bytes)
.map_err(|e| format!("Unable to read from file {:?}: {:?}", path, e))?;
T::from_ssz_bytes(&bytes).map_err(|e| format!("Ssz decode failed: {:?}", e))
}

3
validator_client/Cargo.toml
View File

@ -19,13 +19,14 @@ eth2_config = { path = "../eth2/utils/eth2_config" }
tree_hash = "0.1"
clap = "2.32.0"
lighthouse_bootstrap = { path = "../eth2/utils/lighthouse_bootstrap" }
eth2_interop_keypairs = { path = "../eth2/utils/eth2_interop_keypairs" }
grpcio = { version = "0.4", default-features = false, features = ["protobuf-codec"] }
protos = { path = "../protos" }
slot_clock = { path = "../eth2/utils/slot_clock" }
types = { path = "../eth2/types" }
serde = "1.0"
serde_derive = "1.0"
slog = "^2.2.3"
slog = { version = "^2.2.3" , features = ["max_level_trace", "release_max_level_trace"] }
slog-async = "^2.3.0"
slog-json = "^2.3"
slog-term = "^2.4.0"

9
validator_client/src/attestation_producer/mod.rs
View File

@ -50,9 +50,12 @@ impl<'a, B: BeaconNodeAttestation, S: Signer, E: EthSpec> AttestationProducer<'a
/// Handle outputs and results from attestation production.
pub fn handle_produce_attestation(&mut self, log: slog::Logger) {
match self.produce_attestation() {
Ok(ValidatorEvent::AttestationProduced(_slot)) => {
info!(log, "Attestation produced"; "Validator" => format!("{}", self.signer))
}
Ok(ValidatorEvent::AttestationProduced(slot)) => info!(
log,
"Attestation produced";
"validator" => format!("{}", self.signer),
"slot" => slot,
),
Err(e) => error!(log, "Attestation production error"; "Error" => format!("{:?}", e)),
Ok(ValidatorEvent::SignerRejection(_slot)) => {
error!(log, "Attestation production error"; "Error" => "Signer could not sign the attestation".to_string())

12
validator_client/src/block_producer/mod.rs
View File

@ -59,9 +59,12 @@ impl<'a, B: BeaconNodeBlock, S: Signer, E: EthSpec> BlockProducer<'a, B, S, E> {
/// Handle outputs and results from block production.
pub fn handle_produce_block(&mut self, log: slog::Logger) {
match self.produce_block() {
Ok(ValidatorEvent::BlockProduced(_slot)) => {
info!(log, "Block produced"; "Validator" => format!("{}", self.signer))
}
Ok(ValidatorEvent::BlockProduced(slot)) => info!(
log,
"Block produced";
"validator" => format!("{}", self.signer),
"slot" => slot,
),
Err(e) => error!(log, "Block production error"; "Error" => format!("{:?}", e)),
Ok(ValidatorEvent::SignerRejection(_slot)) => {
error!(log, "Block production error"; "Error" => "Signer Could not sign the block".to_string())
@ -105,12 +108,13 @@ impl<'a, B: BeaconNodeBlock, S: Signer, E: EthSpec> BlockProducer<'a, B, S, E> {
.produce_beacon_block(self.slot, &randao_reveal)?
{
if self.safe_to_produce(&block) {
let slot = block.slot;
let domain = self
.spec
.get_domain(epoch, Domain::BeaconProposer, &self.fork);
if let Some(block) = self.sign_block(block, domain) {
self.beacon_node.publish_beacon_block(block)?;
Ok(ValidatorEvent::BlockProduced(self.slot))
Ok(ValidatorEvent::BlockProduced(slot))
} else {
Ok(ValidatorEvent::SignerRejection(self.slot))
}

21
validator_client/src/config.rs
View File

@ -8,7 +8,10 @@ use std::io::{Error, ErrorKind};
use std::ops::Range;
use std::path::PathBuf;
use std::sync::Mutex;
use types::{test_utils::generate_deterministic_keypair, EthSpec, MainnetEthSpec};
use types::{
test_utils::{generate_deterministic_keypair, load_keypairs_from_yaml},
EthSpec, MainnetEthSpec,
};
pub const DEFAULT_SERVER: &str = "localhost";
pub const DEFAULT_SERVER_GRPC_PORT: &str = "5051";
@ -20,6 +23,8 @@ pub enum KeySource {
Disk,
/// Generate the keypairs (insecure, generates predictable keys).
TestingKeypairRange(Range<usize>),
/// Load testing keypairs from YAML
YamlKeypairs(PathBuf),
}
impl Default for KeySource {
@ -227,9 +232,21 @@ impl Config {
let keypairs = match &self.key_source {
KeySource::Disk => self.fetch_keys_from_disk(log)?,
KeySource::TestingKeypairRange(range) => {
warn!(log, "Using insecure private keys");
warn!(
log,
"Using insecure interop private keys";
"range" => format!("{:?}", range)
);
self.fetch_testing_keypairs(range.clone())?
}
KeySource::YamlKeypairs(path) => {
warn!(
log,
"Private keys are stored insecurely (plain text). Testing use only."
);
load_keypairs_from_yaml(path.to_path_buf())?
}
};
// Check if it's an empty vector, and return none.

53
validator_client/src/main.rs
View File

@ -16,6 +16,7 @@ use eth2_config::Eth2Config;
use lighthouse_bootstrap::Bootstrapper;
use protos::services_grpc::ValidatorServiceClient;
use slog::{crit, error, info, o, Drain, Level, Logger};
use std::path::PathBuf;
use types::{InteropEthSpec, Keypair, MainnetEthSpec, MinimalEthSpec};
pub const DEFAULT_SPEC: &str = "minimal";
@ -81,7 +82,8 @@ fn main() {
)
.arg(
Arg::with_name("server-grpc-port")
.long("g")
.long("server-grpc-port")
.short("g")
.value_name("PORT")
.help("Port to use for gRPC API connection to the server.")
.default_value(DEFAULT_SERVER_GRPC_PORT)
@ -89,7 +91,8 @@ fn main() {
)
.arg(
Arg::with_name("server-http-port")
.long("h")
.long("server-http-port")
.short("h")
.value_name("PORT")
.help("Port to use for HTTP API connection to the server.")
.default_value(DEFAULT_SERVER_HTTP_PORT)
@ -103,7 +106,7 @@ fn main() {
.help("The title of the spec constants for chain config.")
.takes_value(true)
.possible_values(&["info", "debug", "trace", "warn", "error", "crit"])
.default_value("info"),
.default_value("trace"),
)
/*
* The "testnet" sub-command.
@ -131,6 +134,14 @@ fn main() {
.required(true)
.help("The number of validators."))
)
.subcommand(SubCommand::with_name("interop-yaml")
.about("Loads plain-text secret keys from YAML files. Expects the interop format defined
in the ethereum/eth2.0-pm repo.")
.arg(Arg::with_name("path")
.value_name("PATH")
.required(true)
.help("Path to a YAML file."))
)
)
.get_matches();
@ -143,8 +154,10 @@ fn main() {
Some("crit") => drain.filter_level(Level::Critical),
_ => unreachable!("guarded by clap"),
};
let log = slog::Logger::root(drain.fuse(), o!());
let (client_config, eth2_config) = match get_configs(&matches, &log) {
let mut log = slog::Logger::root(drain.fuse(), o!());
let (client_config, eth2_config) = match get_configs(&matches, &mut log) {
Ok(tuple) => tuple,
Err(e) => {
crit!(
@ -195,9 +208,14 @@ fn main() {
/// Parses the CLI arguments and attempts to load the client and eth2 configuration.
///
/// This is not a pure function, it reads from disk and may contact network servers.
pub fn get_configs(cli_args: &ArgMatches, log: &Logger) -> Result<(ClientConfig, Eth2Config)> {
pub fn get_configs(
cli_args: &ArgMatches,
mut log: &mut Logger,
) -> Result<(ClientConfig, Eth2Config)> {
let mut client_config = ClientConfig::default();
client_config.apply_cli_args(&cli_args, &mut log)?;
if let Some(server) = cli_args.value_of("server") {
client_config.server = server.to_string();
}
@ -215,14 +233,14 @@ pub fn get_configs(cli_args: &ArgMatches, log: &Logger) -> Result<(ClientConfig,
}
info!(
log,
*log,
"Beacon node connection info";
"grpc_port" => client_config.server_grpc_port,
"http_port" => client_config.server_http_port,
"server" => &client_config.server,
);
match cli_args.subcommand() {
let (client_config, eth2_config) = match cli_args.subcommand() {
("testnet", Some(sub_cli_args)) => {
if cli_args.is_present("eth2-config") && sub_cli_args.is_present("bootstrap") {
return Err(
@ -234,7 +252,9 @@ pub fn get_configs(cli_args: &ArgMatches, log: &Logger) -> Result<(ClientConfig,
process_testnet_subcommand(sub_cli_args, client_config, log)
}
_ => return Err("You must use the testnet command. See '--help'.".into()),
}
}?;
Ok((client_config, eth2_config))
}
/// Parses the `testnet` CLI subcommand.
@ -296,6 +316,21 @@ fn process_testnet_subcommand(
KeySource::TestingKeypairRange(first..first + count)
}
("interop-yaml", Some(sub_cli_args)) => {
let path = sub_cli_args
.value_of("path")
.ok_or_else(|| "No yaml path supplied")?
.parse::<PathBuf>()
.map_err(|e| format!("Unable to parse yaml path: {:?}", e))?;
info!(
log,
"Loading keypairs from interop YAML format";
"path" => format!("{:?}", path),
);
KeySource::YamlKeypairs(path)
}
_ => KeySource::Disk,
};

37
validator_client/src/service.rs
View File

@ -22,7 +22,7 @@ use protos::services_grpc::{
AttestationServiceClient, BeaconBlockServiceClient, BeaconNodeServiceClient,
ValidatorServiceClient,
};
use slog::{crit, error, info, warn};
use slog::{crit, error, info, trace, warn};
use slot_clock::{SlotClock, SystemTimeSlotClock};
use std::marker::PhantomData;
use std::sync::Arc;
@ -289,6 +289,11 @@ impl<B: BeaconNodeDuties + 'static, S: Signer + 'static, E: EthSpec> Service<B,
/* process any required duties for validators */
self.process_duties();
trace!(
self.log,
"Per slot execution finished";
);
Ok(())
}
@ -328,6 +333,13 @@ impl<B: BeaconNodeDuties + 'static, S: Signer + 'static, E: EthSpec> Service<B,
.current_slot
.expect("The current slot must be updated before checking for duties")
.epoch(self.slots_per_epoch);
trace!(
self.log,
"Checking for duties";
"epoch" => current_epoch
);
// spawn a new thread separate to the runtime
// TODO: Handle thread termination/timeout
// TODO: Add duties thread back in, with channel to process duties in duty change.
@ -345,6 +357,12 @@ impl<B: BeaconNodeDuties + 'static, S: Signer + 'static, E: EthSpec> Service<B,
self.current_slot
.expect("The current slot must be updated before processing duties"),
) {
trace!(
self.log,
"Processing duties";
"work_items" => work.len()
);
for (signer_index, work_type) in work {
if work_type.produce_block {
// we need to produce a block
@ -359,7 +377,12 @@ impl<B: BeaconNodeDuties + 'static, S: Signer + 'static, E: EthSpec> Service<B,
let log = self.log.clone();
let slots_per_epoch = self.slots_per_epoch;
std::thread::spawn(move || {
info!(log, "Producing a block"; "Validator"=> format!("{}", signers[signer_index]));
info!(
log,
"Producing a block";
"validator"=> format!("{}", signers[signer_index]),
"slot"=> slot
);
let signer = &signers[signer_index];
let mut block_producer = BlockProducer {
fork,
@ -376,6 +399,9 @@ impl<B: BeaconNodeDuties + 'static, S: Signer + 'static, E: EthSpec> Service<B,
if work_type.attestation_duty.is_some() {
// we need to produce an attestation
// spawns a thread to produce and sign an attestation
let slot = self
.current_slot
.expect("The current slot must be updated before processing duties");
let signers = self.duties_manager.signers.clone(); // this is an arc
let fork = self.fork.clone();
let spec = self.spec.clone();
@ -383,7 +409,12 @@ impl<B: BeaconNodeDuties + 'static, S: Signer + 'static, E: EthSpec> Service<B,
let log = self.log.clone();
let slots_per_epoch = self.slots_per_epoch;
std::thread::spawn(move || {
info!(log, "Producing an attestation"; "Validator"=> format!("{}", signers[signer_index]));
info!(
log,
"Producing an attestation";
"validator"=> format!("{}", signers[signer_index]),
"slot"=> slot
);
let signer = &signers[signer_index];
let mut attestation_producer = AttestationProducer {
fork,