From a6886219191e2553177a9f45ad2a046faa1674d3 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 9 Aug 2022 06:05:13 +0000 Subject: [PATCH] Add support for beaconAPI in `lcli` functions (#3252) ## Issue Addressed NA ## Proposed Changes Modifies `lcli skip-slots` and `lcli transition-blocks` allow them to source blocks/states from a beaconAPI and also gives them some more features to assist with benchmarking. ## Additional Info Breaks the current `lcli skip-slots` and `lcli transition-blocks` APIs by changing some flag names. It should be simple enough to figure out the changes via `--help`. Currently blocked on #3263. --- Cargo.lock | 2 + beacon_node/beacon_chain/src/lib.rs | 2 +- .../src/validator_pubkey_cache.rs | 5 + common/eth2/src/lib.rs | 2 +- common/sensitive_url/src/lib.rs | 15 + .../src/per_block_processing.rs | 2 +- lcli/Cargo.toml | 2 + lcli/src/main.rs | 150 +++++-- lcli/src/skip_slots.rs | 160 +++++-- lcli/src/transition_blocks.rs | 410 ++++++++++++++---- 10 files changed, 607 insertions(+), 143 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1160609be..3702b9548 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3105,6 +3105,7 @@ name = "lcli" version = "2.5.1" dependencies = [ "account_utils", + "beacon_chain", "bls", "clap", "clap_utils", @@ -3128,6 +3129,7 @@ dependencies = [ "serde_yaml", "snap", "state_processing", + "store", "tree_hash", "types", "validator_dir", diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index ed6c2459e..481b1ae73 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -41,7 +41,7 @@ pub mod sync_committee_verification; pub mod test_utils; mod timeout_rw_lock; pub mod validator_monitor; -mod validator_pubkey_cache; +pub mod validator_pubkey_cache; pub use self::beacon_chain::{ AttestationProcessingOutcome, BeaconChain, BeaconChainTypes, BeaconStore, ChainSegmentResult, diff --git a/beacon_node/beacon_chain/src/validator_pubkey_cache.rs b/beacon_node/beacon_chain/src/validator_pubkey_cache.rs index beb8da8b6..60fdb607c 100644 --- a/beacon_node/beacon_chain/src/validator_pubkey_cache.rs +++ b/beacon_node/beacon_chain/src/validator_pubkey_cache.rs @@ -156,6 +156,11 @@ impl ValidatorPubkeyCache { pub fn len(&self) -> usize { self.indices.len() } + + /// Returns `true` if there are no validators in the cache. + pub fn is_empty(&self) -> bool { + self.indices.is_empty() + } } /// Wrapper for a public key stored in the database. diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 8cd138e98..21608ba6d 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -23,7 +23,7 @@ use lighthouse_network::PeerId; pub use reqwest; use reqwest::{IntoUrl, RequestBuilder, Response}; pub use reqwest::{StatusCode, Url}; -use sensitive_url::SensitiveUrl; +pub use sensitive_url::SensitiveUrl; use serde::{de::DeserializeOwned, Serialize}; use std::convert::TryFrom; use std::fmt; diff --git a/common/sensitive_url/src/lib.rs b/common/sensitive_url/src/lib.rs index 7a3cbae20..aac4cb550 100644 --- a/common/sensitive_url/src/lib.rs +++ b/common/sensitive_url/src/lib.rs @@ -1,5 +1,6 @@ use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; use std::fmt; +use std::str::FromStr; use url::Url; #[derive(Debug)] @@ -9,6 +10,12 @@ pub enum SensitiveError { RedactError(String), } +impl fmt::Display for SensitiveError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + // Wrapper around Url which provides a custom `Display` implementation to protect user secrets. #[derive(Clone, PartialEq)] pub struct SensitiveUrl { @@ -54,6 +61,14 @@ impl<'de> Deserialize<'de> for SensitiveUrl { } } +impl FromStr for SensitiveUrl { + type Err = SensitiveError; + + fn from_str(s: &str) -> Result { + Self::parse(s) + } +} + impl SensitiveUrl { pub fn parse(url: &str) -> Result { let surl = Url::parse(url).map_err(SensitiveError::ParseError)?; diff --git a/consensus/state_processing/src/per_block_processing.rs b/consensus/state_processing/src/per_block_processing.rs index 89cb76e0a..e409372dd 100644 --- a/consensus/state_processing/src/per_block_processing.rs +++ b/consensus/state_processing/src/per_block_processing.rs @@ -40,7 +40,7 @@ use arbitrary::Arbitrary; /// The strategy to be used when validating the block's signatures. #[cfg_attr(feature = "arbitrary-fuzz", derive(Arbitrary))] -#[derive(PartialEq, Clone, Copy)] +#[derive(PartialEq, Clone, Copy, Debug)] pub enum BlockSignatureStrategy { /// Do not validate any signature. Use with caution. NoVerification, diff --git a/lcli/Cargo.toml b/lcli/Cargo.toml index e54d9d8c9..5d94a5046 100644 --- a/lcli/Cargo.toml +++ b/lcli/Cargo.toml @@ -38,3 +38,5 @@ eth1_test_rig = { path = "../testing/eth1_test_rig" } sensitive_url = { path = "../common/sensitive_url" } eth2 = { path = "../common/eth2" } snap = "1.0.1" +beacon_chain = { path = "../beacon_node/beacon_chain" } +store = { path = "../beacon_node/store" } diff --git a/lcli/src/main.rs b/lcli/src/main.rs index c440f5000..2fd053885 100644 --- a/lcli/src/main.rs +++ b/lcli/src/main.rs @@ -22,7 +22,6 @@ use parse_ssz::run_parse_ssz; use std::path::PathBuf; use std::process; use std::str::FromStr; -use transition_blocks::run_transition_blocks; use types::{EthSpec, EthSpecId}; fn main() { @@ -57,52 +56,128 @@ fn main() { "Performs a state transition from some state across some number of skip slots", ) .arg( - Arg::with_name("pre-state") - .value_name("BEACON_STATE") + Arg::with_name("output-path") + .long("output-path") + .value_name("PATH") .takes_value(true) - .required(true) + .help("Path to output a SSZ file."), + ) + .arg( + Arg::with_name("pre-state-path") + .long("pre-state-path") + .value_name("PATH") + .takes_value(true) + .conflicts_with("beacon-url") .help("Path to a SSZ file of the pre-state."), ) .arg( - Arg::with_name("slots") - .value_name("SLOT_COUNT") + Arg::with_name("beacon-url") + .long("beacon-url") + .value_name("URL") .takes_value(true) - .required(true) - .help("Number of slots to skip before outputting a state.."), + .help("URL to a beacon-API provider."), ) .arg( - Arg::with_name("output") - .value_name("SSZ_FILE") + Arg::with_name("state-id") + .long("state-id") + .value_name("STATE_ID") .takes_value(true) - .required(true) - .default_value("./output.ssz") - .help("Path to output a SSZ file."), - ), + .requires("beacon-url") + .help("Identifier for a state as per beacon-API standards (slot, root, etc.)"), + ) + .arg( + Arg::with_name("runs") + .long("runs") + .value_name("INTEGER") + .takes_value(true) + .default_value("1") + .help("Number of repeat runs, useful for benchmarking."), + ) + .arg( + Arg::with_name("state-root") + .long("state-root") + .value_name("HASH256") + .takes_value(true) + .help("Tree hash root of the provided state, to avoid computing it."), + ) + .arg( + Arg::with_name("slots") + .long("slots") + .value_name("INTEGER") + .takes_value(true) + .help("Number of slots to skip forward."), + ) + .arg( + Arg::with_name("partial-state-advance") + .long("partial-state-advance") + .takes_value(false) + .help("If present, don't compute state roots when skipping forward."), + ) ) .subcommand( SubCommand::with_name("transition-blocks") .about("Performs a state transition given a pre-state and block") .arg( - Arg::with_name("pre-state") - .value_name("BEACON_STATE") + Arg::with_name("pre-state-path") + .long("pre-state-path") + .value_name("PATH") .takes_value(true) - .required(true) - .help("Path to a SSZ file of the pre-state."), + .conflicts_with("beacon-url") + .requires("block-path") + .help("Path to load a BeaconState from file as SSZ."), ) .arg( - Arg::with_name("block") - .value_name("BEACON_BLOCK") + Arg::with_name("block-path") + .long("block-path") + .value_name("PATH") .takes_value(true) - .required(true) - .help("Path to a SSZ file of the block to apply to pre-state."), + .conflicts_with("beacon-url") + .requires("pre-state-path") + .help("Path to load a SignedBeaconBlock from file as SSZ."), ) .arg( - Arg::with_name("output") - .value_name("SSZ_FILE") + Arg::with_name("post-state-output-path") + .long("post-state-output-path") + .value_name("PATH") .takes_value(true) - .required(true) - .default_value("./output.ssz") - .help("Path to output a SSZ file."), + .help("Path to output the post-state."), + ) + .arg( + Arg::with_name("pre-state-output-path") + .long("pre-state-output-path") + .value_name("PATH") + .takes_value(true) + .help("Path to output the pre-state, useful when used with --beacon-url."), + ) + .arg( + Arg::with_name("block-output-path") + .long("block-output-path") + .value_name("PATH") + .takes_value(true) + .help("Path to output the block, useful when used with --beacon-url."), + ) + .arg( + Arg::with_name("beacon-url") + .long("beacon-url") + .value_name("URL") + .takes_value(true) + .help("URL to a beacon-API provider."), + ) + .arg( + Arg::with_name("block-id") + .long("block-id") + .value_name("BLOCK_ID") + .takes_value(true) + .requires("beacon-url") + .help("Identifier for a block as per beacon-API standards (slot, root, etc.)"), + ) + .arg( + Arg::with_name("runs") + .long("runs") + .value_name("INTEGER") + .takes_value(true) + .default_value("1") + .help("Number of repeat runs, useful for benchmarking."), ) .arg( Arg::with_name("no-signature-verification") @@ -110,6 +185,20 @@ fn main() { .takes_value(false) .help("Disable signature verification.") ) + .arg( + Arg::with_name("exclude-cache-builds") + .long("exclude-cache-builds") + .takes_value(false) + .help("If present, pre-build the committee and tree-hash caches without \ + including them in the timings."), + ) + .arg( + Arg::with_name("exclude-post-block-thc") + .long("exclude-post-block-thc") + .takes_value(false) + .help("If present, don't rebuild the tree-hash-cache after applying \ + the block."), + ) ) .subcommand( SubCommand::with_name("pretty-ssz") @@ -673,10 +762,11 @@ fn run( )?; match matches.subcommand() { - ("transition-blocks", Some(matches)) => run_transition_blocks::(testnet_dir, matches) + ("transition-blocks", Some(matches)) => transition_blocks::run::(env, matches) .map_err(|e| format!("Failed to transition blocks: {}", e)), - ("skip-slots", Some(matches)) => skip_slots::run::(testnet_dir, matches) - .map_err(|e| format!("Failed to skip slots: {}", e)), + ("skip-slots", Some(matches)) => { + skip_slots::run::(env, matches).map_err(|e| format!("Failed to skip slots: {}", e)) + } ("pretty-ssz", Some(matches)) => { run_parse_ssz::(matches).map_err(|e| format!("Failed to pretty print hex: {}", e)) } diff --git a/lcli/src/skip_slots.rs b/lcli/src/skip_slots.rs index cb502d37a..28310f768 100644 --- a/lcli/src/skip_slots.rs +++ b/lcli/src/skip_slots.rs @@ -1,58 +1,150 @@ +//! # Skip-Slots +//! +//! Use this tool to process a `BeaconState` through empty slots. Useful for benchmarking or +//! troubleshooting consensus failures. +//! +//! It can load states from file or pull them from a beaconAPI. States pulled from a beaconAPI can +//! be saved to disk to reduce future calls to that server. +//! +//! ## Examples +//! +//! ### Example 1. +//! +//! Download a state from a HTTP endpoint and skip forward an epoch, twice (the initial state is +//! advanced 32 slots twice, rather than it being advanced 64 slots): +//! +//! ```ignore +//! lcli skip-slots \ +//! --beacon-url http://localhost:5052 \ +//! --state-id 0x3cdc33cd02713d8d6cc33a6dbe2d3a5bf9af1d357de0d175a403496486ff845e \\ +//! --slots 32 \ +//! --runs 2 +//! ``` +//! +//! ### Example 2. +//! +//! Download a state to a SSZ file (without modifying it): +//! +//! ```ignore +//! lcli skip-slots \ +//! --beacon-url http://localhost:5052 \ +//! --state-id 0x3cdc33cd02713d8d6cc33a6dbe2d3a5bf9af1d357de0d175a403496486ff845e \ +//! --slots 0 \ +//! --runs 0 \ +//! --output-path /tmp/state-0x3cdc.ssz +//! ``` +//! +//! ### Example 3. +//! +//! Do two runs over the state that was downloaded in the previous example: +//! +//! ```ignore +//! lcli skip-slots \ +//! --pre-state-path /tmp/state-0x3cdc.ssz \ +//! --slots 32 \ +//! --runs 2 +//! ``` use crate::transition_blocks::load_from_ssz_with; use clap::ArgMatches; -use eth2_network_config::Eth2NetworkConfig; +use clap_utils::{parse_optional, parse_required}; +use environment::Environment; +use eth2::{types::StateId, BeaconNodeHttpClient, SensitiveUrl, Timeouts}; use ssz::Encode; -use state_processing::per_slot_processing; +use state_processing::state_advance::{complete_state_advance, partial_state_advance}; use std::fs::File; use std::io::prelude::*; use std::path::PathBuf; -use types::{BeaconState, EthSpec}; +use std::time::{Duration, Instant}; +use types::{BeaconState, CloneConfig, EthSpec, Hash256}; -pub fn run(testnet_dir: PathBuf, matches: &ArgMatches) -> Result<(), String> { - let pre_state_path = matches - .value_of("pre-state") - .ok_or("No pre-state file supplied")? - .parse::() - .map_err(|e| format!("Failed to parse pre-state path: {}", e))?; +const HTTP_TIMEOUT: Duration = Duration::from_secs(10); - let slots = matches - .value_of("slots") - .ok_or("No slots supplied")? - .parse::() - .map_err(|e| format!("Failed to parse slots: {}", e))?; +pub fn run(mut env: Environment, matches: &ArgMatches) -> Result<(), String> { + let spec = &T::default_spec(); + let executor = env.core_context().executor; - let output_path = matches - .value_of("output") - .ok_or("No output file supplied")? - .parse::() - .map_err(|e| format!("Failed to parse output path: {}", e))?; + let output_path: Option = parse_optional(matches, "output-path")?; + let state_path: Option = parse_optional(matches, "pre-state-path")?; + let beacon_url: Option = parse_optional(matches, "beacon-url")?; + let runs: usize = parse_required(matches, "runs")?; + let slots: u64 = parse_required(matches, "slots")?; + let cli_state_root: Option = parse_optional(matches, "state-root")?; + let partial: bool = matches.is_present("partial-state-advance"); info!("Using {} spec", T::spec_name()); - info!("Pre-state path: {:?}", pre_state_path); - info!("Slots: {:?}", slots); + info!("Advancing {} slots", slots); + info!("Doing {} runs", runs); - let eth2_network_config = Eth2NetworkConfig::load(testnet_dir)?; - let spec = ð2_network_config.chain_spec::()?; + let (mut state, state_root) = match (state_path, beacon_url) { + (Some(state_path), None) => { + info!("State path: {:?}", state_path); + let state = load_from_ssz_with(&state_path, spec, BeaconState::from_ssz_bytes)?; + (state, None) + } + (None, Some(beacon_url)) => { + let state_id: StateId = parse_required(matches, "state-id")?; + let client = BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(HTTP_TIMEOUT)); + let state = executor + .handle() + .ok_or("shutdown in progress")? + .block_on(async move { + client + .get_debug_beacon_states::(state_id) + .await + .map_err(|e| format!("Failed to download state: {:?}", e)) + }) + .map_err(|e| format!("Failed to complete task: {:?}", e))? + .ok_or_else(|| format!("Unable to locate state at {:?}", state_id))? + .data; + let state_root = match state_id { + StateId::Root(root) => Some(root), + _ => None, + }; + (state, state_root) + } + _ => return Err("must supply either --state-path or --beacon-url".into()), + }; - let mut state: BeaconState = - load_from_ssz_with(&pre_state_path, spec, BeaconState::from_ssz_bytes)?; + let initial_slot = state.slot(); + let target_slot = initial_slot + slots; state .build_all_caches(spec) .map_err(|e| format!("Unable to build caches: {:?}", e))?; - // Transition the parent state to the block slot. - for i in 0..slots { - per_slot_processing(&mut state, None, spec) - .map_err(|e| format!("Failed to advance slot on iteration {}: {:?}", i, e))?; + let state_root = if let Some(root) = cli_state_root.or(state_root) { + root + } else { + state + .update_tree_hash_cache() + .map_err(|e| format!("Unable to build THC: {:?}", e))? + }; + + for i in 0..runs { + let mut state = state.clone_with(CloneConfig::committee_caches_only()); + + let start = Instant::now(); + + if partial { + partial_state_advance(&mut state, Some(state_root), target_slot, spec) + .map_err(|e| format!("Unable to perform partial advance: {:?}", e))?; + } else { + complete_state_advance(&mut state, Some(state_root), target_slot, spec) + .map_err(|e| format!("Unable to perform complete advance: {:?}", e))?; + } + + let duration = Instant::now().duration_since(start); + info!("Run {}: {:?}", i, duration); } - let mut output_file = - File::create(output_path).map_err(|e| format!("Unable to create output file: {:?}", e))?; + if let Some(output_path) = output_path { + let mut output_file = File::create(output_path) + .map_err(|e| format!("Unable to create output file: {:?}", e))?; - output_file - .write_all(&state.as_ssz_bytes()) - .map_err(|e| format!("Unable to write to output file: {:?}", e))?; + output_file + .write_all(&state.as_ssz_bytes()) + .map_err(|e| format!("Unable to write to output file: {:?}", e))?; + } Ok(()) } diff --git a/lcli/src/transition_blocks.rs b/lcli/src/transition_blocks.rs index 74be1e628..793bdb642 100644 --- a/lcli/src/transition_blocks.rs +++ b/lcli/src/transition_blocks.rs @@ -1,125 +1,387 @@ +//! # Transition Blocks +//! +//! Use this tool to apply a `SignedBeaconBlock` to a `BeaconState`. Useful for benchmarking or +//! troubleshooting consensus failures. +//! +//! It can load states and blocks from file or pull them from a beaconAPI. Objects pulled from a +//! beaconAPI can be saved to disk to reduce future calls to that server. +//! +//! ## Examples +//! +//! ### Run using a block from a beaconAPI +//! +//! Download the 0x6c69 block and its pre-state (the state from its parent block) from the +//! beaconAPI. Advance the pre-state to the slot of the 0x6c69 block and apply that block to the +//! pre-state. +//! +//! ```ignore +//! lcli transition-blocks \ +//! --beacon-url http://localhost:5052 \ +//! --block-id 0x6c69cf50a451f1ec905e954bf1fa22970f371a72a5aa9f8e3a43a18fdd980bec \ +//! --runs 10 +//! ``` +//! +//! ### Download a block and pre-state from a beaconAPI to the filesystem +//! +//! Download a block and pre-state to the filesystem, without performing any transitions: +//! +//! ```ignore +//! lcli transition-blocks \ +//! --beacon-url http://localhost:5052 \ +//! --block-id 0x6c69cf50a451f1ec905e954bf1fa22970f371a72a5aa9f8e3a43a18fdd980bec \ +//! --runs 0 \ +//! --block-output-path /tmp/block-0x6c69.ssz \ +//! --pre-state-output-path /tmp/pre-state-0x6c69.ssz +//! ``` +//! +//! ### Use a block and pre-state from the filesystem +//! +//! Do one run over the block and pre-state downloaded in the previous example and save the post +//! state to file: +//! +//! ```ignore +//! lcli transition-blocks \ +//! --block-path /tmp/block-0x6c69.ssz \ +//! --pre-state-path /tmp/pre-state-0x6c69.ssz +//! --post-state-output-path /tmp/post-state-0x6c69.ssz +//! ``` +//! +//! ### Isolate block processing for benchmarking +//! +//! Try to isolate block processing as much as possible for benchmarking: +//! +//! ```ignore +//! lcli transition-blocks \ +//! --block-path /tmp/block-0x6c69.ssz \ +//! --pre-state-path /tmp/pre-state-0x6c69.ssz \ +//! --runs 10 \ +//! --exclude-cache-builds \ +//! --exclude-post-block-thc +//! ``` +use beacon_chain::{ + test_utils::EphemeralHarnessType, validator_pubkey_cache::ValidatorPubkeyCache, +}; use clap::ArgMatches; -use eth2_network_config::Eth2NetworkConfig; +use clap_utils::{parse_optional, parse_required}; +use environment::{null_logger, Environment}; +use eth2::{ + types::{BlockId, StateId}, + BeaconNodeHttpClient, SensitiveUrl, Timeouts, +}; use ssz::Encode; use state_processing::{ - per_block_processing, per_slot_processing, BlockSignatureStrategy, VerifyBlockRoot, + block_signature_verifier::BlockSignatureVerifier, per_block_processing, per_slot_processing, + BlockSignatureStrategy, VerifyBlockRoot, }; +use std::borrow::Cow; use std::fs::File; use std::io::prelude::*; use std::path::{Path, PathBuf}; -use std::time::Instant; -use types::{BeaconState, ChainSpec, EthSpec, SignedBeaconBlock}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use store::HotColdDB; +use types::{BeaconState, ChainSpec, CloneConfig, EthSpec, Hash256, SignedBeaconBlock}; -pub fn run_transition_blocks( - testnet_dir: PathBuf, - matches: &ArgMatches, -) -> Result<(), String> { - let pre_state_path = matches - .value_of("pre-state") - .ok_or("No pre-state file supplied")? - .parse::() - .map_err(|e| format!("Failed to parse pre-state path: {}", e))?; +const HTTP_TIMEOUT: Duration = Duration::from_secs(10); - let block_path = matches - .value_of("block") - .ok_or("No block file supplied")? - .parse::() - .map_err(|e| format!("Failed to parse block path: {}", e))?; +#[derive(Debug)] +struct Config { + no_signature_verification: bool, + exclude_cache_builds: bool, + exclude_post_block_thc: bool, +} - let output_path = matches - .value_of("output") - .ok_or("No output file supplied")? - .parse::() - .map_err(|e| format!("Failed to parse output path: {}", e))?; +pub fn run(mut env: Environment, matches: &ArgMatches) -> Result<(), String> { + let spec = &T::default_spec(); + let executor = env.core_context().executor; - let no_signature_verification = matches.is_present("no-signature-verification"); - let signature_strategy = if no_signature_verification { - BlockSignatureStrategy::NoVerification - } else { - BlockSignatureStrategy::VerifyIndividual + /* + * Parse (most) CLI arguments. + */ + + let pre_state_path: Option = parse_optional(matches, "pre-state-path")?; + let block_path: Option = parse_optional(matches, "block-path")?; + let post_state_output_path: Option = + parse_optional(matches, "post-state-output-path")?; + let pre_state_output_path: Option = parse_optional(matches, "pre-state-output-path")?; + let block_output_path: Option = parse_optional(matches, "block-output-path")?; + let beacon_url: Option = parse_optional(matches, "beacon-url")?; + let runs: usize = parse_required(matches, "runs")?; + let config = Config { + no_signature_verification: matches.is_present("no-signature-verification"), + exclude_cache_builds: matches.is_present("exclude-cache-builds"), + exclude_post_block_thc: matches.is_present("exclude-post-block-thc"), }; info!("Using {} spec", T::spec_name()); - info!("Pre-state path: {:?}", pre_state_path); - info!("Block path: {:?}", block_path); + info!("Doing {} runs", runs); + info!("{:?}", &config); - let eth2_network_config = Eth2NetworkConfig::load(testnet_dir)?; - let spec = ð2_network_config.chain_spec::()?; + /* + * Load the block and pre-state from disk or beaconAPI URL. + */ - let pre_state: BeaconState = - load_from_ssz_with(&pre_state_path, spec, BeaconState::from_ssz_bytes)?; - let block: SignedBeaconBlock = - load_from_ssz_with(&block_path, spec, SignedBeaconBlock::from_ssz_bytes)?; + let (mut pre_state, mut state_root_opt, block) = match (pre_state_path, block_path, beacon_url) + { + (Some(pre_state_path), Some(block_path), None) => { + info!("Block path: {:?}", pre_state_path); + info!("Pre-state path: {:?}", block_path); + let pre_state = load_from_ssz_with(&pre_state_path, spec, BeaconState::from_ssz_bytes)?; + let block = load_from_ssz_with(&block_path, spec, SignedBeaconBlock::from_ssz_bytes)?; + (pre_state, None, block) + } + (None, None, Some(beacon_url)) => { + let block_id: BlockId = parse_required(matches, "block-id")?; + let client = BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(HTTP_TIMEOUT)); + executor + .handle() + .ok_or("shutdown in progress")? + .block_on(async move { + let block = client + .get_beacon_blocks(block_id) + .await + .map_err(|e| format!("Failed to download block: {:?}", e))? + .ok_or_else(|| format!("Unable to locate block at {:?}", block_id))? + .data; - let t = Instant::now(); - let post_state = do_transition(pre_state, block, signature_strategy, spec)?; - println!("Total transition time: {}ms", t.elapsed().as_millis()); + if block.slot() == spec.genesis_slot { + return Err("Cannot run on the genesis block".to_string()); + } - let mut output_file = - File::create(output_path).map_err(|e| format!("Unable to create output file: {:?}", e))?; + let parent_block: SignedBeaconBlock = client + .get_beacon_blocks(BlockId::Root(block.parent_root())) + .await + .map_err(|e| format!("Failed to download parent block: {:?}", e))? + .ok_or_else(|| format!("Unable to locate parent block at {:?}", block_id))? + .data; - output_file - .write_all(&post_state.as_ssz_bytes()) - .map_err(|e| format!("Unable to write to output file: {:?}", e))?; + let state_root = parent_block.state_root(); + let state_id = StateId::Root(state_root); + let pre_state = client + .get_debug_beacon_states::(state_id) + .await + .map_err(|e| format!("Failed to download state: {:?}", e))? + .ok_or_else(|| format!("Unable to locate state at {:?}", state_id))? + .data; + + Ok((pre_state, Some(state_root), block)) + }) + .map_err(|e| format!("Failed to complete task: {:?}", e))? + } + _ => { + return Err( + "must supply *both* --pre-state-path and --block-path *or* only --beacon-url" + .into(), + ) + } + }; + + // Compute the block root. + let block_root = block.canonical_root(); + + /* + * Create a `BeaconStore` and `ValidatorPubkeyCache` for block signature verification. + */ + + let store = HotColdDB::open_ephemeral( + <_>::default(), + spec.clone(), + null_logger().map_err(|e| format!("Failed to create null_logger: {:?}", e))?, + ) + .map_err(|e| format!("Failed to create ephemeral store: {:?}", e))?; + let store = Arc::new(store); + + debug!("Building pubkey cache (might take some time)"); + let validator_pubkey_cache = ValidatorPubkeyCache::new(&pre_state, store) + .map_err(|e| format!("Failed to create pubkey cache: {:?}", e))?; + + /* + * If cache builds are excluded from the timings, build them early so they are available for + * each run. + */ + + if config.exclude_cache_builds { + pre_state + .build_all_caches(spec) + .map_err(|e| format!("Unable to build caches: {:?}", e))?; + let state_root = pre_state + .update_tree_hash_cache() + .map_err(|e| format!("Unable to build THC: {:?}", e))?; + + if state_root_opt.map_or(false, |expected| expected != state_root) { + return Err(format!( + "State root mismatch! Expected {}, computed {}", + state_root_opt.unwrap(), + state_root + )); + } + state_root_opt = Some(state_root); + } + + /* + * Perform the core "runs". + */ + + let mut output_post_state = None; + for i in 0..runs { + let pre_state = pre_state.clone_with(CloneConfig::all()); + let block = block.clone(); + + let start = Instant::now(); + + let post_state = do_transition( + pre_state, + block_root, + block, + state_root_opt, + &config, + &validator_pubkey_cache, + spec, + )?; + + let duration = Instant::now().duration_since(start); + info!("Run {}: {:?}", i, duration); + + if output_post_state.is_none() { + output_post_state = Some(post_state) + } + } + + /* + * Write artifacts to disk, if required. + */ + + if let Some(path) = post_state_output_path { + let output_post_state = output_post_state.ok_or_else(|| { + format!( + "Post state was not computed, cannot save to disk (runs = {})", + runs + ) + })?; + + let mut output_file = + File::create(path).map_err(|e| format!("Unable to create output file: {:?}", e))?; + + output_file + .write_all(&output_post_state.as_ssz_bytes()) + .map_err(|e| format!("Unable to write to output file: {:?}", e))?; + } + + if let Some(path) = pre_state_output_path { + let mut output_file = + File::create(path).map_err(|e| format!("Unable to create output file: {:?}", e))?; + + output_file + .write_all(&pre_state.as_ssz_bytes()) + .map_err(|e| format!("Unable to write to output file: {:?}", e))?; + } + + if let Some(path) = block_output_path { + let mut output_file = + File::create(path).map_err(|e| format!("Unable to create output file: {:?}", e))?; + + output_file + .write_all(&block.as_ssz_bytes()) + .map_err(|e| format!("Unable to write to output file: {:?}", e))?; + } Ok(()) } fn do_transition( mut pre_state: BeaconState, + block_root: Hash256, block: SignedBeaconBlock, - signature_strategy: BlockSignatureStrategy, + mut state_root_opt: Option, + config: &Config, + validator_pubkey_cache: &ValidatorPubkeyCache>, spec: &ChainSpec, ) -> Result, String> { - let t = Instant::now(); - pre_state - .build_all_caches(spec) - .map_err(|e| format!("Unable to build caches: {:?}", e))?; - println!("Build caches: {}ms", t.elapsed().as_millis()); + if !config.exclude_cache_builds { + let t = Instant::now(); + pre_state + .build_all_caches(spec) + .map_err(|e| format!("Unable to build caches: {:?}", e))?; + debug!("Build caches: {:?}", t.elapsed()); - let t = Instant::now(); - pre_state - .update_tree_hash_cache() - .map_err(|e| format!("Unable to build tree hash cache: {:?}", e))?; - println!("Initial tree hash: {}ms", t.elapsed().as_millis()); + let t = Instant::now(); + let state_root = pre_state + .update_tree_hash_cache() + .map_err(|e| format!("Unable to build tree hash cache: {:?}", e))?; + debug!("Initial tree hash: {:?}", t.elapsed()); + + if state_root_opt.map_or(false, |expected| expected != state_root) { + return Err(format!( + "State root mismatch! Expected {}, computed {}", + state_root_opt.unwrap(), + state_root + )); + } + state_root_opt = Some(state_root); + } + + let state_root = state_root_opt.ok_or("Failed to compute state root, internal error")?; // Transition the parent state to the block slot. let t = Instant::now(); for i in pre_state.slot().as_u64()..block.slot().as_u64() { - per_slot_processing(&mut pre_state, None, spec) + per_slot_processing(&mut pre_state, Some(state_root), spec) .map_err(|e| format!("Failed to advance slot on iteration {}: {:?}", i, e))?; } - println!("Slot processing: {}ms", t.elapsed().as_millis()); - - let t = Instant::now(); - pre_state - .update_tree_hash_cache() - .map_err(|e| format!("Unable to build tree hash cache: {:?}", e))?; - println!("Pre-block tree hash: {}ms", t.elapsed().as_millis()); + debug!("Slot processing: {:?}", t.elapsed()); let t = Instant::now(); pre_state .build_all_caches(spec) .map_err(|e| format!("Unable to build caches: {:?}", e))?; - println!("Build all caches (again): {}ms", t.elapsed().as_millis()); + debug!("Build all caches (again): {:?}", t.elapsed()); + + if !config.no_signature_verification { + let get_pubkey = move |validator_index| { + validator_pubkey_cache + .get(validator_index) + .map(Cow::Borrowed) + }; + + let decompressor = move |pk_bytes| { + // Map compressed pubkey to validator index. + let validator_index = validator_pubkey_cache.get_index(pk_bytes)?; + // Map validator index to pubkey (respecting guard on unknown validators). + get_pubkey(validator_index) + }; + + let t = Instant::now(); + BlockSignatureVerifier::verify_entire_block( + &pre_state, + get_pubkey, + decompressor, + &block, + Some(block_root), + spec, + ) + .map_err(|e| format!("Invalid block signature: {:?}", e))?; + debug!("Batch verify block signatures: {:?}", t.elapsed()); + } let t = Instant::now(); per_block_processing( &mut pre_state, &block, None, - signature_strategy, + BlockSignatureStrategy::NoVerification, VerifyBlockRoot::True, spec, ) .map_err(|e| format!("State transition failed: {:?}", e))?; - println!("Process block: {}ms", t.elapsed().as_millis()); + debug!("Process block: {:?}", t.elapsed()); - let t = Instant::now(); - pre_state - .update_tree_hash_cache() - .map_err(|e| format!("Unable to build tree hash cache: {:?}", e))?; - println!("Post-block tree hash: {}ms", t.elapsed().as_millis()); + if !config.exclude_post_block_thc { + let t = Instant::now(); + pre_state + .update_tree_hash_cache() + .map_err(|e| format!("Unable to build tree hash cache: {:?}", e))?; + debug!("Post-block tree hash: {:?}", t.elapsed()); + } Ok(pre_state) } @@ -136,10 +398,6 @@ pub fn load_from_ssz_with( .map_err(|e| format!("Unable to read from file {:?}: {:?}", path, e))?; let t = Instant::now(); let result = decoder(&bytes, spec).map_err(|e| format!("Ssz decode failed: {:?}", e)); - println!( - "SSZ decoding {}: {}ms", - path.display(), - t.elapsed().as_millis() - ); + debug!("SSZ decoding {}: {:?}", path.display(), t.elapsed()); result }