cerc-io/lighthouse
16
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Add support for beaconAPI in lcli functions (#3252)

Browse Source 
## Issue Addressed

NA

## Proposed Changes

Modifies `lcli skip-slots` and `lcli transition-blocks` allow them to source blocks/states from a beaconAPI and also gives them some more features to assist with benchmarking.

## Additional Info

Breaks the current `lcli skip-slots` and `lcli transition-blocks` APIs by changing some flag names. It should be simple enough to figure out the changes via `--help`.

Currently blocked on #3263.
This commit is contained in:
Paul Hauner 2022-08-09 06:05:13 +00:00
parent 68bd7cae21
commit a688621919
10 changed files with 607 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock generated
View File

@ -3105,6 +3105,7 @@ name = "lcli"
version = "2.5.1" version = "2.5.1"
dependencies = [ dependencies = [
"account_utils", "account_utils",
"beacon_chain",
"bls", "bls",
"clap", "clap",
"clap_utils", "clap_utils",
@ -3128,6 +3129,7 @@ dependencies = [
"serde_yaml", "serde_yaml",
"snap", "snap",
"state_processing", "state_processing",
"store",
"tree_hash", "tree_hash",
"types", "types",
"validator_dir", "validator_dir",

2
beacon_node/beacon_chain/src/lib.rs
View File

@ -41,7 +41,7 @@ pub mod sync_committee_verification;
pub mod test_utils; pub mod test_utils;
mod timeout_rw_lock; mod timeout_rw_lock;
pub mod validator_monitor; pub mod validator_monitor;
mod validator_pubkey_cache; pub mod validator_pubkey_cache;
pub use self::beacon_chain::{ pub use self::beacon_chain::{
AttestationProcessingOutcome, BeaconChain, BeaconChainTypes, BeaconStore, ChainSegmentResult, AttestationProcessingOutcome, BeaconChain, BeaconChainTypes, BeaconStore, ChainSegmentResult,

5
beacon_node/beacon_chain/src/validator_pubkey_cache.rs
View File

@ -156,6 +156,11 @@ impl<T: BeaconChainTypes> ValidatorPubkeyCache<T> {
pub fn len(&self) -> usize { pub fn len(&self) -> usize {
self.indices.len() self.indices.len()
} }
/// Returns `true` if there are no validators in the cache.
pub fn is_empty(&self) -> bool {
self.indices.is_empty()
}
} }
/// Wrapper for a public key stored in the database. /// Wrapper for a public key stored in the database.

2
common/eth2/src/lib.rs
View File

@ -23,7 +23,7 @@ use lighthouse_network::PeerId;
pub use reqwest; pub use reqwest;
use reqwest::{IntoUrl, RequestBuilder, Response}; use reqwest::{IntoUrl, RequestBuilder, Response};
pub use reqwest::{StatusCode, Url}; pub use reqwest::{StatusCode, Url};
use sensitive_url::SensitiveUrl; pub use sensitive_url::SensitiveUrl;
use serde::{de::DeserializeOwned, Serialize}; use serde::{de::DeserializeOwned, Serialize};
use std::convert::TryFrom; use std::convert::TryFrom;
use std::fmt; use std::fmt;

15
common/sensitive_url/src/lib.rs
View File

@ -1,5 +1,6 @@
use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
use std::fmt; use std::fmt;
use std::str::FromStr;
use url::Url; use url::Url;
#[derive(Debug)] #[derive(Debug)]
@ -9,6 +10,12 @@ pub enum SensitiveError {
RedactError(String), RedactError(String),
} }
impl fmt::Display for SensitiveError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?}", self)
}
}
// Wrapper around Url which provides a custom `Display` implementation to protect user secrets. // Wrapper around Url which provides a custom `Display` implementation to protect user secrets.
#[derive(Clone, PartialEq)] #[derive(Clone, PartialEq)]
pub struct SensitiveUrl { pub struct SensitiveUrl {
@ -54,6 +61,14 @@ impl<'de> Deserialize<'de> for SensitiveUrl {
} }
} }
impl FromStr for SensitiveUrl {
type Err = SensitiveError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Self::parse(s)
}
}
impl SensitiveUrl { impl SensitiveUrl {
pub fn parse(url: &str) -> Result<Self, SensitiveError> { pub fn parse(url: &str) -> Result<Self, SensitiveError> {
let surl = Url::parse(url).map_err(SensitiveError::ParseError)?; let surl = Url::parse(url).map_err(SensitiveError::ParseError)?;

2
consensus/state_processing/src/per_block_processing.rs
View File

@ -40,7 +40,7 @@ use arbitrary::Arbitrary;
/// The strategy to be used when validating the block's signatures. /// The strategy to be used when validating the block's signatures.
#[cfg_attr(feature = "arbitrary-fuzz", derive(Arbitrary))] #[cfg_attr(feature = "arbitrary-fuzz", derive(Arbitrary))]
#[derive(PartialEq, Clone, Copy)] #[derive(PartialEq, Clone, Copy, Debug)]
pub enum BlockSignatureStrategy { pub enum BlockSignatureStrategy {
/// Do not validate any signature. Use with caution. /// Do not validate any signature. Use with caution.
NoVerification, NoVerification,

2
lcli/Cargo.toml
View File

@ -38,3 +38,5 @@ eth1_test_rig = { path = "../testing/eth1_test_rig" }
sensitive_url = { path = "../common/sensitive_url" } sensitive_url = { path = "../common/sensitive_url" }
eth2 = { path = "../common/eth2" } eth2 = { path = "../common/eth2" }
snap = "1.0.1" snap = "1.0.1"
beacon_chain = { path = "../beacon_node/beacon_chain" }
store = { path = "../beacon_node/store" }

150
lcli/src/main.rs
View File

@ -22,7 +22,6 @@ use parse_ssz::run_parse_ssz;
use std::path::PathBuf; use std::path::PathBuf;
use std::process; use std::process;
use std::str::FromStr; use std::str::FromStr;
use transition_blocks::run_transition_blocks;
use types::{EthSpec, EthSpecId}; use types::{EthSpec, EthSpecId};
fn main() { fn main() {
@ -57,52 +56,128 @@ fn main() {
"Performs a state transition from some state across some number of skip slots", "Performs a state transition from some state across some number of skip slots",
) )
.arg( .arg(
Arg::with_name("pre-state") Arg::with_name("output-path")
.value_name("BEACON_STATE") .long("output-path")
.value_name("PATH")
.takes_value(true) .takes_value(true)
.required(true) .help("Path to output a SSZ file."),
)
.arg(
Arg::with_name("pre-state-path")
.long("pre-state-path")
.value_name("PATH")
.takes_value(true)
.conflicts_with("beacon-url")
.help("Path to a SSZ file of the pre-state."), .help("Path to a SSZ file of the pre-state."),
) )
.arg( .arg(
Arg::with_name("slots") Arg::with_name("beacon-url")
.value_name("SLOT_COUNT") .long("beacon-url")
.value_name("URL")
.takes_value(true) .takes_value(true)
.required(true) .help("URL to a beacon-API provider."),
.help("Number of slots to skip before outputting a state.."),
) )
.arg( .arg(
Arg::with_name("output") Arg::with_name("state-id")
.value_name("SSZ_FILE") .long("state-id")
.value_name("STATE_ID")
.takes_value(true) .takes_value(true)
.required(true) .requires("beacon-url")
.default_value("./output.ssz") .help("Identifier for a state as per beacon-API standards (slot, root, etc.)"),
.help("Path to output a SSZ file."), )
), .arg(
Arg::with_name("runs")
.long("runs")
.value_name("INTEGER")
.takes_value(true)
.default_value("1")
.help("Number of repeat runs, useful for benchmarking."),
)
.arg(
Arg::with_name("state-root")
.long("state-root")
.value_name("HASH256")
.takes_value(true)
.help("Tree hash root of the provided state, to avoid computing it."),
)
.arg(
Arg::with_name("slots")
.long("slots")
.value_name("INTEGER")
.takes_value(true)
.help("Number of slots to skip forward."),
)
.arg(
Arg::with_name("partial-state-advance")
.long("partial-state-advance")
.takes_value(false)
.help("If present, don't compute state roots when skipping forward."),
)
) )
.subcommand( .subcommand(
SubCommand::with_name("transition-blocks") SubCommand::with_name("transition-blocks")
.about("Performs a state transition given a pre-state and block") .about("Performs a state transition given a pre-state and block")
.arg( .arg(
Arg::with_name("pre-state") Arg::with_name("pre-state-path")
.value_name("BEACON_STATE") .long("pre-state-path")
.value_name("PATH")
.takes_value(true) .takes_value(true)
.required(true) .conflicts_with("beacon-url")
.help("Path to a SSZ file of the pre-state."), .requires("block-path")
.help("Path to load a BeaconState from file as SSZ."),
) )
.arg( .arg(
Arg::with_name("block") Arg::with_name("block-path")
.value_name("BEACON_BLOCK") .long("block-path")
.value_name("PATH")
.takes_value(true) .takes_value(true)
.required(true) .conflicts_with("beacon-url")
.help("Path to a SSZ file of the block to apply to pre-state."), .requires("pre-state-path")
.help("Path to load a SignedBeaconBlock from file as SSZ."),
) )
.arg( .arg(
Arg::with_name("output") Arg::with_name("post-state-output-path")
.value_name("SSZ_FILE") .long("post-state-output-path")
.value_name("PATH")
.takes_value(true) .takes_value(true)
.required(true) .help("Path to output the post-state."),
.default_value("./output.ssz") )
.help("Path to output a SSZ file."), .arg(
Arg::with_name("pre-state-output-path")
.long("pre-state-output-path")
.value_name("PATH")
.takes_value(true)
.help("Path to output the pre-state, useful when used with --beacon-url."),
)
.arg(
Arg::with_name("block-output-path")
.long("block-output-path")
.value_name("PATH")
.takes_value(true)
.help("Path to output the block, useful when used with --beacon-url."),
)
.arg(
Arg::with_name("beacon-url")
.long("beacon-url")
.value_name("URL")
.takes_value(true)
.help("URL to a beacon-API provider."),
)
.arg(
Arg::with_name("block-id")
.long("block-id")
.value_name("BLOCK_ID")
.takes_value(true)
.requires("beacon-url")
.help("Identifier for a block as per beacon-API standards (slot, root, etc.)"),
)
.arg(
Arg::with_name("runs")
.long("runs")
.value_name("INTEGER")
.takes_value(true)
.default_value("1")
.help("Number of repeat runs, useful for benchmarking."),
) )
.arg( .arg(
Arg::with_name("no-signature-verification") Arg::with_name("no-signature-verification")
@ -110,6 +185,20 @@ fn main() {
.takes_value(false) .takes_value(false)
.help("Disable signature verification.") .help("Disable signature verification.")
) )
.arg(
Arg::with_name("exclude-cache-builds")
.long("exclude-cache-builds")
.takes_value(false)
.help("If present, pre-build the committee and tree-hash caches without \
including them in the timings."),
)
.arg(
Arg::with_name("exclude-post-block-thc")
.long("exclude-post-block-thc")
.takes_value(false)
.help("If present, don't rebuild the tree-hash-cache after applying \
the block."),
)
) )
.subcommand( .subcommand(
SubCommand::with_name("pretty-ssz") SubCommand::with_name("pretty-ssz")
@ -673,10 +762,11 @@ fn run<T: EthSpec>(
)?; )?;
match matches.subcommand() { match matches.subcommand() {
("transition-blocks", Some(matches)) => run_transition_blocks::<T>(testnet_dir, matches) ("transition-blocks", Some(matches)) => transition_blocks::run::<T>(env, matches)
.map_err(|e| format!("Failed to transition blocks: {}", e)), .map_err(|e| format!("Failed to transition blocks: {}", e)),
("skip-slots", Some(matches)) => skip_slots::run::<T>(testnet_dir, matches) ("skip-slots", Some(matches)) => {
.map_err(|e| format!("Failed to skip slots: {}", e)), skip_slots::run::<T>(env, matches).map_err(|e| format!("Failed to skip slots: {}", e))
}
("pretty-ssz", Some(matches)) => { ("pretty-ssz", Some(matches)) => {
run_parse_ssz::<T>(matches).map_err(|e| format!("Failed to pretty print hex: {}", e)) run_parse_ssz::<T>(matches).map_err(|e| format!("Failed to pretty print hex: {}", e))
} }

160
lcli/src/skip_slots.rs
View File

@ -1,58 +1,150 @@
//! # Skip-Slots
//!
//! Use this tool to process a `BeaconState` through empty slots. Useful for benchmarking or
//! troubleshooting consensus failures.
//!
//! It can load states from file or pull them from a beaconAPI. States pulled from a beaconAPI can
//! be saved to disk to reduce future calls to that server.
//!
//! ## Examples
//!
//! ### Example 1.
//!
//! Download a state from a HTTP endpoint and skip forward an epoch, twice (the initial state is
//! advanced 32 slots twice, rather than it being advanced 64 slots):
//!
//! ```ignore
//! lcli skip-slots \
//! --beacon-url http://localhost:5052 \
//! --state-id 0x3cdc33cd02713d8d6cc33a6dbe2d3a5bf9af1d357de0d175a403496486ff845e \\
//! --slots 32 \
//! --runs 2
//! ```
//!
//! ### Example 2.
//!
//! Download a state to a SSZ file (without modifying it):
//!
//! ```ignore
//! lcli skip-slots \
//! --beacon-url http://localhost:5052 \
//! --state-id 0x3cdc33cd02713d8d6cc33a6dbe2d3a5bf9af1d357de0d175a403496486ff845e \
//! --slots 0 \
//! --runs 0 \
//! --output-path /tmp/state-0x3cdc.ssz
//! ```
//!
//! ### Example 3.
//!
//! Do two runs over the state that was downloaded in the previous example:
//!
//! ```ignore
//! lcli skip-slots \
//! --pre-state-path /tmp/state-0x3cdc.ssz \
//! --slots 32 \
//! --runs 2
//! ```
use crate::transition_blocks::load_from_ssz_with; use crate::transition_blocks::load_from_ssz_with;
use clap::ArgMatches; use clap::ArgMatches;
use eth2_network_config::Eth2NetworkConfig; use clap_utils::{parse_optional, parse_required};
use environment::Environment;
use eth2::{types::StateId, BeaconNodeHttpClient, SensitiveUrl, Timeouts};
use ssz::Encode; use ssz::Encode;
use state_processing::per_slot_processing; use state_processing::state_advance::{complete_state_advance, partial_state_advance};
use std::fs::File; use std::fs::File;
use std::io::prelude::*; use std::io::prelude::*;
use std::path::PathBuf; use std::path::PathBuf;
use types::{BeaconState, EthSpec}; use std::time::{Duration, Instant};
use types::{BeaconState, CloneConfig, EthSpec, Hash256};
pub fn run<T: EthSpec>(testnet_dir: PathBuf, matches: &ArgMatches) -> Result<(), String> { const HTTP_TIMEOUT: Duration = Duration::from_secs(10);
let pre_state_path = matches
.value_of("pre-state")
.ok_or("No pre-state file supplied")?
.parse::<PathBuf>()
.map_err(|e| format!("Failed to parse pre-state path: {}", e))?;
let slots = matches pub fn run<T: EthSpec>(mut env: Environment<T>, matches: &ArgMatches) -> Result<(), String> {
.value_of("slots") let spec = &T::default_spec();
.ok_or("No slots supplied")? let executor = env.core_context().executor;
.parse::<usize>()
.map_err(|e| format!("Failed to parse slots: {}", e))?;
let output_path = matches let output_path: Option<PathBuf> = parse_optional(matches, "output-path")?;
.value_of("output") let state_path: Option<PathBuf> = parse_optional(matches, "pre-state-path")?;
.ok_or("No output file supplied")? let beacon_url: Option<SensitiveUrl> = parse_optional(matches, "beacon-url")?;
.parse::<PathBuf>() let runs: usize = parse_required(matches, "runs")?;
.map_err(|e| format!("Failed to parse output path: {}", e))?; let slots: u64 = parse_required(matches, "slots")?;
let cli_state_root: Option<Hash256> = parse_optional(matches, "state-root")?;
let partial: bool = matches.is_present("partial-state-advance");
info!("Using {} spec", T::spec_name()); info!("Using {} spec", T::spec_name());
info!("Pre-state path: {:?}", pre_state_path); info!("Advancing {} slots", slots);
info!("Slots: {:?}", slots); info!("Doing {} runs", runs);
let eth2_network_config = Eth2NetworkConfig::load(testnet_dir)?; let (mut state, state_root) = match (state_path, beacon_url) {
let spec = &eth2_network_config.chain_spec::<T>()?; (Some(state_path), None) => {
info!("State path: {:?}", state_path);
let state = load_from_ssz_with(&state_path, spec, BeaconState::from_ssz_bytes)?;
(state, None)
}
(None, Some(beacon_url)) => {
let state_id: StateId = parse_required(matches, "state-id")?;
let client = BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(HTTP_TIMEOUT));
let state = executor
.handle()
.ok_or("shutdown in progress")?
.block_on(async move {
client
.get_debug_beacon_states::<T>(state_id)
.await
.map_err(|e| format!("Failed to download state: {:?}", e))
})
.map_err(|e| format!("Failed to complete task: {:?}", e))?
.ok_or_else(|| format!("Unable to locate state at {:?}", state_id))?
.data;
let state_root = match state_id {
StateId::Root(root) => Some(root),
_ => None,
};
(state, state_root)
}
_ => return Err("must supply either --state-path or --beacon-url".into()),
};
let mut state: BeaconState<T> = let initial_slot = state.slot();
load_from_ssz_with(&pre_state_path, spec, BeaconState::from_ssz_bytes)?; let target_slot = initial_slot + slots;
state state
.build_all_caches(spec) .build_all_caches(spec)
.map_err(|e| format!("Unable to build caches: {:?}", e))?; .map_err(|e| format!("Unable to build caches: {:?}", e))?;
// Transition the parent state to the block slot. let state_root = if let Some(root) = cli_state_root.or(state_root) {
for i in 0..slots { root
per_slot_processing(&mut state, None, spec) } else {
.map_err(|e| format!("Failed to advance slot on iteration {}: {:?}", i, e))?; state
.update_tree_hash_cache()
.map_err(|e| format!("Unable to build THC: {:?}", e))?
};
for i in 0..runs {
let mut state = state.clone_with(CloneConfig::committee_caches_only());
let start = Instant::now();
if partial {
partial_state_advance(&mut state, Some(state_root), target_slot, spec)
.map_err(|e| format!("Unable to perform partial advance: {:?}", e))?;
} else {
complete_state_advance(&mut state, Some(state_root), target_slot, spec)
.map_err(|e| format!("Unable to perform complete advance: {:?}", e))?;
}
let duration = Instant::now().duration_since(start);
info!("Run {}: {:?}", i, duration);
} }
let mut output_file = if let Some(output_path) = output_path {
File::create(output_path).map_err(|e| format!("Unable to create output file: {:?}", e))?; let mut output_file = File::create(output_path)
.map_err(|e| format!("Unable to create output file: {:?}", e))?;
output_file output_file
.write_all(&state.as_ssz_bytes()) .write_all(&state.as_ssz_bytes())
.map_err(|e| format!("Unable to write to output file: {:?}", e))?; .map_err(|e| format!("Unable to write to output file: {:?}", e))?;
}
Ok(()) Ok(())
} }

410
lcli/src/transition_blocks.rs
View File

@ -1,125 +1,387 @@
//! # Transition Blocks
//!
//! Use this tool to apply a `SignedBeaconBlock` to a `BeaconState`. Useful for benchmarking or
//! troubleshooting consensus failures.
//!
//! It can load states and blocks from file or pull them from a beaconAPI. Objects pulled from a
//! beaconAPI can be saved to disk to reduce future calls to that server.
//!
//! ## Examples
//!
//! ### Run using a block from a beaconAPI
//!
//! Download the 0x6c69 block and its pre-state (the state from its parent block) from the
//! beaconAPI. Advance the pre-state to the slot of the 0x6c69 block and apply that block to the
//! pre-state.
//!
//! ```ignore
//! lcli transition-blocks \
//! --beacon-url http://localhost:5052 \
//! --block-id 0x6c69cf50a451f1ec905e954bf1fa22970f371a72a5aa9f8e3a43a18fdd980bec \
//! --runs 10
//! ```
//!
//! ### Download a block and pre-state from a beaconAPI to the filesystem
//!
//! Download a block and pre-state to the filesystem, without performing any transitions:
//!
//! ```ignore
//! lcli transition-blocks \
//! --beacon-url http://localhost:5052 \
//! --block-id 0x6c69cf50a451f1ec905e954bf1fa22970f371a72a5aa9f8e3a43a18fdd980bec \
//! --runs 0 \
//! --block-output-path /tmp/block-0x6c69.ssz \
//! --pre-state-output-path /tmp/pre-state-0x6c69.ssz
//! ```
//!
//! ### Use a block and pre-state from the filesystem
//!
//! Do one run over the block and pre-state downloaded in the previous example and save the post
//! state to file:
//!
//! ```ignore
//! lcli transition-blocks \
//! --block-path /tmp/block-0x6c69.ssz \
//! --pre-state-path /tmp/pre-state-0x6c69.ssz
//! --post-state-output-path /tmp/post-state-0x6c69.ssz
//! ```
//!
//! ### Isolate block processing for benchmarking
//!
//! Try to isolate block processing as much as possible for benchmarking:
//!
//! ```ignore
//! lcli transition-blocks \
//! --block-path /tmp/block-0x6c69.ssz \
//! --pre-state-path /tmp/pre-state-0x6c69.ssz \
//! --runs 10 \
//! --exclude-cache-builds \
//! --exclude-post-block-thc
//! ```
use beacon_chain::{
test_utils::EphemeralHarnessType, validator_pubkey_cache::ValidatorPubkeyCache,
};
use clap::ArgMatches; use clap::ArgMatches;
use eth2_network_config::Eth2NetworkConfig; use clap_utils::{parse_optional, parse_required};
use environment::{null_logger, Environment};
use eth2::{
types::{BlockId, StateId},
BeaconNodeHttpClient, SensitiveUrl, Timeouts,
};
use ssz::Encode; use ssz::Encode;
use state_processing::{ use state_processing::{
per_block_processing, per_slot_processing, BlockSignatureStrategy, VerifyBlockRoot, block_signature_verifier::BlockSignatureVerifier, per_block_processing, per_slot_processing,
BlockSignatureStrategy, VerifyBlockRoot,
}; };
use std::borrow::Cow;
use std::fs::File; use std::fs::File;
use std::io::prelude::*; use std::io::prelude::*;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::time::Instant; use std::sync::Arc;
use types::{BeaconState, ChainSpec, EthSpec, SignedBeaconBlock}; use std::time::{Duration, Instant};
use store::HotColdDB;
use types::{BeaconState, ChainSpec, CloneConfig, EthSpec, Hash256, SignedBeaconBlock};
pub fn run_transition_blocks<T: EthSpec>( const HTTP_TIMEOUT: Duration = Duration::from_secs(10);
testnet_dir: PathBuf,
matches: &ArgMatches,
) -> Result<(), String> {
let pre_state_path = matches
.value_of("pre-state")
.ok_or("No pre-state file supplied")?
.parse::<PathBuf>()
.map_err(|e| format!("Failed to parse pre-state path: {}", e))?;
let block_path = matches #[derive(Debug)]
.value_of("block") struct Config {
.ok_or("No block file supplied")? no_signature_verification: bool,
.parse::<PathBuf>() exclude_cache_builds: bool,
.map_err(|e| format!("Failed to parse block path: {}", e))?; exclude_post_block_thc: bool,
}
let output_path = matches pub fn run<T: EthSpec>(mut env: Environment<T>, matches: &ArgMatches) -> Result<(), String> {
.value_of("output") let spec = &T::default_spec();
.ok_or("No output file supplied")? let executor = env.core_context().executor;
.parse::<PathBuf>()
.map_err(|e| format!("Failed to parse output path: {}", e))?;
let no_signature_verification = matches.is_present("no-signature-verification"); /*
let signature_strategy = if no_signature_verification { * Parse (most) CLI arguments.
BlockSignatureStrategy::NoVerification */
} else {
BlockSignatureStrategy::VerifyIndividual let pre_state_path: Option<PathBuf> = parse_optional(matches, "pre-state-path")?;
let block_path: Option<PathBuf> = parse_optional(matches, "block-path")?;
let post_state_output_path: Option<PathBuf> =
parse_optional(matches, "post-state-output-path")?;
let pre_state_output_path: Option<PathBuf> = parse_optional(matches, "pre-state-output-path")?;
let block_output_path: Option<PathBuf> = parse_optional(matches, "block-output-path")?;
let beacon_url: Option<SensitiveUrl> = parse_optional(matches, "beacon-url")?;
let runs: usize = parse_required(matches, "runs")?;
let config = Config {
no_signature_verification: matches.is_present("no-signature-verification"),
exclude_cache_builds: matches.is_present("exclude-cache-builds"),
exclude_post_block_thc: matches.is_present("exclude-post-block-thc"),
}; };
info!("Using {} spec", T::spec_name()); info!("Using {} spec", T::spec_name());
info!("Pre-state path: {:?}", pre_state_path); info!("Doing {} runs", runs);
info!("Block path: {:?}", block_path); info!("{:?}", &config);
let eth2_network_config = Eth2NetworkConfig::load(testnet_dir)?; /*
let spec = &eth2_network_config.chain_spec::<T>()?; * Load the block and pre-state from disk or beaconAPI URL.
*/
let pre_state: BeaconState<T> = let (mut pre_state, mut state_root_opt, block) = match (pre_state_path, block_path, beacon_url)
load_from_ssz_with(&pre_state_path, spec, BeaconState::from_ssz_bytes)?; {
let block: SignedBeaconBlock<T> = (Some(pre_state_path), Some(block_path), None) => {
load_from_ssz_with(&block_path, spec, SignedBeaconBlock::from_ssz_bytes)?; info!("Block path: {:?}", pre_state_path);
info!("Pre-state path: {:?}", block_path);
let pre_state = load_from_ssz_with(&pre_state_path, spec, BeaconState::from_ssz_bytes)?;
let block = load_from_ssz_with(&block_path, spec, SignedBeaconBlock::from_ssz_bytes)?;
(pre_state, None, block)
}
(None, None, Some(beacon_url)) => {
let block_id: BlockId = parse_required(matches, "block-id")?;
let client = BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(HTTP_TIMEOUT));
executor
.handle()
.ok_or("shutdown in progress")?
.block_on(async move {
let block = client
.get_beacon_blocks(block_id)
.await
.map_err(|e| format!("Failed to download block: {:?}", e))?
.ok_or_else(|| format!("Unable to locate block at {:?}", block_id))?
.data;
let t = Instant::now(); if block.slot() == spec.genesis_slot {
let post_state = do_transition(pre_state, block, signature_strategy, spec)?; return Err("Cannot run on the genesis block".to_string());
println!("Total transition time: {}ms", t.elapsed().as_millis()); }
let mut output_file = let parent_block: SignedBeaconBlock<T> = client
File::create(output_path).map_err(|e| format!("Unable to create output file: {:?}", e))?; .get_beacon_blocks(BlockId::Root(block.parent_root()))
.await
.map_err(|e| format!("Failed to download parent block: {:?}", e))?
.ok_or_else(|| format!("Unable to locate parent block at {:?}", block_id))?
.data;
output_file let state_root = parent_block.state_root();
.write_all(&post_state.as_ssz_bytes()) let state_id = StateId::Root(state_root);
.map_err(|e| format!("Unable to write to output file: {:?}", e))?; let pre_state = client
.get_debug_beacon_states::<T>(state_id)
.await
.map_err(|e| format!("Failed to download state: {:?}", e))?
.ok_or_else(|| format!("Unable to locate state at {:?}", state_id))?
.data;
Ok((pre_state, Some(state_root), block))
})
.map_err(|e| format!("Failed to complete task: {:?}", e))?
}
_ => {
return Err(
"must supply *both* --pre-state-path and --block-path *or* only --beacon-url"
.into(),
)
}
};
// Compute the block root.
let block_root = block.canonical_root();
/*
* Create a `BeaconStore` and `ValidatorPubkeyCache` for block signature verification.
*/
let store = HotColdDB::open_ephemeral(
<_>::default(),
spec.clone(),
null_logger().map_err(|e| format!("Failed to create null_logger: {:?}", e))?,
)
.map_err(|e| format!("Failed to create ephemeral store: {:?}", e))?;
let store = Arc::new(store);
debug!("Building pubkey cache (might take some time)");
let validator_pubkey_cache = ValidatorPubkeyCache::new(&pre_state, store)
.map_err(|e| format!("Failed to create pubkey cache: {:?}", e))?;
/*
* If cache builds are excluded from the timings, build them early so they are available for
* each run.
*/
if config.exclude_cache_builds {
pre_state
.build_all_caches(spec)
.map_err(|e| format!("Unable to build caches: {:?}", e))?;
let state_root = pre_state
.update_tree_hash_cache()
.map_err(|e| format!("Unable to build THC: {:?}", e))?;
if state_root_opt.map_or(false, |expected| expected != state_root) {
return Err(format!(
"State root mismatch! Expected {}, computed {}",
state_root_opt.unwrap(),
state_root
));
}
state_root_opt = Some(state_root);
}
/*
* Perform the core "runs".
*/
let mut output_post_state = None;
for i in 0..runs {
let pre_state = pre_state.clone_with(CloneConfig::all());
let block = block.clone();
let start = Instant::now();
let post_state = do_transition(
pre_state,
block_root,
block,
state_root_opt,
&config,
&validator_pubkey_cache,
spec,
)?;
let duration = Instant::now().duration_since(start);
info!("Run {}: {:?}", i, duration);
if output_post_state.is_none() {
output_post_state = Some(post_state)
}
}
/*
* Write artifacts to disk, if required.
*/
if let Some(path) = post_state_output_path {
let output_post_state = output_post_state.ok_or_else(|| {
format!(
"Post state was not computed, cannot save to disk (runs = {})",
runs
)
})?;
let mut output_file =
File::create(path).map_err(|e| format!("Unable to create output file: {:?}", e))?;
output_file
.write_all(&output_post_state.as_ssz_bytes())
.map_err(|e| format!("Unable to write to output file: {:?}", e))?;
}
if let Some(path) = pre_state_output_path {
let mut output_file =
File::create(path).map_err(|e| format!("Unable to create output file: {:?}", e))?;
output_file
.write_all(&pre_state.as_ssz_bytes())
.map_err(|e| format!("Unable to write to output file: {:?}", e))?;
}
if let Some(path) = block_output_path {
let mut output_file =
File::create(path).map_err(|e| format!("Unable to create output file: {:?}", e))?;
output_file
.write_all(&block.as_ssz_bytes())
.map_err(|e| format!("Unable to write to output file: {:?}", e))?;
}
Ok(()) Ok(())
} }
fn do_transition<T: EthSpec>( fn do_transition<T: EthSpec>(
mut pre_state: BeaconState<T>, mut pre_state: BeaconState<T>,
block_root: Hash256,
block: SignedBeaconBlock<T>, block: SignedBeaconBlock<T>,
signature_strategy: BlockSignatureStrategy, mut state_root_opt: Option<Hash256>,
config: &Config,
validator_pubkey_cache: &ValidatorPubkeyCache<EphemeralHarnessType<T>>,
spec: &ChainSpec, spec: &ChainSpec,
) -> Result<BeaconState<T>, String> { ) -> Result<BeaconState<T>, String> {
let t = Instant::now(); if !config.exclude_cache_builds {
pre_state let t = Instant::now();
.build_all_caches(spec) pre_state
.map_err(|e| format!("Unable to build caches: {:?}", e))?; .build_all_caches(spec)
println!("Build caches: {}ms", t.elapsed().as_millis()); .map_err(|e| format!("Unable to build caches: {:?}", e))?;
debug!("Build caches: {:?}", t.elapsed());
let t = Instant::now(); let t = Instant::now();
pre_state let state_root = pre_state
.update_tree_hash_cache() .update_tree_hash_cache()
.map_err(|e| format!("Unable to build tree hash cache: {:?}", e))?; .map_err(|e| format!("Unable to build tree hash cache: {:?}", e))?;
println!("Initial tree hash: {}ms", t.elapsed().as_millis()); debug!("Initial tree hash: {:?}", t.elapsed());
if state_root_opt.map_or(false, |expected| expected != state_root) {
return Err(format!(
"State root mismatch! Expected {}, computed {}",
state_root_opt.unwrap(),
state_root
));
}
state_root_opt = Some(state_root);
}
let state_root = state_root_opt.ok_or("Failed to compute state root, internal error")?;
// Transition the parent state to the block slot. // Transition the parent state to the block slot.
let t = Instant::now(); let t = Instant::now();
for i in pre_state.slot().as_u64()..block.slot().as_u64() { for i in pre_state.slot().as_u64()..block.slot().as_u64() {
per_slot_processing(&mut pre_state, None, spec) per_slot_processing(&mut pre_state, Some(state_root), spec)
.map_err(|e| format!("Failed to advance slot on iteration {}: {:?}", i, e))?; .map_err(|e| format!("Failed to advance slot on iteration {}: {:?}", i, e))?;
} }
println!("Slot processing: {}ms", t.elapsed().as_millis()); debug!("Slot processing: {:?}", t.elapsed());
let t = Instant::now();
pre_state
.update_tree_hash_cache()
.map_err(|e| format!("Unable to build tree hash cache: {:?}", e))?;
println!("Pre-block tree hash: {}ms", t.elapsed().as_millis());
let t = Instant::now(); let t = Instant::now();
pre_state pre_state
.build_all_caches(spec) .build_all_caches(spec)
.map_err(|e| format!("Unable to build caches: {:?}", e))?; .map_err(|e| format!("Unable to build caches: {:?}", e))?;
println!("Build all caches (again): {}ms", t.elapsed().as_millis()); debug!("Build all caches (again): {:?}", t.elapsed());
if !config.no_signature_verification {
let get_pubkey = move |validator_index| {
validator_pubkey_cache
.get(validator_index)
.map(Cow::Borrowed)
};
let decompressor = move |pk_bytes| {
// Map compressed pubkey to validator index.
let validator_index = validator_pubkey_cache.get_index(pk_bytes)?;
// Map validator index to pubkey (respecting guard on unknown validators).
get_pubkey(validator_index)
};
let t = Instant::now();
BlockSignatureVerifier::verify_entire_block(
&pre_state,
get_pubkey,
decompressor,
&block,
Some(block_root),
spec,
)
.map_err(|e| format!("Invalid block signature: {:?}", e))?;
debug!("Batch verify block signatures: {:?}", t.elapsed());
}
let t = Instant::now(); let t = Instant::now();
per_block_processing( per_block_processing(
&mut pre_state, &mut pre_state,
&block, &block,
None, None,
signature_strategy, BlockSignatureStrategy::NoVerification,
VerifyBlockRoot::True, VerifyBlockRoot::True,
spec, spec,
) )
.map_err(|e| format!("State transition failed: {:?}", e))?; .map_err(|e| format!("State transition failed: {:?}", e))?;
println!("Process block: {}ms", t.elapsed().as_millis()); debug!("Process block: {:?}", t.elapsed());
let t = Instant::now(); if !config.exclude_post_block_thc {
pre_state let t = Instant::now();
.update_tree_hash_cache() pre_state
.map_err(|e| format!("Unable to build tree hash cache: {:?}", e))?; .update_tree_hash_cache()
println!("Post-block tree hash: {}ms", t.elapsed().as_millis()); .map_err(|e| format!("Unable to build tree hash cache: {:?}", e))?;
debug!("Post-block tree hash: {:?}", t.elapsed());
}
Ok(pre_state) Ok(pre_state)
} }
@ -136,10 +398,6 @@ pub fn load_from_ssz_with<T>(
.map_err(|e| format!("Unable to read from file {:?}: {:?}", path, e))?; .map_err(|e| format!("Unable to read from file {:?}: {:?}", path, e))?;
let t = Instant::now(); let t = Instant::now();
let result = decoder(&bytes, spec).map_err(|e| format!("Ssz decode failed: {:?}", e)); let result = decoder(&bytes, spec).map_err(|e| format!("Ssz decode failed: {:?}", e));
println!( debug!("SSZ decoding {}: {:?}", path.display(), t.elapsed());
"SSZ decoding {}: {}ms",
path.display(),
t.elapsed().as_millis()
);
result result
} }