Improve compile time (#1989)

## Issue Addressed

Closes #1264

## Proposed Changes

* Milagro BLS: tweak the feature flags so that Milagro doesn't get compiled if we're using BLST. Profiling showed that it was consuming about 1 minute of CPU time out of 60 minutes of CPU time (real time ~15 mins). A 1.6% saving.
* Reduce monomorphization: compiling for 3 different `EthSpec` types causes a heck of a lot of generic functions to be instantiated (monomorphized). Removing 2 of 3 cuts the LLVM+linking step from around 250 seconds to 180 seconds, a saving of 70 seconds (real time!). This applies only to `make` and not the CI build, because we test with the minimal spec on CI.
* Update `web3` crate to v0.13. This is perhaps the most controversial change, because it requires axing some deposit contract tools from `lcli`. I suspect these tools weren't used much anyway, and could be maintained separately, but I'm also happy to revert this change. However, it does save us a lot of compile time. With #1839, we now have 3 versions of Tokio (and all of Tokio's deps). This change brings us down to 2 versions, but 1 should be achievable once web3 (and reqwest) move to Tokio 0.3.
* Remove `lcli` from the Docker image. It's a dev tool and can be built from the repo if required.
This commit is contained in:
Michael Sproul 2020-12-09 01:34:58 +00:00
parent 4f85371ce8
commit 82753f842d
20 changed files with 286 additions and 1114 deletions

1048
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -4,7 +4,6 @@ COPY . lighthouse
ARG PORTABLE
ENV PORTABLE $PORTABLE
RUN cd lighthouse && make
RUN cd lighthouse && make install-lcli
FROM debian:buster-slim
RUN apt-get update && apt-get install -y --no-install-recommends \
@ -13,4 +12,3 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
COPY --from=builder /usr/local/cargo/bin/lighthouse /usr/local/bin/lighthouse
COPY --from=builder /usr/local/cargo/bin/lcli /usr/local/bin/lcli

View File

@ -20,7 +20,7 @@ eth2_ssz_derive = "0.1.0"
hex = "0.4.2"
rayon = "1.4.1"
eth2_network_config = { path = "../common/eth2_network_config" }
futures = { version = "0.3.7", features = ["compat"] }
futures = "0.3.7"
clap_utils = { path = "../common/clap_utils" }
directory = { path = "../common/directory" }
eth2_wallet = { path = "../crypto/eth2_wallet" }

View File

@ -7,14 +7,14 @@ edition = "2018"
[dev-dependencies]
eth1_test_rig = { path = "../../testing/eth1_test_rig" }
toml = "0.5.6"
web3 = "0.11.0"
web3 = "0.13.0"
sloggers = "1.0.1"
environment = { path = "../../lighthouse/environment" }
tokio-compat-02 = "0.1"
[dependencies]
reqwest = { version = "0.10.8", features = ["native-tls-vendored"] }
futures = { version = "0.3.7", features = ["compat"] }
futures = "0.3.7"
serde_json = "1.0.58"
serde = { version = "1.0.116", features = ["derive"] }
hex = "0.4.2"

View File

@ -4,7 +4,6 @@ use eth1::http::{get_deposit_count, get_deposit_logs_in_range, get_deposit_root,
use eth1::{Config, Service};
use eth1::{DepositCache, DEFAULT_CHAIN_ID, DEFAULT_NETWORK_ID};
use eth1_test_rig::GanacheEth1Instance;
use futures::compat::Future01CompatExt;
use merkle_proof::verify_merkle_proof;
use slog::Logger;
use sloggers::{null::NullLoggerBuilder, Build};

View File

@ -6,7 +6,6 @@
use environment::{Environment, EnvironmentBuilder};
use eth1::{DEFAULT_CHAIN_ID, DEFAULT_NETWORK_ID};
use eth1_test_rig::{DelayThenDeposit, GanacheEth1Instance};
use futures::compat::Future01CompatExt;
use genesis::{Eth1Config, Eth1GenesisService};
use state_processing::is_valid_genesis_state;
use std::time::Duration;

View File

@ -7,7 +7,7 @@ edition = "2018"
[dependencies]
eth2_ssz = "0.1.2"
tree_hash = "0.1.1"
milagro_bls = { git = "https://github.com/sigp/milagro_bls", tag = "v1.4.0" }
milagro_bls = { git = "https://github.com/sigp/milagro_bls", tag = "v1.4.0", optional = true }
rand = "0.7.3"
serde = "1.0.116"
serde_derive = "1.0.116"
@ -22,7 +22,7 @@ blst = "0.3.2"
[features]
default = ["supranational"]
fake_crypto = []
milagro = []
milagro = ["milagro_bls"]
supranational = []
supranational-portable = ["supranational", "blst/portable"]
supranational-force-adx = ["supranational", "blst/force-adx"]

View File

@ -1,3 +1,4 @@
pub mod blst;
pub mod fake_crypto;
#[cfg(feature = "milagro")]
pub mod milagro;

View File

@ -42,6 +42,7 @@ pub use get_withdrawal_credentials::get_withdrawal_credentials;
pub use zeroize_hash::ZeroizeHash;
use blst::BLST_ERROR as BlstError;
#[cfg(feature = "milagro")]
use milagro_bls::AmclError;
pub type Hash256 = ethereum_types::H256;
@ -49,6 +50,7 @@ pub type Hash256 = ethereum_types::H256;
#[derive(Clone, Debug, PartialEq)]
pub enum Error {
/// An error was raised from the Milagro BLS library.
#[cfg(feature = "milagro")]
MilagroError(AmclError),
/// An error was raised from the Supranational BLST BLS library.
BlstError(BlstError),
@ -62,6 +64,7 @@ pub enum Error {
InvalidZeroSecretKey,
}
#[cfg(feature = "milagro")]
impl From<AmclError> for Error {
fn from(e: AmclError) -> Error {
Error::MilagroError(e)
@ -122,6 +125,7 @@ macro_rules! define_mod {
};
}
#[cfg(feature = "milagro")]
define_mod!(milagro_implementations, crate::impls::milagro::types);
define_mod!(blst_implementations, crate::impls::blst::types);
#[cfg(feature = "fake_crypto")]

View File

@ -498,7 +498,7 @@ mod blst {
test_suite!(blst_implementations);
}
#[cfg(not(debug_assertions))]
#[cfg(all(feature = "milagro", not(debug_assertions)))]
mod milagro {
test_suite!(milagro_implementations);
}

View File

@ -20,9 +20,8 @@ types = { path = "../consensus/types" }
state_processing = { path = "../consensus/state_processing" }
eth2_ssz = "0.1.2"
regex = "1.3.9"
futures = { version = "0.3.7", features = ["compat"] }
futures = "0.3.7"
environment = { path = "../lighthouse/environment" }
web3 = "0.11.0"
eth2_network_config = { path = "../common/eth2_network_config" }
dirs = "3.0.1"
genesis = { path = "../beacon_node/genesis" }

View File

@ -1,71 +0,0 @@
use clap::ArgMatches;
use deposit_contract::{
testnet::{ABI, BYTECODE},
CONTRACT_DEPLOY_GAS,
};
use environment::Environment;
use futures::compat::Future01CompatExt;
use std::path::PathBuf;
use tokio_compat_02::FutureExt;
use types::EthSpec;
use web3::{
contract::{Contract, Options},
transports::Ipc,
types::{Address, U256},
Web3,
};
pub fn run<T: EthSpec>(env: Environment<T>, matches: &ArgMatches<'_>) -> Result<(), String> {
let eth1_ipc_path: PathBuf = clap_utils::parse_required(matches, "eth1-ipc")?;
let from_address: Address = clap_utils::parse_required(matches, "from-address")?;
let confirmations: usize = clap_utils::parse_required(matches, "confirmations")?;
let (_event_loop_handle, transport) =
Ipc::new(eth1_ipc_path).map_err(|e| format!("Unable to connect to eth1 IPC: {:?}", e))?;
let web3 = Web3::new(transport);
let bytecode = String::from_utf8(BYTECODE.to_vec()).map_err(|e| {
format!(
"Unable to parse deposit contract bytecode as utf-8: {:?}",
e
)
})?;
env.runtime().block_on(
async {
// It's unlikely that this will be the _actual_ deployment block, however it'll be close
// enough to serve our purposes.
//
// We only need the deposit block to put a lower bound on the block number we need to search
// for deposit logs.
let deploy_block = web3
.eth()
.block_number()
.compat()
.await
.map_err(|e| format!("Failed to get block number: {}", e))?;
let pending_contract = Contract::deploy(web3.eth(), &ABI)
.map_err(|e| format!("Unable to build contract deployer: {:?}", e))?
.confirmations(confirmations)
.options(Options {
gas: Some(U256::from(CONTRACT_DEPLOY_GAS)),
..Options::default()
})
.execute(bytecode, (), from_address)
.map_err(|e| format!("Unable to execute deployment: {:?}", e))?;
let address = pending_contract
.compat()
.await
.map_err(|e| format!("Unable to await pending contract: {:?}", e))?
.address();
println!("deposit_contract_address: {:?}", address);
println!("deposit_contract_deploy_block: {}", deploy_block);
Ok(())
}
.compat(),
)
}

View File

@ -2,14 +2,12 @@
extern crate log;
mod change_genesis_time;
mod check_deposit_data;
mod deploy_deposit_contract;
mod eth1_genesis;
mod generate_bootnode_enr;
mod insecure_validators;
mod interop_genesis;
mod new_testnet;
mod parse_hex;
mod refund_deposit_contract;
mod skip_slots;
mod transition_blocks;
@ -35,9 +33,7 @@ fn main() {
let matches = App::new("Lighthouse CLI Tool")
.version(lighthouse_version::VERSION)
.about(
"Performs various testing-related tasks, including defining testnets.",
)
.about("Performs various testing-related tasks, including defining testnets.")
.arg(
Arg::with_name("spec")
.short("s")
@ -46,7 +42,7 @@ fn main() {
.takes_value(true)
.required(true)
.possible_values(&["minimal", "mainnet"])
.default_value("mainnet")
.default_value("mainnet"),
)
.arg(
Arg::with_name("testnet-dir")
@ -87,7 +83,9 @@ fn main() {
)
.subcommand(
SubCommand::with_name("skip-slots")
.about("Performs a state transition from some state across some number of skip slots")
.about(
"Performs a state transition from some state across some number of skip slots",
)
.arg(
Arg::with_name("pre-state")
.value_name("BEACON_STATE")
@ -156,77 +154,9 @@ fn main() {
.help("SSZ encoded as 0x-prefixed hex"),
),
)
.subcommand(
SubCommand::with_name("deploy-deposit-contract")
.about(
"Deploy a testing eth1 deposit contract.",
)
.arg(
Arg::with_name("eth1-ipc")
.long("eth1-ipc")
.short("e")
.value_name("ETH1_IPC_PATH")
.help("Path to an Eth1 JSON-RPC IPC endpoint")
.takes_value(true)
.required(true)
)
.arg(
Arg::with_name("from-address")
.long("from-address")
.short("f")
.value_name("FROM_ETH1_ADDRESS")
.help("The address that will submit the contract creation. Must be unlocked.")
.takes_value(true)
.required(true)
)
.arg(
Arg::with_name("confirmations")
.value_name("INTEGER")
.long("confirmations")
.takes_value(true)
.default_value("3")
.help("The number of block confirmations before declaring the contract deployed."),
)
)
.subcommand(
SubCommand::with_name("refund-deposit-contract")
.about(
"Calls the steal() function on a testnet eth1 contract.",
)
.arg(
Arg::with_name("eth1-ipc")
.long("eth1-ipc")
.short("e")
.value_name("ETH1_IPC_PATH")
.help("Path to an Eth1 JSON-RPC IPC endpoint")
.takes_value(true)
.required(true)
)
.arg(
Arg::with_name("from-address")
.long("from-address")
.short("f")
.value_name("FROM_ETH1_ADDRESS")
.help("The address that will submit the contract creation. Must be unlocked.")
.takes_value(true)
.required(true)
)
.arg(
Arg::with_name("contract-address")
.long("contract-address")
.short("c")
.value_name("CONTRACT_ETH1_ADDRESS")
.help("The address of the contract to be refunded. Its owner must match
--from-address.")
.takes_value(true)
.required(true)
)
)
.subcommand(
SubCommand::with_name("eth1-genesis")
.about(
"Listens to the eth1 chain and finds the genesis beacon state",
)
.about("Listens to the eth1 chain and finds the genesis beacon state")
.arg(
Arg::with_name("eth1-endpoint")
.short("e")
@ -241,16 +171,16 @@ fn main() {
.value_name("HTTP_SERVER_LIST")
.takes_value(true)
.conflicts_with("eth1-endpoint")
.help("One or more comma-delimited URLs to eth1 JSON-RPC http APIs. \
.help(
"One or more comma-delimited URLs to eth1 JSON-RPC http APIs. \
If multiple endpoints are given the endpoints are used as \
fallback in the given order."),
)
fallback in the given order.",
),
),
)
.subcommand(
SubCommand::with_name("interop-genesis")
.about(
"Produces an interop-compatible genesis state using deterministic keypairs",
)
.about("Produces an interop-compatible genesis state using deterministic keypairs")
.arg(
Arg::with_name("validator-count")
.long("validator-count")
@ -273,9 +203,11 @@ fn main() {
.long("genesis-fork-version")
.value_name("HEX")
.takes_value(true)
.help("Used to avoid reply attacks between testnets. Recommended to set to
non-default."),
)
.help(
"Used to avoid reply attacks between testnets. Recommended to set to
non-default.",
),
),
)
.subcommand(
SubCommand::with_name("change-genesis-time")
@ -297,7 +229,7 @@ fn main() {
.takes_value(true)
.required(true)
.help("The value for state.genesis_time."),
)
),
)
.subcommand(
SubCommand::with_name("new-testnet")
@ -317,8 +249,10 @@ fn main() {
.long("min-genesis-time")
.value_name("UNIX_SECONDS")
.takes_value(true)
.help("The minimum permitted genesis time. For non-eth1 testnets will be
the genesis time. Defaults to now."),
.help(
"The minimum permitted genesis time. For non-eth1 testnets will be
the genesis time. Defaults to now.",
),
)
.arg(
Arg::with_name("min-genesis-active-validator-count")
@ -374,8 +308,10 @@ fn main() {
.long("genesis-fork-version")
.value_name("HEX")
.takes_value(true)
.help("Used to avoid reply attacks between testnets. Recommended to set to
non-default."),
.help(
"Used to avoid reply attacks between testnets. Recommended to set to
non-default.",
),
)
.arg(
Arg::with_name("deposit-contract-address")
@ -391,15 +327,15 @@ fn main() {
.value_name("ETH1_BLOCK_NUMBER")
.takes_value(true)
.default_value("0")
.help("The block the deposit contract was deployed. Setting this is a huge
optimization for nodes, please do it."),
)
.help(
"The block the deposit contract was deployed. Setting this is a huge
optimization for nodes, please do it.",
),
),
)
.subcommand(
SubCommand::with_name("check-deposit-data")
.about(
"Checks the integrity of some deposit data.",
)
.about("Checks the integrity of some deposit data.")
.arg(
Arg::with_name("deposit-amount")
.index(1)
@ -414,15 +350,15 @@ fn main() {
.value_name("HEX")
.takes_value(true)
.required(true)
.help("A 0x-prefixed hex string of the deposit data. Should include the
function signature."),
)
.help(
"A 0x-prefixed hex string of the deposit data. Should include the
function signature.",
),
),
)
.subcommand(
SubCommand::with_name("generate-bootnode-enr")
.about(
"Generates an ENR address to be used as a pre-genesis boot node.",
)
.about("Generates an ENR address to be used as a pre-genesis boot node.")
.arg(
Arg::with_name("ip")
.long("ip")
@ -445,7 +381,9 @@ fn main() {
.value_name("TCP_PORT")
.takes_value(true)
.required(true)
.help("The TCP port to be included in the ENR and used for application comms"),
.help(
"The TCP port to be included in the ENR and used for application comms",
),
)
.arg(
Arg::with_name("output-dir")
@ -461,15 +399,15 @@ fn main() {
.value_name("HEX")
.takes_value(true)
.required(true)
.help("Used to avoid reply attacks between testnets. Recommended to set to
non-default."),
)
.help(
"Used to avoid reply attacks between testnets. Recommended to set to
non-default.",
),
),
)
.subcommand(
SubCommand::with_name("insecure-validators")
.about(
"Produces validator directories with INSECURE, deterministic keypairs.",
)
.about("Produces validator directories with INSECURE, deterministic keypairs.")
.arg(
Arg::with_name("count")
.long("count")
@ -490,7 +428,7 @@ fn main() {
.value_name("SECRETS_DIR")
.takes_value(true)
.help("The directory for storing secrets."),
)
),
)
.get_matches();
@ -572,14 +510,6 @@ fn run<T: EthSpec>(
("pretty-hex", Some(matches)) => {
run_parse_hex::<T>(matches).map_err(|e| format!("Failed to pretty print hex: {}", e))
}
("deploy-deposit-contract", Some(matches)) => {
deploy_deposit_contract::run::<T>(env, matches)
.map_err(|e| format!("Failed to run deploy-deposit-contract command: {}", e))
}
("refund-deposit-contract", Some(matches)) => {
refund_deposit_contract::run::<T>(env, matches)
.map_err(|e| format!("Failed to run refund-deposit-contract command: {}", e))
}
("eth1-genesis", Some(matches)) => eth1_genesis::run::<T>(env, matches)
.map_err(|e| format!("Failed to run eth1-genesis command: {}", e)),
("interop-genesis", Some(matches)) => interop_genesis::run::<T>(env, matches)

View File

@ -1,47 +0,0 @@
use clap::ArgMatches;
use environment::Environment;
use futures::compat::Future01CompatExt;
use std::path::PathBuf;
use tokio_compat_02::FutureExt;
use types::EthSpec;
use web3::{
transports::Ipc,
types::{Address, TransactionRequest, U256},
Web3,
};
/// `keccak("steal()")[0..4]`
pub const STEAL_FN_SIGNATURE: &[u8] = &[0xcf, 0x7a, 0x89, 0x65];
pub fn run<T: EthSpec>(env: Environment<T>, matches: &ArgMatches<'_>) -> Result<(), String> {
let eth1_ipc_path: PathBuf = clap_utils::parse_required(matches, "eth1-ipc")?;
let from: Address = clap_utils::parse_required(matches, "from-address")?;
let contract_address: Address = clap_utils::parse_required(matches, "contract-address")?;
let (_event_loop_handle, transport) =
Ipc::new(eth1_ipc_path).map_err(|e| format!("Unable to connect to eth1 IPC: {:?}", e))?;
let web3 = Web3::new(transport);
env.runtime().block_on(
async {
let _ = web3
.eth()
.send_transaction(TransactionRequest {
from,
to: Some(contract_address),
gas: Some(U256::from(400_000)),
gas_price: None,
value: Some(U256::zero()),
data: Some(STEAL_FN_SIGNATURE.into()),
nonce: None,
condition: None,
})
.compat()
.await
.map_err(|e| format!("Failed to call steal fn: {:?}", e))?;
Ok(())
}
.compat(),
)
}

View File

@ -13,6 +13,10 @@ portable = ["bls/supranational-portable"]
modern = ["bls/supranational-force-adx"]
# Uses the slower Milagro BLS library, which is written in native Rust.
milagro = ["bls/milagro"]
# Support minimal spec (used for testing only).
spec-minimal = []
# Support spec v0.12 (used by Medalla testnet).
spec-v12 = []
[dependencies]
beacon_node = { "path" = "../beacon_node" }

View File

@ -38,8 +38,10 @@ fn main() {
.long_version(
format!(
"{}\n\
BLS Library: {}",
VERSION.replace("Lighthouse/", ""), bls_library_name()
BLS Library: {}\n\
Specs: mainnet (true), minimal ({}), v0.12.3 ({})",
VERSION.replace("Lighthouse/", ""), bls_library_name(),
cfg!(feature = "spec-minimal"), cfg!(feature = "spec-v12"),
).as_str()
)
.arg(
@ -151,11 +153,22 @@ fn main() {
}
match eth_spec_id {
EthSpecId::Minimal => run(EnvironmentBuilder::minimal(), &matches, testnet_config),
EthSpecId::Mainnet => run(EnvironmentBuilder::mainnet(), &matches, testnet_config),
#[cfg(feature = "spec-minimal")]
EthSpecId::Minimal => run(EnvironmentBuilder::minimal(), &matches, testnet_config),
#[cfg(feature = "spec-v12")]
EthSpecId::V012Legacy => {
run(EnvironmentBuilder::v012_legacy(), &matches, testnet_config)
}
#[cfg(any(not(feature = "spec-minimal"), not(feature = "spec-v12")))]
other => {
eprintln!(
"Eth spec `{}` is not supported by this build of Lighthouse",
other
);
eprintln!("You must compile with a feature flag to enable this spec variant");
exit(1);
}
}
});

View File

@ -6,8 +6,9 @@ edition = "2018"
[dependencies]
tokio = { version = "0.3.2", features = ["time"] }
web3 = "0.11.0"
futures = { version = "0.3.7", features = ["compat"] }
tokio-compat-02 = "0.1"
web3 = "0.13.0"
futures = "0.3.7"
types = { path = "../../consensus/types"}
serde_json = "1.0.58"
deposit_contract = { path = "../../common/deposit_contract"}

View File

@ -1,15 +1,11 @@
use futures::compat::Future01CompatExt;
use serde_json::json;
use std::io::prelude::*;
use std::io::BufReader;
use std::net::TcpListener;
use std::process::{Child, Command, Stdio};
use std::sync::Arc;
use std::time::{Duration, Instant};
use web3::{
transports::{EventLoopHandle, Http},
Transport, Web3,
};
use tokio_compat_02::FutureExt;
use web3::{transports::Http, Transport, Web3};
/// How long we will wait for ganache to indicate that it is ready.
const GANACHE_STARTUP_TIMEOUT_MILLIS: u64 = 10_000;
@ -20,7 +16,6 @@ const GANACHE_STARTUP_TIMEOUT_MILLIS: u64 = 10_000;
pub struct GanacheInstance {
pub port: u16,
child: Child,
_event_loop: Arc<EventLoopHandle>,
pub web3: Web3<Http>,
network_id: u64,
chain_id: u64,
@ -56,7 +51,7 @@ impl GanacheInstance {
}
}?;
let (event_loop, transport) = Http::new(&endpoint(port)).map_err(|e| {
let transport = Http::new(&endpoint(port)).map_err(|e| {
format!(
"Failed to start HTTP transport connected to ganache: {:?}",
e
@ -69,7 +64,6 @@ impl GanacheInstance {
Ok(Self {
child,
port,
_event_loop: Arc::new(event_loop),
web3,
network_id,
chain_id,

View File

@ -10,10 +10,10 @@ mod ganache;
use deposit_contract::{
encode_eth1_tx_data, testnet, ABI, BYTECODE, CONTRACT_DEPLOY_GAS, DEPOSIT_GAS,
};
use futures::compat::Future01CompatExt;
use ganache::GanacheInstance;
use std::time::Duration;
use tokio::time::sleep;
use tokio_compat_02::FutureExt;
use types::DepositData;
use types::{test_utils::generate_deterministic_keypair, EthSpec, Hash256, Keypair, Signature};
use web3::contract::{Contract, Options};

View File

@ -32,7 +32,7 @@ slog = { version = "2.5.2", features = ["max_level_trace", "release_max_level_tr
slog-async = "2.5.0"
slog-term = "2.6.0"
tokio = { version = "0.3.2", features = ["time"] }
futures = { version = "0.3.7", features = ["compat"] }
futures = "0.3.7"
dirs = "3.0.1"
directory = { path = "../common/directory" }
lockfile = { path = "../common/lockfile" }