From 10a134792bd90497729411f30eabddd022181123 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 9 Dec 2019 23:14:13 +1100 Subject: [PATCH] Testnet2 (#685) * Apply clippy lints to beacon node * Remove unnecessary logging and correct formatting * Initial bones of load-balanced range-sync * Port bump meshsup tests * Further structure and network handling logic added * Basic structure, ignoring error handling * Correct max peers delay bug * Clean up and re-write message processor and sync manager * Restructure directory, correct type issues * Fix compiler issues * Completed first testing of new sync * Correct merge issues * Clean up warnings * Push attestation processed log down to dbg * Add state enc/dec benches * Correct math error, downgraded logs * Add example for flamegraph * Use `PublicKeyBytes` for `Validator` * Ripple PublicKeyBytes change through codebase * Add RPC error handling and improved syncing code * Add benches, optimizations to store BeaconState * Store BeaconState in StorageContainer too * Optimize StorageContainer with std::mem magic * Add libp2p stream error handling and dropping of invalid peers * Lower logs * Update lcli to parse spec at boot, remove pycli * Fix issues when starting with mainnet spec * Set default spec to mainnet * Fix lcli --spec param * Add discovery tweak * Ensure ETH1_FOLLOW_DISTANCE is in YamlConfig * Set testnet ETH1_FOLLOW_DISTANCE to 16 * Fix rest_api tests * Set testnet min validator count * Update with new testnet dir * Remove some dbg, println * Add timeout when notifier waits for libp2p lock * Add validator count CLI flag to lcli contract deploy * Extend genesis delay time * Correct libp2p service locking * Update testnet dir * Add basic block/state caching on beacon chain * Add decimals display to notifier sync speed * Try merge in change to reduce fork choice calls * Remove fork choice from process block * Minor log fix * Check successes > 0 * Adds checkpoint cache * Stop storing the tree hash cache in the db * Handles peer disconnects for sync * Fix failing beacon chain tests * Change eth2_testnet_config tests to Mainnet * Add logs downgrade discovery log * Remove dedunant beacon state write * Fix re-org warnings * Use caching get methods in fork choice * Fix mistake in prev commit * Use caching state getting in state_by_slot * Add state.cacheless_clone * Less fork choice (#679) * Try merge in change to reduce fork choice calls * Remove fork choice from process block * Minor log fix * Check successes > 0 * Fix failing beacon chain tests * Fix re-org warnings * Fix mistake in prev commit * Attempt to improve attestation processing times * Introduce HeadInfo struct * Used cache tree hash for block processing * Use cached tree hash for block production too * Range sync refactor - Introduces `ChainCollection` - Correct Disconnect node handling - Removes duplicate code * Add more logging for DB * Various bug fixes * Remove unnecessary logs * Maintain syncing state in the transition from finalied to head * Improved disconnect handling * Add `Speedo` struct * Fix bugs in speedo * Fix bug in speedo * Fix rounding bug in speedo * Move code around, reduce speedo observation count * Adds forwards block interator * Fix inf NaN * Add first draft of validator onboarding * Update docs * Add documentation link to main README * Continue docs development * Update book readme * Update docs * Allow vc to run without testnet subcommand * Small change to onboarding docs * Tidy CLI help messages * Update docs * Add check to val client see if beacon node is synced * Attempt to fix NaN bug * Fix compile bug * Add notifier service to validator client * Re-order onboarding steps * Update deposit contract address * Update testnet dir * Add note about public eth1 node * Fix installation link * Set default eth1 endpoint to sigp * Fix broken test * Try fix eth1 cache locking * Be more specific about eth1 endpoint * Increase gas limit for deposit * Fix default deposit amount * Fix re-org log --- beacon_node/beacon_chain/src/beacon_chain.rs | 9 +++++++- beacon_node/store/src/leveldb_store.rs | 23 ++++++++++++-------- beacon_node/store/src/metrics.rs | 8 +++++++ book/src/SUMMARY.md | 2 +- lcli/src/main.rs | 19 ++++++++-------- 5 files changed, 40 insertions(+), 21 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index c8ad15e2e..20dd3f049 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -1504,7 +1504,14 @@ impl BeaconChain { let previous_slot = self.head_info().slot; let new_slot = beacon_block.slot; - let is_reorg = self.head_info().block_root != beacon_block.parent_root; + // Note: this will declare a re-org if we skip `SLOTS_PER_HISTORICAL_ROOT` blocks + // between calls to fork choice without swapping between chains. This seems like an + // extreme-enough scenario that a warning is fine. + let is_reorg = self.head_info().block_root + != beacon_state + .get_block_root(self.head_info().slot) + .map(|root| *root) + .unwrap_or_else(|_| Hash256::random()); // If we switched to a new chain (instead of building atop the present chain). if is_reorg { diff --git a/beacon_node/store/src/leveldb_store.rs b/beacon_node/store/src/leveldb_store.rs index b9ebd25d1..051d09170 100644 --- a/beacon_node/store/src/leveldb_store.rs +++ b/beacon_node/store/src/leveldb_store.rs @@ -69,17 +69,18 @@ impl Store for LevelDB { let column_key = Self::get_key_for_col(col, key); metrics::inc_counter(&metrics::DISK_DB_READ_COUNT); + let timer = metrics::start_timer(&metrics::DISK_DB_READ_TIMES); - let result = self - .db + self.db .get(self.read_options(), column_key) - .map_err(Into::into); - - if let Ok(Some(bytes)) = &result { - metrics::inc_counter_by(&metrics::DISK_DB_READ_BYTES, bytes.len() as i64) - } - - result + .map_err(Into::into) + .map(|opt| { + opt.map(|bytes| { + metrics::inc_counter_by(&metrics::DISK_DB_READ_BYTES, bytes.len() as i64); + metrics::stop_timer(timer); + bytes + }) + }) } /// Store some `value` in `column`, indexed with `key`. @@ -88,10 +89,14 @@ impl Store for LevelDB { metrics::inc_counter(&metrics::DISK_DB_WRITE_COUNT); metrics::inc_counter_by(&metrics::DISK_DB_WRITE_BYTES, val.len() as i64); + let timer = metrics::start_timer(&metrics::DISK_DB_WRITE_TIMES); self.db .put(self.write_options(), column_key, val) .map_err(Into::into) + .map(|()| { + metrics::stop_timer(timer); + }) } /// Return `true` if `key` exists in `column`. diff --git a/beacon_node/store/src/metrics.rs b/beacon_node/store/src/metrics.rs index 6f51c71ba..022d81446 100644 --- a/beacon_node/store/src/metrics.rs +++ b/beacon_node/store/src/metrics.rs @@ -25,6 +25,14 @@ lazy_static! { "store_disk_db_write_count_total", "Total number of writes to the on-disk DB" ); + pub static ref DISK_DB_READ_TIMES: Result = try_create_histogram( + "store_disk_db_read_seconds", + "Time taken to write bytes to store." + ); + pub static ref DISK_DB_WRITE_TIMES: Result = try_create_histogram( + "store_disk_db_write_seconds", + "Time taken to write bytes to store." + ); pub static ref DISK_DB_EXISTS_COUNT: Result = try_create_int_counter( "store_disk_db_exists_count_total", "Total number of checks if a key is in the on-disk DB" diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index 8ca5d49f6..d547594da 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -2,7 +2,7 @@ * [Introduction](./intro.md) * [Become a Validator](./become-a-validator.md) -* [Introduction](./intro.md) +* [Installation](./installation.md) * [Docker](./docker.md) * [CLI](./cli.md) * [Testnets](./testnets.md) diff --git a/lcli/src/main.rs b/lcli/src/main.rs index 76811192a..93100c01d 100644 --- a/lcli/src/main.rs +++ b/lcli/src/main.rs @@ -25,6 +25,15 @@ fn main() { "Performs various testing-related tasks, modelled after zcli. \ by @protolambda.", ) + .arg( + Arg::with_name("spec") + .short("s") + .value_name("STRING") + .takes_value(true) + .required(true) + .possible_values(&["minimal", "mainnet"]) + .default_value("mainnet") + ) .subcommand( SubCommand::with_name("genesis_yaml") .about("Generates a genesis YAML file") @@ -44,16 +53,6 @@ fn main() { .required(false) .help("Eth2 genesis time (seconds since UNIX epoch)."), ) - .arg( - Arg::with_name("spec") - .short("s") - .value_name("STRING") - .takes_value(true) - .required(true) - .possible_values(&["minimal", "mainnet"]) - .default_value("mainnet") - .help("Eth2 genesis time (seconds since UNIX epoch)."), - ) .arg( Arg::with_name("output_file") .short("f")