Reduce some EE and builder related ERRO logs to WARN (#3966)
## Issue Addressed NA ## Proposed Changes Our `ERRO` stream has been rather noisy since the merge due to some unexpected behaviours of builders and EEs. Now that we've been running post-merge for a while, I think we can drop some of these `ERRO` to `WARN` so we're not "crying wolf". The modified logs are: #### `ERRO Execution engine call failed` I'm seeing this quite frequently on Geth nodes. They seem to timeout when they're busy and it rarely indicates a serious issue. We also have logging across block import, fork choice updating and payload production that raise `ERRO` or `CRIT` when the EE times out, so I think we're not at risk of silencing actual issues. #### `ERRO "Builder failed to reveal payload"` In #3775 we reduced this log from `CRIT` to `ERRO` since it's common for builders to fail to reveal the block to the producer directly whilst still broadcasting it to the networ. I think it's worth dropping this to `WARN` since it's rarely interesting. I elected to stay with `WARN` since I really do wish builders would fulfill their API promises by returning the block to us. Perhaps I'm just being pedantic here, I could be convinced otherwise. #### `ERRO "Relay error when registering validator(s)"` It seems like builders and/or mev-boost struggle to handle heavy loads of validator registrations. I haven't observed issues with validators not actually being registered, but I see timeouts on these endpoints many times a day. It doesn't seem like this `ERRO` is worth it. #### `ERRO Error fetching block for peer ExecutionLayerErrorPayloadReconstruction` This means we failed to respond to a peer on the P2P network with a block they requested because of an error in the `execution_layer`. It's very common to see timeouts or incomplete responses on this endpoint whilst the EE is busy and I don't think it's important enough for an `ERRO`. As long as the peer count stays high, I don't think the user needs to be actively concerned about how we're responding to peers. ## Additional Info NA
This commit is contained in:
parent
3b4c677727
commit
84843d67d7
1
Cargo.lock
generated
1
Cargo.lock
generated
@ -4988,6 +4988,7 @@ dependencies = [
|
||||
"eth2_ssz",
|
||||
"eth2_ssz_types",
|
||||
"ethereum-types 0.14.1",
|
||||
"execution_layer",
|
||||
"exit-future",
|
||||
"fnv",
|
||||
"futures",
|
||||
|
@ -5,7 +5,7 @@ use crate::engine_api::{
|
||||
};
|
||||
use crate::HttpJsonRpc;
|
||||
use lru::LruCache;
|
||||
use slog::{debug, error, info, Logger};
|
||||
use slog::{debug, error, info, warn, Logger};
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use task_executor::TaskExecutor;
|
||||
@ -325,7 +325,7 @@ impl Engine {
|
||||
Ok(result)
|
||||
}
|
||||
Err(error) => {
|
||||
error!(
|
||||
warn!(
|
||||
self.log,
|
||||
"Execution engine call failed";
|
||||
"error" => ?error,
|
||||
|
@ -1575,10 +1575,10 @@ impl<T: EthSpec> ExecutionLayer<T> {
|
||||
&metrics::EXECUTION_LAYER_BUILDER_REVEAL_PAYLOAD_OUTCOME,
|
||||
&[metrics::FAILURE],
|
||||
);
|
||||
error!(
|
||||
warn!(
|
||||
self.log(),
|
||||
"Builder failed to reveal payload";
|
||||
"info" => "this relay failure may cause a missed proposal",
|
||||
"info" => "this is common behaviour for some builders and may not indicate an issue",
|
||||
"error" => ?e,
|
||||
"relay_response_ms" => duration.as_millis(),
|
||||
"block_root" => ?block_root,
|
||||
|
@ -2855,7 +2855,7 @@ pub fn serve<T: BeaconChainTypes>(
|
||||
.await
|
||||
.map(|resp| warp::reply::json(&resp))
|
||||
.map_err(|e| {
|
||||
error!(
|
||||
warn!(
|
||||
log,
|
||||
"Relay error when registering validator(s)";
|
||||
"num_registrations" => filtered_registration_data.len(),
|
||||
|
@ -45,6 +45,7 @@ tokio-util = { version = "0.6.3", features = ["time"] }
|
||||
derivative = "2.2.0"
|
||||
delay_map = "0.1.1"
|
||||
ethereum-types = { version = "0.14.1", optional = true }
|
||||
execution_layer = { path = "../execution_layer" }
|
||||
|
||||
[features]
|
||||
deterministic_long_lived_attnets = [ "ethereum-types" ]
|
||||
|
@ -7,7 +7,7 @@ use itertools::process_results;
|
||||
use lighthouse_network::rpc::StatusMessage;
|
||||
use lighthouse_network::rpc::*;
|
||||
use lighthouse_network::{PeerId, PeerRequestId, ReportSource, Response, SyncInfo};
|
||||
use slog::{debug, error};
|
||||
use slog::{debug, error, warn};
|
||||
use slot_clock::SlotClock;
|
||||
use std::sync::Arc;
|
||||
use task_executor::TaskExecutor;
|
||||
@ -392,12 +392,26 @@ impl<T: BeaconChainTypes> Worker<T> {
|
||||
break;
|
||||
}
|
||||
Err(e) => {
|
||||
error!(
|
||||
self.log,
|
||||
"Error fetching block for peer";
|
||||
"block_root" => ?root,
|
||||
"error" => ?e
|
||||
);
|
||||
if matches!(
|
||||
e,
|
||||
BeaconChainError::ExecutionLayerErrorPayloadReconstruction(_block_hash, ref boxed_error)
|
||||
if matches!(**boxed_error, execution_layer::Error::EngineError(_))
|
||||
) {
|
||||
warn!(
|
||||
self.log,
|
||||
"Error rebuilding payload for peer";
|
||||
"info" => "this may occur occasionally when the EE is busy",
|
||||
"block_root" => ?root,
|
||||
"error" => ?e,
|
||||
);
|
||||
} else {
|
||||
error!(
|
||||
self.log,
|
||||
"Error fetching block for peer";
|
||||
"block_root" => ?root,
|
||||
"error" => ?e
|
||||
);
|
||||
}
|
||||
|
||||
// send the stream terminator
|
||||
self.send_error_response(
|
||||
|
Loading…
Reference in New Issue
Block a user