Reduce verbosity of reprocess queue logs (#4101)
## Issue Addressed NA ## Proposed Changes Replaces #4058 to attempt to reduce `ERRO Failed to send scheduled attestation` spam and provide more information for diagnosis. With this PR we achieve: - When dequeuing attestations after a block is received, send only one log which reports `n` failures (rather than `n` logs reporting `n` failures). - Make a distinction in logs between two separate attestation dequeuing events. - Add more information to both log events to help assist with troubleshooting. ## Additional Info NA
This commit is contained in:
parent
785a9171e6
commit
59e45fe349
@ -573,6 +573,9 @@ impl<T: BeaconChainTypes> ReprocessQueue<T> {
|
|||||||
}) => {
|
}) => {
|
||||||
// Unqueue the attestations we have for this root, if any.
|
// Unqueue the attestations we have for this root, if any.
|
||||||
if let Some(queued_ids) = self.awaiting_attestations_per_root.remove(&block_root) {
|
if let Some(queued_ids) = self.awaiting_attestations_per_root.remove(&block_root) {
|
||||||
|
let mut sent_count = 0;
|
||||||
|
let mut failed_to_send_count = 0;
|
||||||
|
|
||||||
for id in queued_ids {
|
for id in queued_ids {
|
||||||
metrics::inc_counter(
|
metrics::inc_counter(
|
||||||
&metrics::BEACON_PROCESSOR_REPROCESSING_QUEUE_MATCHED_ATTESTATIONS,
|
&metrics::BEACON_PROCESSOR_REPROCESSING_QUEUE_MATCHED_ATTESTATIONS,
|
||||||
@ -597,10 +600,9 @@ impl<T: BeaconChainTypes> ReprocessQueue<T> {
|
|||||||
|
|
||||||
// Send the work.
|
// Send the work.
|
||||||
if self.ready_work_tx.try_send(work).is_err() {
|
if self.ready_work_tx.try_send(work).is_err() {
|
||||||
error!(
|
failed_to_send_count += 1;
|
||||||
log,
|
} else {
|
||||||
"Failed to send scheduled attestation";
|
sent_count += 1;
|
||||||
);
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// There is a mismatch between the attestation ids registered for this
|
// There is a mismatch between the attestation ids registered for this
|
||||||
@ -613,6 +615,18 @@ impl<T: BeaconChainTypes> ReprocessQueue<T> {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if failed_to_send_count > 0 {
|
||||||
|
error!(
|
||||||
|
log,
|
||||||
|
"Ignored scheduled attestation(s) for block";
|
||||||
|
"hint" => "system may be overloaded",
|
||||||
|
"parent_root" => ?parent_root,
|
||||||
|
"block_root" => ?block_root,
|
||||||
|
"failed_count" => failed_to_send_count,
|
||||||
|
"sent_count" => sent_count,
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// Unqueue the light client optimistic updates we have for this root, if any.
|
// Unqueue the light client optimistic updates we have for this root, if any.
|
||||||
if let Some(queued_lc_id) = self
|
if let Some(queued_lc_id) = self
|
||||||
@ -727,7 +741,9 @@ impl<T: BeaconChainTypes> ReprocessQueue<T> {
|
|||||||
if self.ready_work_tx.try_send(work).is_err() {
|
if self.ready_work_tx.try_send(work).is_err() {
|
||||||
error!(
|
error!(
|
||||||
log,
|
log,
|
||||||
"Failed to send scheduled attestation";
|
"Ignored scheduled attestation";
|
||||||
|
"hint" => "system may be overloaded",
|
||||||
|
"beacon_block_root" => ?root
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user