Reduce verbosity of reprocess queue logs (#4101)

## Issue Addressed

NA

## Proposed Changes

Replaces #4058 to attempt to reduce `ERRO Failed to send scheduled attestation` spam and provide more information for diagnosis. With this PR we achieve:

- When dequeuing attestations after a block is received, send only one log which reports `n` failures (rather than `n` logs reporting `n` failures).
- Make a distinction in logs between two separate attestation dequeuing events.
- Add more information to both log events to help assist with troubleshooting.

## Additional Info

NA
This commit is contained in:
Paul Hauner 2023-03-21 05:15:00 +00:00
parent 785a9171e6
commit 59e45fe349

View File

@ -573,6 +573,9 @@ impl<T: BeaconChainTypes> ReprocessQueue<T> {
}) => { }) => {
// Unqueue the attestations we have for this root, if any. // Unqueue the attestations we have for this root, if any.
if let Some(queued_ids) = self.awaiting_attestations_per_root.remove(&block_root) { if let Some(queued_ids) = self.awaiting_attestations_per_root.remove(&block_root) {
let mut sent_count = 0;
let mut failed_to_send_count = 0;
for id in queued_ids { for id in queued_ids {
metrics::inc_counter( metrics::inc_counter(
&metrics::BEACON_PROCESSOR_REPROCESSING_QUEUE_MATCHED_ATTESTATIONS, &metrics::BEACON_PROCESSOR_REPROCESSING_QUEUE_MATCHED_ATTESTATIONS,
@ -597,10 +600,9 @@ impl<T: BeaconChainTypes> ReprocessQueue<T> {
// Send the work. // Send the work.
if self.ready_work_tx.try_send(work).is_err() { if self.ready_work_tx.try_send(work).is_err() {
error!( failed_to_send_count += 1;
log, } else {
"Failed to send scheduled attestation"; sent_count += 1;
);
} }
} else { } else {
// There is a mismatch between the attestation ids registered for this // There is a mismatch between the attestation ids registered for this
@ -613,6 +615,18 @@ impl<T: BeaconChainTypes> ReprocessQueue<T> {
); );
} }
} }
if failed_to_send_count > 0 {
error!(
log,
"Ignored scheduled attestation(s) for block";
"hint" => "system may be overloaded",
"parent_root" => ?parent_root,
"block_root" => ?block_root,
"failed_count" => failed_to_send_count,
"sent_count" => sent_count,
);
}
} }
// Unqueue the light client optimistic updates we have for this root, if any. // Unqueue the light client optimistic updates we have for this root, if any.
if let Some(queued_lc_id) = self if let Some(queued_lc_id) = self
@ -727,7 +741,9 @@ impl<T: BeaconChainTypes> ReprocessQueue<T> {
if self.ready_work_tx.try_send(work).is_err() { if self.ready_work_tx.try_send(work).is_err() {
error!( error!(
log, log,
"Failed to send scheduled attestation"; "Ignored scheduled attestation";
"hint" => "system may be overloaded",
"beacon_block_root" => ?root
); );
} }