cerc-io/stack-orchestrator
17
1
Fork 3
Code Issues 113 Pull Requests 18 Actions 1 Packages Projects Releases 480 Wiki Activity

[WIP] Add alerts on blackbox metrics for monitoring endpoints #803

Draft
prathamesh wants to merge 4 commits from deep-stack/stack-orchestrator:pm-endpoint-alerts into main
pull from: deep-stack/stack-orchestrator:pm-endpoint-alerts
merge into: cerc-io:main
Conversation 0 Commits 4 Files Changed 5 +20 -52
1 changed files with 20 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit ee0436588b - Show all commits

72
stack_orchestrator/data/config/monitoring/blackbox-alert-rules.yml
View File

@ -10,20 +10,6 @@ groups:
title: azimuth_gateway_endpoint_tracking
condition: condition
data:
- refId: probe
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
editorMode: code
expr: probe_success{destination="azimuth_gateway"}
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: probe
- refId: http_status_code
relativeTimeRange:
from: 600
@ -47,53 +33,37 @@ groups:
conditions:
- evaluator:
params:
- 0
- 0
type: gt
- 200
- 200
type: outside_range
operator:
type: and
query:
params: []
params:
- http_status_code
reducer:
params: []
type: avg
type: last
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ${probe} != 1 || ${http_status_code} != 200
intervalMs: 1000
maxDataPoints: 43200
expression: ""
hide: false
refId: condition
type: math
type: classic_conditions
noDataState: Alerting
execErrState: Alerting
for: 5m
annotations:
summary: Probe failed for Azimuth gateway endpoint
labels:
probe_success: '{{ index $values "probe" }}'
summary: Probe failed for Azimuth gateway endpoint, http status {{ index $values "http_status_code" }}
isPaused: false
# Laconicd GQL endpoint
- uid: laconicd_gql
title: laconicd_gql_endpoint_tracking
condition: condition
data:
- refId: probe
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
editorMode: code
expr: probe_success{destination="laconicd_gql"}
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: probe
- refId: http_status_code
relativeTimeRange:
from: 600
@ -117,31 +87,29 @@ groups:
conditions:
- evaluator:
params:
- 0
- 0
type: gt
- 200
- 200
type: outside_range
operator:
type: and
query:
params: []
params:
- http_status_code
reducer:
params: []
type: avg
type: last
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ${probe} != 1 || ${http_status_code} != 200
intervalMs: 1000
maxDataPoints: 43200
expression: ""
hide: false
refId: condition
type: math
type: classic_conditions
noDataState: Alerting
execErrState: Alerting
for: 5m
annotations:
summary: Probe failed for Laconicd GQL endpoint
labels:
probe_success: '{{ index $values "probe" }}'
summary: Probe failed for Laconicd GQL endpoint, http status {{ index $values "http_status_code" }}
isPaused: false