From abf4d39a226ad290cf221a6a9f4616ad8ca65a96 Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Mon, 15 Apr 2024 18:42:26 +0530
Subject: [PATCH] Add http status code check to alert rule

---
 .../monitoring/blackbox-alert-rules.yml       | 38 ++++++++++++++++---
 1 file changed, 32 insertions(+), 6 deletions(-)

diff --git a/stack_orchestrator/data/config/monitoring/blackbox-alert-rules.yml b/stack_orchestrator/data/config/monitoring/blackbox-alert-rules.yml
index aaf7c8c0..39f55990 100644
--- a/stack_orchestrator/data/config/monitoring/blackbox-alert-rules.yml
+++ b/stack_orchestrator/data/config/monitoring/blackbox-alert-rules.yml
@@ -24,6 +24,20 @@ groups:
                 maxDataPoints: 43200
                 range: false
                 refId: probe
+            - refId: http_status_code
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: PBFA97CFB590B2093
+              model:
+                editorMode: code
+                expr: probe_http_status_code{destination="azimuth_gateway"}
+                instant: true
+                intervalMs: 1000
+                legendFormat: __auto
+                maxDataPoints: 43200
+                range: false
+                refId: http_status_code
             - refId: condition
               relativeTimeRange:
                 from: 600
@@ -48,7 +62,7 @@ groups:
                     name: Expression
                     type: __expr__
                     uid: __expr__
-                expression: ${probe} != 1
+                expression: ${probe} != 1 ||  ${http_status_code} != 200
                 intervalMs: 1000
                 maxDataPoints: 43200
                 refId: condition
@@ -57,9 +71,8 @@ groups:
           execErrState: Alerting
           for: 5m
           annotations:
-            summary: Probe failed for Azimuth gateway endpoint {{ index $labels "instance" }}
+            summary: Probe failed for Azimuth gateway endpoint
           labels:
-            endpoint: '{{ index $labels "instance" }}'
             probe_success: '{{ index $values "probe" }}'
           isPaused: false
         # Laconicd GQL endpoint
@@ -81,6 +94,20 @@ groups:
                 maxDataPoints: 43200
                 range: false
                 refId: probe
+            - refId: http_status_code
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: PBFA97CFB590B2093
+              model:
+                editorMode: code
+                expr: probe_http_status_code{destination="laconicd_gql"}
+                instant: true
+                intervalMs: 1000
+                legendFormat: __auto
+                maxDataPoints: 43200
+                range: false
+                refId: http_status_code
             - refId: condition
               relativeTimeRange:
                 from: 600
@@ -105,7 +132,7 @@ groups:
                     name: Expression
                     type: __expr__
                     uid: __expr__
-                expression: ${probe} != 1
+                expression: ${probe} != 1 ||  ${http_status_code} != 200
                 intervalMs: 1000
                 maxDataPoints: 43200
                 refId: condition
@@ -114,8 +141,7 @@ groups:
           execErrState: Alerting
           for: 5m
           annotations:
-            summary: Probe failed for Laconicd GQL endpoint {{ index $labels "instance" }}
+            summary: Probe failed for Laconicd GQL endpoint
           labels:
-            endpoint: '{{ index $labels "instance" }}'
             probe_success: '{{ index $values "probe" }}'
           isPaused: false