From 6ee69623a3f454f81705f8dc6504fc61853377c1 Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Wed, 10 Apr 2024 16:03:49 +0530
Subject: [PATCH] Add alert rule for Ajna watcher

---
 .../config/monitoring/watcher-alert-rules.yml | 78 +++++++++++++++++++
 .../stacks/monitoring/monitoring-watchers.md  | 10 +++
 2 files changed, 88 insertions(+)

diff --git a/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml b/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml
index a190a1fd..c2025029 100644
--- a/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml
+++ b/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml
@@ -771,3 +771,81 @@ groups:
         annotations:
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
+
+      # Ajna
+      - uid: ajna_diff_external
+        title: ajna_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="ajna", instance="ajna", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
diff --git a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md
index ebd67a5b..2f057c3c 100644
--- a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md
+++ b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md
@@ -103,6 +103,16 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
         labels:
           instance: 'merkl_sushiswap'
           chain: 'filecoin'
+
+  - job_name: ajna
+    scrape_interval: 20s
+    metrics_path: /metrics
+    scheme: http
+    static_configs:
+      - targets: ['AJNA_WATCHER_HOST:AJNA_WATCHER_PORT']
+        labels:
+          instance: 'ajna'
+          chain: 'filecoin'
   ```
 
 Add scrape config as done above for any additional watcher to add it to the Watchers dashboard.