From 842d999792541bf3c8d3f57cdf7b804c921e6900 Mon Sep 17 00:00:00 2001
From: Nabarun <nabarun@deepstacksoft.com>
Date: Tue, 18 Jun 2024 12:28:02 +0000
Subject: [PATCH] Add alert rules for secured secured-finance subgraph watcher
 (#842)

Part of [Generate secured-finance subgraph watcher with codegen](https://www.notion.so/Generate-secured-finance-subgraph-watcher-with-codegen-2923413e0af54ea787c5435d6966f3bb)
- Update watcher dashboard labels

Reviewed-on: https://git.vdb.to/cerc-io/stack-orchestrator/pulls/842
Co-authored-by: Nabarun <nabarun@deepstacksoft.com>
Co-committed-by: Nabarun <nabarun@deepstacksoft.com>
---
 .../grafana/dashboards/watcher-dashboard.json |  4 +-
 .../config/monitoring/watcher-alert-rules.yml | 78 +++++++++++++++++++
 2 files changed, 80 insertions(+), 2 deletions(-)

diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json
index b2582615..f93bbe36 100644
--- a/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json
+++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json
@@ -322,7 +322,7 @@
               "useBackend": false
             }
           ],
-          "title": "Latest processed block",
+          "title": "Event processed block",
           "type": "stat"
         },
         {
@@ -391,7 +391,7 @@
               "useBackend": false
             }
           ],
-          "title": "Latest indexed block",
+          "title": "Inserted processed block",
           "type": "stat"
         },
         {
diff --git a/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml b/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml
index e5710565..c1c3e8e2 100644
--- a/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml
+++ b/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml
@@ -849,3 +849,81 @@ groups:
         annotations:
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
+      
+      # Secured Finance
+      - uid: secured_finance_diff_external
+        title: secured_finance_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="secured-finance", instance="secured-finance", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false