From 96b41e730789c5a4777d17f727742674bf41c134 Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Fri, 10 May 2024 14:49:20 +0530
Subject: [PATCH] Make subgraph ids for alerts configurable

---
 .../data/compose/docker-compose-grafana.yml           |  6 ++++++
 .../data/config/monitoring/subgraph-alert-rules.yml   |  4 ++--
 .../config/monitoring/update-grafana-alerts-config.sh |  7 +++++++
 .../data/stacks/monitoring/monitoring-watchers.md     | 11 +++++++++--
 4 files changed, 24 insertions(+), 4 deletions(-)
 create mode 100755 stack_orchestrator/data/config/monitoring/update-grafana-alerts-config.sh

diff --git a/stack_orchestrator/data/compose/docker-compose-grafana.yml b/stack_orchestrator/data/compose/docker-compose-grafana.yml
index d559b246..5054a0db 100644
--- a/stack_orchestrator/data/compose/docker-compose-grafana.yml
+++ b/stack_orchestrator/data/compose/docker-compose-grafana.yml
@@ -6,10 +6,16 @@ services:
     restart: always
     environment:
       GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL}
+      CERC_GRAFANA_ALERTS_SUBGRAPH_IDS: ${CERC_GRAFANA_ALERTS_SUBGRAPH_IDS}
     volumes:
       - ../config/monitoring/grafana/provisioning:/etc/grafana/provisioning
       - ../config/monitoring/grafana/dashboards:/etc/grafana/dashboards
+      - ../config/monitoring/update-grafana-alerts-config.sh:/update-grafana-alerts-config.sh
       - grafana_storage:/var/lib/grafana
+    user: root
+    entrypoint: ["bash", "-c"]
+    command: |
+      "/update-grafana-alerts-config.sh && /run.sh"
     ports:
       - "3000"
     healthcheck:
diff --git a/stack_orchestrator/data/config/monitoring/subgraph-alert-rules.yml b/stack_orchestrator/data/config/monitoring/subgraph-alert-rules.yml
index 5936c685..ed59e8ef 100644
--- a/stack_orchestrator/data/config/monitoring/subgraph-alert-rules.yml
+++ b/stack_orchestrator/data/config/monitoring/subgraph-alert-rules.yml
@@ -19,7 +19,7 @@ groups:
                   type: prometheus
                   uid: PBFA97CFB590B2093
               editorMode: code
-              expr: ethereum_chain_head_number - on(network) group_right deployment_head{}
+              expr: ethereum_chain_head_number - on(network) group_right deployment_head{deployment=~"REPLACE_WITH_SUBGRAPH_IDS"}
               instant: true
               intervalMs: 1000
               legendFormat: __auto
@@ -55,7 +55,7 @@ groups:
               maxDataPoints: 43200
               refId: condition
               type: threshold
-        noDataState: Alerting
+        noDataState: OK
         execErrState: Alerting
         for: 5m
         annotations:
diff --git a/stack_orchestrator/data/config/monitoring/update-grafana-alerts-config.sh b/stack_orchestrator/data/config/monitoring/update-grafana-alerts-config.sh
new file mode 100755
index 00000000..b6ec932c
--- /dev/null
+++ b/stack_orchestrator/data/config/monitoring/update-grafana-alerts-config.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo Using CERC_GRAFANA_ALERTS_SUBGRAPH_IDS ${CERC_GRAFANA_ALERTS_SUBGRAPH_IDS}
+
+# Replace subgraph ids in subgraph alerting config
+# Note: Requires the grafana container to be run with user root
+sed -i "s/REPLACE_WITH_SUBGRAPH_IDS/$CERC_GRAFANA_ALERTS_SUBGRAPH_IDS/g" /etc/grafana/provisioning/alerting/subgraph-alert-rules.yml
diff --git a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md
index 2f057c3c..dff8af9d 100644
--- a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md
+++ b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md
@@ -119,10 +119,14 @@ Add scrape config as done above for any additional watcher to add it to the Watc
 
 ### Grafana alerts config
 
-Place the pre-configured watcher alerts rules in Grafana provisioning directory:
+Place the pre-configured alerts rules in Grafana provisioning directory:
 
   ```bash
+  # watcher alert rules
   cp monitoring-watchers-deployment/config/monitoring/watcher-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
+
+  # subgraph alert rules
+  cp monitoring-watchers-deployment/config/monitoring/subgraph-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
   ```
 
 Update the alerting contact points config (`monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/contactpoints.yml`) with desired contact points
@@ -135,7 +139,7 @@ Add corresponding routes to the notification policies config (`monitoring-watche
       - receiver: SlackNotifier
       object_matchers:
         # Add matchers below
-        - ['grafana_folder', '=', 'WatcherAlerts']
+        - ['grafana_folder', '=~', 'WatcherAlerts|SubgraphAlerts']
   ```
 
 ### Env
@@ -149,6 +153,9 @@ Set the following env variables in the deployment env config file (`monitoring-w
   # Grafana server host URL to be used
   # (Optional, default: http://localhost:3000)
   GF_SERVER_ROOT_URL=
+
+  # List of subgraph ids to configure alerts for (separated by |)
+  CERC_GRAFANA_ALERTS_SUBGRAPH_IDS=
   ```
 
 ## Start the stack