From 00b3f52646a7eab0b3ff4e437bc9598ff8be9e29 Mon Sep 17 00:00:00 2001
From: Prathamesh Musale <prathamesh.musale0@gmail.com>
Date: Fri, 31 May 2024 18:09:08 +0530
Subject: [PATCH] Configure prometheus alert manager with Slack notifications

---
 .../data/compose/docker-compose-prom-server.yml        |  9 +++++++++
 .../config/monitoring/alertmanager/alertmanager.yml    | 10 ++++++++++
 .../data/config/monitoring/prometheus/rules.yml        | 10 ++++++++++
 3 files changed, 29 insertions(+)
 create mode 100644 stack_orchestrator/data/config/monitoring/alertmanager/alertmanager.yml
 create mode 100644 stack_orchestrator/data/config/monitoring/prometheus/rules.yml

diff --git a/stack_orchestrator/data/compose/docker-compose-prom-server.yml b/stack_orchestrator/data/compose/docker-compose-prom-server.yml
index 9095b6dc..706d359d 100644
--- a/stack_orchestrator/data/compose/docker-compose-prom-server.yml
+++ b/stack_orchestrator/data/compose/docker-compose-prom-server.yml
@@ -18,6 +18,14 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
+  alert-manager:
+    image: prom/alertmanager
+    volumes:
+      - ../config/monitoring/alertmanager.yml:/alertmanager/alertmanager.yml
+      - alertmanager_data:/data
+    ports:
+      - "9093"
+
   blackbox:
     image: prom/blackbox-exporter:latest
     restart: always
@@ -55,3 +63,4 @@ services:
 
 volumes:
   prometheus_data:
+  alertmanager_data:
diff --git a/stack_orchestrator/data/config/monitoring/alertmanager/alertmanager.yml b/stack_orchestrator/data/config/monitoring/alertmanager/alertmanager.yml
new file mode 100644
index 00000000..f192b5da
--- /dev/null
+++ b/stack_orchestrator/data/config/monitoring/alertmanager/alertmanager.yml
@@ -0,0 +1,10 @@
+route:
+  receiver: 'slack'
+  # group_by: [ alertname ]
+
+receivers:
+  - name: 'slack'
+    slack_configs:
+      - send_resolved: true
+        api_url: <YOUR_SLACK_WEBHOOK_URL>
+        channel: <YOUR_SLACK_CHANNEL>
diff --git a/stack_orchestrator/data/config/monitoring/prometheus/rules.yml b/stack_orchestrator/data/config/monitoring/prometheus/rules.yml
new file mode 100644
index 00000000..94ce3c36
--- /dev/null
+++ b/stack_orchestrator/data/config/monitoring/prometheus/rules.yml
@@ -0,0 +1,10 @@
+groups:
+  - name: WatcherAlerts
+    rules:
+      - alert: SushiswapWatcherHeadTracking
+        expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"} > 16
+        for: 5m
+        labels:
+          severity: 'critical'
+        annotations:
+          summary: 'Watcher {{ $labels.instance }} of job {{ $labels.job }} is falling behind external head'