Configure prometheus alert manager with Slack notifications

stack_orchestrator/data/compose/docker-compose-prom-server.yml

@@ -18,6 +18,14 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
+  alert-manager:
+    image: prom/alertmanager
+    volumes:
+      - ../config/monitoring/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml
+      - alertmanager_data:/data
+    ports:
+      - "9093"
+
   blackbox:
     image: prom/blackbox-exporter:latest
     restart: always
@@ -55,3 +63,4 @@ services:
 
 volumes:
   prometheus_data:
+  alertmanager_data:

stack_orchestrator/data/config/monitoring/alertmanager/alertmanager.yml

@@ -0,0 +1,10 @@
+route:
+  receiver: 'slack'
+  # group_by: [ alertname ]
+
+receivers:
+  - name: 'slack'
+    slack_configs:
+      - send_resolved: true
+        api_url: <YOUR_SLACK_WEBHOOK_URL>
+        channel: <YOUR_SLACK_CHANNEL>

stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml

@@ -3,8 +3,13 @@ global:
   evaluation_interval: 15s
 
 rule_files:
-  # - "first.rules"
-  # - "second.rules"
+  - rules.yml
+
+alerting:
+  alertmanagers:
+   - static_configs:
+     - targets:
+        - alert-manager:9093
 
 scrape_configs:
   - job_name: prometheus

stack_orchestrator/data/config/monitoring/prometheus/rules.yml

@@ -0,0 +1,10 @@
+groups:
+  - name: WatcherAlerts
+    rules:
+      - alert: SushiswapWatcherHeadTracking
+        expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"} > 16
+        for: 5m
+        labels:
+          severity: 'critical'
+        annotations:
+          summary: 'Watcher {{ $labels.instance }} of job {{ $labels.job }} is falling behind external head'