Add a custom label to blackbox metrics and update instructions

stack_orchestrator/data/config/monitoring/blackbox-alert-rules.yml

@@ -17,7 +17,7 @@ groups:
               datasourceUid: PBFA97CFB590B2093
               model:
                 editorMode: code
-                expr: probe_success{instance="<AZIMUTH_GATEWAY_GQL_ENDPOINT>"}
+                expr: probe_success{destination="azimuth_gateway"}
                 instant: true
                 intervalMs: 1000
                 legendFormat: __auto
@@ -74,7 +74,7 @@ groups:
               datasourceUid: PBFA97CFB590B2093
               model:
                 editorMode: code
-                expr: probe_success{instance="<LACONICD_GQL_ENDPOINT>"}
+                expr: probe_success{destination="laconicd_gql"}
                 instant: true
                 intervalMs: 1000
                 legendFormat: __auto

stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml

@@ -24,9 +24,10 @@ scrape_configs:
     params:
       module: [http_2xx]
     static_configs:
-      # Add URLs to be monitored below
-      - targets:
-        # - https://github.com
+      # Add URLs for targets to be monitored below
+      # - targets: [https://github.com]
+      #   labels:
+      #     destination: 'github'
     relabel_configs:
       - source_labels: [__address__]
         regex: (.*)(:80)?

stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md

@@ -44,9 +44,13 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
   - job_name: 'blackbox'
     ...
     static_configs:
-      - targets:
-        - <AZIMUTH_GATEWAY_GQL_ENDPOINT>
-        - <LACONICD_GQL_ENDPOINT>
+      - targets: [<AZIMUTH_GATEWAY_GQL_ENDPOINT>]
+        labels:
+          # Add destination label for pre-configured alerts
+          destination: 'azimuth_gateway'
+      - targets: [<LACONICD_GQL_ENDPOINT>]
+        labels:
+          destination: 'laconicd_gql'
   ...
   - job_name: laconicd
     ...
@@ -120,10 +124,12 @@ Add scrape config as done above for any additional watcher to add it to the Watc
 
 ### Grafana alerts config
 
-Place the pre-configured watcher alerts rules in Grafana provisioning directory:
+Place the pre-configured watcher and blackbox endpoint alerts rules in Grafana provisioning directory:
 
   ```bash
   cp monitoring-watchers-deployment/config/monitoring/watcher-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
+
+  cp monitoring-watchers-deployment/config/monitoring/blackbox-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
   ```
 
 Update the alerting contact points config (`monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/contactpoints.yml`) with desired contact points
@@ -136,7 +142,7 @@ Add corresponding routes to the notification policies config (`monitoring-watche
       - receiver: SlackNotifier
       object_matchers:
         # Add matchers below
-        - ['grafana_folder', '=', 'WatcherAlerts']
+        - ['grafana_folder', '=~', 'WatcherAlerts|BlackboxAlerts']
   ```
 
 ### Env