stack_orchestrator/data/config/monitoring/blackbox-alert-rules.yml

@@ -0,0 +1,115 @@
+apiVersion: 1
+groups:
+    - orgId: 1
+      name: blackbox
+      folder: BlackboxAlerts
+      interval: 30s
+      rules:
+        # Azimuth Gateway endpoint
+        - uid: azimuth_gateway
+          title: azimuth_gateway_endpoint_tracking
+          condition: condition
+          data:
+            - refId: http_status_code
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: PBFA97CFB590B2093
+              model:
+                editorMode: code
+                expr: probe_http_status_code{destination="azimuth_gateway"}
+                instant: true
+                intervalMs: 1000
+                legendFormat: __auto
+                maxDataPoints: 43200
+                range: false
+                refId: http_status_code
+            - refId: condition
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: __expr__
+              model:
+                conditions:
+                    - evaluator:
+                        params:
+                          - 200
+                          - 200
+                        type: outside_range
+                      operator:
+                        type: and
+                      query:
+                        params:
+                          - http_status_code
+                      reducer:
+                        params: []
+                        type: last
+                      type: query
+                datasource:
+                    name: Expression
+                    type: __expr__
+                    uid: __expr__
+                expression: ""
+                hide: false
+                refId: condition
+                type: classic_conditions
+          noDataState: Alerting
+          execErrState: Alerting
+          for: 5m
+          annotations:
+            summary: Probe failed for Azimuth gateway endpoint, http status {{ index $values "http_status_code" }}
+          isPaused: false
+        # Laconicd GQL endpoint
+        - uid: laconicd_gql
+          title: laconicd_gql_endpoint_tracking
+          condition: condition
+          data:
+            - refId: http_status_code
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: PBFA97CFB590B2093
+              model:
+                editorMode: code
+                expr: probe_http_status_code{destination="laconicd_gql"}
+                instant: true
+                intervalMs: 1000
+                legendFormat: __auto
+                maxDataPoints: 43200
+                range: false
+                refId: http_status_code
+            - refId: condition
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: __expr__
+              model:
+                conditions:
+                    - evaluator:
+                        params:
+                          - 200
+                          - 200
+                        type: outside_range
+                      operator:
+                        type: and
+                      query:
+                        params:
+                          - http_status_code
+                      reducer:
+                        params: []
+                        type: last
+                      type: query
+                datasource:
+                    name: Expression
+                    type: __expr__
+                    uid: __expr__
+                expression: ""
+                hide: false
+                refId: condition
+                type: classic_conditions
+          noDataState: Alerting
+          execErrState: Alerting
+          for: 5m
+          annotations:
+            summary: Probe failed for Laconicd GQL endpoint, http status {{ index $values "http_status_code" }}
+          isPaused: false

stack_orchestrator/data/config/monitoring/grafana/dashboards/laconicd-dashboard.json

@@ -49,7 +49,7 @@
       },
       "gridPos": {
         "h": 3,
-        "w": 3,
+        "w": 4,
         "x": 0,
         "y": 0
       },

stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml

@@ -24,9 +24,10 @@ scrape_configs:
     params:
       module: [http_2xx]
     static_configs:
-      # Add URLs to be monitored below
+      # Add URLs for targets to be monitored below
-      - targets:
+      # - targets: [https://github.com]
-        # - https://github.com
+      #   labels:
+      #     destination: 'github'
     relabel_configs:
       - source_labels: [__address__]
         regex: (.*)(:80)?

stack_orchestrator/data/stacks/monitoring/README.md

@@ -123,6 +123,7 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --de
   ```yml
   ...
   - job_name: laconicd
+    ...
     static_configs:
       - targets: ['example-host:1317']
   ...

stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md

@@ -44,11 +44,16 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
   - job_name: 'blackbox'
     ...
     static_configs:
-      - targets:
+      - targets: [<AZIMUTH_GATEWAY_GQL_ENDPOINT>]
-        - <AZIMUTH_GATEWAY_GQL_ENDPOINT>
+        labels:
-        - <LACONICD_GQL_ENDPOINT>
+          # Add destination label for pre-configured alerts
+          destination: 'azimuth_gateway'
+      - targets: [<LACONICD_GQL_ENDPOINT>]
+        labels:
+          destination: 'laconicd_gql'
   ...
   - job_name: laconicd
+    ...
     static_configs:
       - targets: ['LACONICD_REST_HOST:LACONICD_REST_PORT']
   ...
@@ -119,10 +124,12 @@ Add scrape config as done above for any additional watcher to add it to the Watc
 
 ### Grafana alerts config
 
-Place the pre-configured watcher alerts rules in Grafana provisioning directory:
+Place the pre-configured watcher and blackbox endpoint alerts rules in Grafana provisioning directory:
 
   ```bash
   cp monitoring-watchers-deployment/config/monitoring/watcher-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
+
+  cp monitoring-watchers-deployment/config/monitoring/blackbox-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
   ```
 
 Update the alerting contact points config (`monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/contactpoints.yml`) with desired contact points
@@ -135,7 +142,7 @@ Add corresponding routes to the notification policies config (`monitoring-watche
       - receiver: SlackNotifier
       object_matchers:
         # Add matchers below
-        - ['grafana_folder', '=', 'WatcherAlerts']
+        - ['grafana_folder', '=~', 'WatcherAlerts|BlackboxAlerts']
   ```
 
 ### Env