Add panels for GQL metrics to watcher dashboard (#834)

Part of [Metrics and logging for GQL queries in watcher](https://www.notion.so/Metrics-and-logging-for-GQL-queries-in-watcher-928c692292b140a2a4f52cda9795df5e)

Reviewed-on: #834
Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com>
Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>

stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml

@@ -76,6 +76,7 @@ services:
       - ../config/watcher-azimuth/start-server.sh:/app/packages/azimuth-watcher/start-server.sh
     ports:
       - "3001"
+      - "9001"
     healthcheck:
       test: ["CMD", "nc", "-vz", "127.0.0.1", "3001"]
       interval: 20s
@@ -137,6 +138,7 @@ services:
       - ../config/watcher-azimuth/start-server.sh:/app/packages/censures-watcher/start-server.sh
     ports:
       - "3002"
+      - "9003"
     healthcheck:
       test: ["CMD", "nc", "-vz", "127.0.0.1", "3002"]
       interval: 20s
@@ -198,6 +200,7 @@ services:
       - ../config/watcher-azimuth/start-server.sh:/app/packages/claims-watcher/start-server.sh
     ports:
       - "3003"
+      - "9005"
     healthcheck:
       test: ["CMD", "nc", "-vz", "127.0.0.1", "3003"]
       interval: 20s
@@ -259,6 +262,7 @@ services:
       - ../config/watcher-azimuth/start-server.sh:/app/packages/conditional-star-release-watcher/start-server.sh
     ports:
       - "3004"
+      - "9007"
     healthcheck:
       test: ["CMD", "nc", "-vz", "127.0.0.1", "3004"]
       interval: 20s
@@ -320,6 +324,7 @@ services:
       - ../config/watcher-azimuth/start-server.sh:/app/packages/delegated-sending-watcher/start-server.sh
     ports:
       - "3005"
+      - "9009"
     healthcheck:
       test: ["CMD", "nc", "-vz", "127.0.0.1", "3005"]
       interval: 20s
@@ -381,6 +386,7 @@ services:
       - ../config/watcher-azimuth/start-server.sh:/app/packages/ecliptic-watcher/start-server.sh
     ports:
       - "3006"
+      - "9011"
     healthcheck:
       test: ["CMD", "nc", "-vz", "127.0.0.1", "3006"]
       interval: 20s
@@ -442,6 +448,7 @@ services:
       - ../config/watcher-azimuth/start-server.sh:/app/packages/linear-star-release-watcher/start-server.sh
     ports:
       - "3007"
+      - "9013"
     healthcheck:
       test: ["CMD", "nc", "-vz", "127.0.0.1", "3007"]
       interval: 20s
@@ -503,6 +510,7 @@ services:
       - ../config/watcher-azimuth/start-server.sh:/app/packages/polls-watcher/start-server.sh
     ports:
       - "3008"
+      - "9015"
     healthcheck:
       test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
       interval: 20s

stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json

@@ -18,7 +18,7 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "id": 6,
+  "id": 4,
   "links": [
     {
       "asDropdown": false,
@@ -117,7 +117,7 @@
               "disableTextWrap": false,
               "editorMode": "code",
               "exemplar": false,
-              "expr": "max(watcher_info{job=\"$job\", instance=\"$watcher\"}) by (version, commitHash)",
+              "expr": "max(watcher_info{job=\"$job\", instance=\"$watcher\"}) by (repository, version, commitHash)",
               "format": "table",
               "fullMetaSearch": false,
               "includeNullMetadata": true,
@@ -139,12 +139,14 @@
                 "includeByName": {},
                 "indexByName": {
                   "Time": 0,
-                  "Value": 3,
+                  "Value": 4,
-                  "commitHash": 2,
+                  "commitHash": 3,
-                  "version": 1
+                  "repository": 1,
+                  "version": 2
                 },
                 "renameByName": {
                   "commitHash": "Commit hash",
+                  "repository": "Repository",
                   "version": "Release"
                 }
               }
@@ -2099,6 +2101,282 @@
           ],
           "type": "table"
         },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 6,
+            "w": 8,
+            "x": 0,
+            "y": 33
+          },
+          "id": 35,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "editorMode": "code",
+              "expr": "gql_query_count_total{job=~\"$job\", instance=~\"$watcher\"}",
+              "instant": false,
+              "legendFormat": "{{__name__}}",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Total GQL query count",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 6,
+            "w": 8,
+            "x": 8,
+            "y": 33
+          },
+          "id": 36,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "editorMode": "code",
+              "expr": "gql_query_count{job=~\"$job\", instance=~\"$watcher\"}",
+              "instant": false,
+              "legendFormat": "{{name}}",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "GQL query count",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 6,
+            "w": 8,
+            "x": 16,
+            "y": 33
+          },
+          "id": 37,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "gql_query_duration_seconds{job=~\"$job\", instance=~\"$watcher\"}",
+              "instant": false,
+              "legendFormat": "{{name}}",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "GQL queries duration",
+          "type": "timeseries"
+        },
         {
           "datasource": {
             "type": "prometheus",
@@ -2138,7 +2416,7 @@
             "h": 5,
             "w": 8,
             "x": 0,
-            "y": 33
+            "y": 39
           },
           "id": 18,
           "options": {
@@ -2284,7 +2562,7 @@
             "h": 5,
             "w": 8,
             "x": 8,
-            "y": 33
+            "y": 39
           },
           "id": 17,
           "options": {
@@ -2432,7 +2710,7 @@
             "h": 5,
             "w": 8,
             "x": 16,
-            "y": 33
+            "y": 39
           },
           "id": 19,
           "options": {
@@ -2599,7 +2877,7 @@
             "h": 6,
             "w": 8,
             "x": 0,
-            "y": 38
+            "y": 44
           },
           "hide": true,
           "id": 8,
@@ -2697,7 +2975,7 @@
             "h": 6,
             "w": 8,
             "x": 8,
-            "y": 38
+            "y": 44
           },
           "id": 9,
           "options": {
@@ -2794,7 +3072,7 @@
             "h": 6,
             "w": 8,
             "x": 16,
-            "y": 38
+            "y": 44
           },
           "id": 10,
           "options": {
@@ -2937,6 +3215,6 @@
   "timepicker": {},
   "timezone": "",
   "title": "Watchers",
-  "version": 3,
+  "version": 4,
   "weekStart": ""
 }

stack_orchestrator/data/stacks/ajna/stack.yml

@@ -2,7 +2,7 @@ version: "1.0"
 name: ajna
 description: "Ajna watcher stack"
 repos:
-  - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.11
+  - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.12
 containers:
   - cerc/watcher-ajna
 pods:

stack_orchestrator/data/stacks/azimuth/README.md

@@ -44,34 +44,42 @@ network:
      - 0.0.0.0:9000:9000
     azimuth-watcher-server:
      - 0.0.0.0:3001:3001
+     - 0.0.0.0:9001:9001
     censures-watcher-job-runner:
      - 0.0.0.0:9002:9002
     censures-watcher-server:
      - 0.0.0.0:3002:3002
+     - 0.0.0.0:9003:9003
     claims-watcher-job-runner:
      - 0.0.0.0:9004:9004
     claims-watcher-server:
      - 0.0.0.0:3003:3003
+     - 0.0.0.0:9005:9005
     conditional-star-release-watcher-job-runner:
      - 0.0.0.0:9006:9006
     conditional-star-release-watcher-server:
      - 0.0.0.0:3004:3004
+     - 0.0.0.0:9007:9007
     delegated-sending-watcher-job-runner:
      - 0.0.0.0:9008:9008
     delegated-sending-watcher-server:
      - 0.0.0.0:3005:3005
+     - 0.0.0.0:9009:9009
     ecliptic-watcher-job-runner:
      - 0.0.0.0:9010:9010
     ecliptic-watcher-server:
      - 0.0.0.0:3006:3006
+     - 0.0.0.0:9011:9011
     linear-star-release-watcher-job-runner:
      - 0.0.0.0:9012:9012
     linear-star-release-watcher-server:
      - 0.0.0.0:3007:3007
+     - 0.0.0.0:9013:9013
     polls-watcher-job-runner:
      - 0.0.0.0:9014:9014
     polls-watcher-server:
      - 0.0.0.0:3008:3008
+     - 0.0.0.0:9015:9015
     gateway-server:
      - 0.0.0.0:4000:4000
 ...

stack_orchestrator/data/stacks/azimuth/stack.yml

@@ -1,7 +1,7 @@
 version: "1.0"
 name: azimuth
 repos:
-  - github.com/cerc-io/azimuth-watcher-ts@0.1.5
+  - github.com/cerc-io/azimuth-watcher-ts@0.1.6
 containers:
   - cerc/watcher-azimuth
 pods:

stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml

@@ -2,7 +2,7 @@ version: "1.0"
 name: merkl-sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.12
+  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.13
 containers:
   - cerc/watcher-merkl-sushiswap-v3
 pods:

stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md

@@ -57,35 +57,35 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
     metrics_path: /metrics
     scheme: http
     static_configs:
-      - targets: ['AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_PORT']
+      - targets: ['AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_METRICS_PORT', 'AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'azimuth'
           chain: 'ethereum'
-      - targets: ['CENSURES_WATCHER_HOST:CENSURES_WATCHER_PORT']
+      - targets: ['CENSURES_WATCHER_HOST:CENSURES_WATCHER_METRICS_PORT', 'CENSURES_WATCHER_HOST:CENSURES_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'censures'
           chain: 'ethereum'
-      - targets: ['CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_PORT']
+      - targets: ['CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_METRICS_PORT', 'CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'claims'
           chain: 'ethereum'
-      - targets: ['CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_PORT']
+      - targets: ['CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_METRICS_PORT', 'CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'conditional_star_release'
           chain: 'ethereum'
-      - targets: ['DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_PORT']
+      - targets: ['DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_METRICS_PORT', 'DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'delegated_sending'
           chain: 'ethereum'
-      - targets: ['ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_PORT']
+      - targets: ['ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_METRICS_PORT', 'ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'ecliptic'
           chain: 'ethereum'
-      - targets: ['LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_PORT']
+      - targets: ['LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_METRICS_PORT', 'LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'linear_star_release'
           chain: 'ethereum'
-      - targets: ['POLLS_WATCHER_HOST:POLLS_WATCHER_PORT']
+      - targets: ['POLLS_WATCHER_HOST:POLLS_WATCHER_METRICS_PORT', 'POLLS_WATCHER_HOST:POLLS_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'polls'
           chain: 'ethereum'
@@ -95,11 +95,11 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
     metrics_path: /metrics
     scheme: http
     static_configs:
-      - targets: ['SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_PORT']
+      - targets: ['SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_METRICS_PORT', 'SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'sushiswap'
           chain: 'filecoin'
-      - targets: ['MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_PORT']
+      - targets: ['MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_METRICS_PORT', 'MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'merkl_sushiswap'
           chain: 'filecoin'
@@ -109,7 +109,7 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
     metrics_path: /metrics
     scheme: http
     static_configs:
-      - targets: ['AJNA_WATCHER_HOST:AJNA_WATCHER_PORT']
+      - targets: ['AJNA_WATCHER_HOST:AJNA_WATCHER_METRICS_PORT', 'AJNA_WATCHER_HOST:AJNA_WATCHER_GQL_METRICS_PORT']
         labels:
           instance: 'ajna'
           chain: 'filecoin'

stack_orchestrator/data/stacks/sushiswap-v3/stack.yml

@@ -2,7 +2,7 @@ version: "1.0"
 name: sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.12
+  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.13
 containers:
   - cerc/watcher-sushiswap-v3
 pods: