From b9369a13e6dfc5b58a5a35da03757c142ccf99dc Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Wed, 15 May 2024 13:52:18 +0000 Subject: [PATCH] Update watcher dashboard with panels for ETH RPC request failures and durations (#825) Part of [Regenerate ajna watcher with updated subgraph config ](https://www.notion.so/Regenerate-ajna-watcher-with-updated-subgraph-config-c9bbecb033024c13a7515c7f1efc3363) Requires [Add metrics to monitor errors and duration for ETH RPC requests](https://github.com/cerc-io/watcher-ts/pull/507) Reviewed-on: https://git.vdb.to/cerc-io/stack-orchestrator/pulls/825 Co-authored-by: Prathamesh Musale Co-committed-by: Prathamesh Musale --- .../grafana/dashboards/watcher-dashboard.json | 476 +++++++++++++++++- stack_orchestrator/data/stacks/ajna/stack.yml | 2 +- 2 files changed, 453 insertions(+), 25 deletions(-) diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json index fe5fd244..0b62be5e 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json @@ -18,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 4, + "id": 6, "links": [ { "asDropdown": false, @@ -55,7 +55,7 @@ "x": 0, "y": 0 }, - "id": 1, + "id": 29, "panels": [ { "datasource": { @@ -103,7 +103,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.2", + "pluginVersion": "10.2.3", "targets": [ { "datasource": { @@ -173,7 +173,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.2", + "pluginVersion": "10.2.3", "targets": [ { "datasource": { @@ -242,7 +242,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.2", + "pluginVersion": "10.2.3", "targets": [ { "datasource": { @@ -438,7 +438,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.2", + "pluginVersion": "10.2.3", "targets": [ { "datasource": { @@ -518,7 +518,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.2", + "pluginVersion": "10.2.3", "targets": [ { "datasource": { @@ -629,7 +629,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.2", + "pluginVersion": "10.2.3", "targets": [ { "datasource": { @@ -730,7 +730,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.2", + "pluginVersion": "10.2.3", "targets": [ { "datasource": { @@ -854,7 +854,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.2", + "pluginVersion": "10.2.3", "targets": [ { "datasource": { @@ -1025,7 +1025,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.2", + "pluginVersion": "10.2.3", "targets": [ { "datasource": { @@ -1111,7 +1111,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.2", + "pluginVersion": "10.2.3", "targets": [ { "datasource": { @@ -1350,6 +1350,434 @@ "title": "Num event-processing jobs", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of failed ETH RPC requests by method and provider endpoint ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 90, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 28, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 1, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xField": "Time", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "10.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_eth_rpc_errors{job=~\"$job\", instance=~\"$watcher\"}", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "{{method}}, {{provider}}", + "range": false, + "refId": "A", + "useBackend": false + } + ], + "title": "ETH RPC request failures", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Time taken by last ETH RPC requests by method and provider endpoint", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_eth_rpc_request_duration{job=~\"$job\", instance=~\"$watcher\"}", + "instant": false, + "legendFormat": "{{method}}, {{provider}}", + "range": true, + "refId": "A" + } + ], + "title": "ETH RPC request durations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of failed ETH RPC requests by method and provider endpoint (across all watchers)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 90, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 30, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 1, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xField": "Time", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "10.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (method, provider) (watcher_eth_rpc_errors{chain=\"$target_chain\"})", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "{{method}}, {{provider}}", + "range": false, + "refId": "A", + "useBackend": false + } + ], + "title": "Total ETH RPC request failures (across all watchers)", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Configured upstream ETH RPC endpoints", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "palette-classic" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 0, + "text": "inactive" + }, + "1": { + "color": "green", + "index": 1, + "text": "active" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "status" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 32, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 0, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "status" + } + ] + }, + "pluginVersion": "10.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_config_upstream_endpoints{job=~\"$job\", instance=~\"$watcher\"}", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Configured upstream endpoints", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "mode": "columns" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "chain": true, + "instance": true, + "job": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "status" + } + } + } + ], + "type": "table" + }, { "datasource": { "type": "prometheus", @@ -1388,7 +1816,7 @@ "h": 5, "w": 8, "x": 0, - "y": 18 + "y": 32 }, "id": 18, "options": { @@ -1410,7 +1838,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.2", + "pluginVersion": "10.2.3", "targets": [ { "datasource": { @@ -1533,7 +1961,7 @@ "h": 5, "w": 8, "x": 8, - "y": 18 + "y": 32 }, "id": 17, "options": { @@ -1555,7 +1983,7 @@ "textMode": "value_and_name", "wideLayout": true }, - "pluginVersion": "10.2.2", + "pluginVersion": "10.2.3", "targets": [ { "datasource": { @@ -1680,7 +2108,7 @@ "h": 5, "w": 8, "x": 16, - "y": 18 + "y": 32 }, "id": 19, "options": { @@ -1702,7 +2130,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.2", + "pluginVersion": "10.2.3", "targets": [ { "datasource": { @@ -1792,7 +2220,6 @@ "uid": "PBFA97CFB590B2093" }, "description": "Total number of subgraph entities loaded in event processing", - "hide": true, "fieldConfig": { "defaults": { "color": { @@ -1847,8 +2274,9 @@ "h": 6, "w": 8, "x": 0, - "y": 23 + "y": 37 }, + "hide": true, "id": 8, "options": { "legend": { @@ -1943,7 +2371,7 @@ "h": 6, "w": 8, "x": 8, - "y": 23 + "y": 37 }, "id": 9, "options": { @@ -2039,7 +2467,7 @@ "h": 6, "w": 8, "x": 16, - "y": 23 + "y": 37 }, "id": 10, "options": { @@ -2084,7 +2512,7 @@ } ], "refresh": "10s", - "schemaVersion": 38, + "schemaVersion": 39, "tags": [ "watcher" ], @@ -2182,6 +2610,6 @@ "timepicker": {}, "timezone": "", "title": "Watchers", - "version": 1, + "version": 2, "weekStart": "" } diff --git a/stack_orchestrator/data/stacks/ajna/stack.yml b/stack_orchestrator/data/stacks/ajna/stack.yml index 4116a564..0e102c9e 100644 --- a/stack_orchestrator/data/stacks/ajna/stack.yml +++ b/stack_orchestrator/data/stacks/ajna/stack.yml @@ -2,7 +2,7 @@ version: "1.0" name: ajna description: "Ajna watcher stack" repos: - - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.5 + - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.6 containers: - cerc/watcher-ajna pods: