From 28bddea3a749035d4c145e7068b0eb45219444d8 Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Mon, 8 Apr 2024 19:15:11 +0530 Subject: [PATCH 01/10] Add a laconicd Grafana dashboard to monitoring stack --- .../dashboards/laconicd-dashboard.json | 761 ++++++++++++++++++ .../monitoring/prometheus/prometheus.yml | 9 + .../data/stacks/monitoring/README.md | 11 + .../stacks/monitoring/monitoring-watchers.md | 4 + 4 files changed, 785 insertions(+) create mode 100644 stack_orchestrator/data/config/monitoring/grafana/dashboards/laconicd-dashboard.json diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/laconicd-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/laconicd-dashboard.json new file mode 100644 index 00000000..9179d283 --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/laconicd-dashboard.json @@ -0,0 +1,761 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 6, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 9, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "tendermint_consensus_latest_block_height", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "latest_block_height", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "tx_count", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "tx_count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "tx_gas_used", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "tx_gas_used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "tx_successful", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "tx_successful", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "tx_gas_wanted", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "tx_gas_wanted", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "tx_failed", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "tx_failed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 30 + }, + { + "color": "red", + "value": 50 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "tendermint_mempool_size", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "tendermint_mempool_size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "query_count", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "query_count", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "tags": [ + "registry" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Laconicd", + "uid": "fc7ea92d-8846-474a-89ed-5256c514f8c6", + "version": 14, + "weekStart": "" +} diff --git a/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml b/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml index 2d6a44da..d95c0373 100644 --- a/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml +++ b/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml @@ -65,3 +65,12 @@ scrape_configs: target_label: instance - target_label: __address__ replacement: postgres-exporter:9187 + + - job_name: laconicd + metrics_path: /metrics + scrape_interval: 30s + static_configs: + # Add laconicd REST endpoint target with host and port (1317) + - targets: ['LACONICD_REST_HOST:LACONICD_REST_PORT'] + params: + format: ['prometheus'] diff --git a/stack_orchestrator/data/stacks/monitoring/README.md b/stack_orchestrator/data/stacks/monitoring/README.md index 74b1bff8..1c44c34b 100644 --- a/stack_orchestrator/data/stacks/monitoring/README.md +++ b/stack_orchestrator/data/stacks/monitoring/README.md @@ -4,6 +4,7 @@ * Comes with the following built-in exporters / dashboards: * Chain Head Exporter - for tracking chain heads given external ETH RPC endpoints * Watchers dashboard + * Laconicd dashboard * [Prometheus Blackbox](https://grafana.com/grafana/dashboards/7587-prometheus-blackbox-exporter/) - for tracking HTTP endpoints * [NodeJS Application Dashboard](https://grafana.com/grafana/dashboards/11159-nodejs-application-dashboard/) - for default NodeJS metrics * [PostgreSQL Database](https://grafana.com/grafana/dashboards/9628-postgresql-database/) - for monitoring Postgres dbs @@ -116,6 +117,16 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --de ``` * Add database credentials to be used in `auth_modules` in the postgres-exporter config file (`monitoring-deployment/config/monitoring/postgres-exporter.yml`) +* Laconicd: update the `laconicd` job with a Laconicd node's REST endpoint host and port: + + ```yml + ... + - job_name: laconicd + static_configs: + - targets: ['example-host:1317'] + ... + ``` + Note: Use `host.docker.internal` as host to access ports on the host machine ### Grafana Config diff --git a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md index bce2d84b..f7acafc7 100644 --- a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md +++ b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md @@ -47,6 +47,10 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers - targets: - ... + - job_name: laconicd + static_configs: + - targets: ['LACONICD_REST_HOST:LACONICD_REST_PORT'] + ... - job_name: azimuth scrape_interval: 10s metrics_path: /metrics -- 2.45.2 From 14388068bf0243065e224f0e6d6cfbe5087bb42a Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Wed, 10 Apr 2024 11:17:14 +0530 Subject: [PATCH 02/10] Add panels for laconicd module queries --- .../dashboards/laconicd-dashboard.json | 316 +++++++++++++++++- 1 file changed, 308 insertions(+), 8 deletions(-) diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/laconicd-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/laconicd-dashboard.json index 9179d283..d5ab3e74 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/dashboards/laconicd-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/laconicd-dashboard.json @@ -170,7 +170,7 @@ "editorMode": "code", "expr": "tx_count", "instant": false, - "legendFormat": "__auto", + "legendFormat": "{{__name__}}", "range": true, "refId": "A" } @@ -262,7 +262,7 @@ "editorMode": "code", "expr": "tx_gas_used", "instant": false, - "legendFormat": "__auto", + "legendFormat": "{{__name__}}", "range": true, "refId": "A" } @@ -354,7 +354,7 @@ "editorMode": "code", "expr": "tx_successful", "instant": false, - "legendFormat": "__auto", + "legendFormat": "{{__name__}}", "range": true, "refId": "A" } @@ -447,7 +447,7 @@ "editorMode": "code", "expr": "tx_gas_wanted", "instant": false, - "legendFormat": "__auto", + "legendFormat": "{{__name__}}", "range": true, "refId": "A" } @@ -539,7 +539,7 @@ "editorMode": "code", "expr": "tx_failed", "instant": false, - "legendFormat": "__auto", + "legendFormat": "{{__name__}}", "range": true, "refId": "A" } @@ -639,7 +639,7 @@ "editorMode": "code", "expr": "tendermint_mempool_size", "instant": false, - "legendFormat": "__auto", + "legendFormat": "{{__name__}}", "range": true, "refId": "A" } @@ -731,13 +731,313 @@ "editorMode": "code", "expr": "query_count", "instant": false, - "legendFormat": "__auto", + "legendFormat": "{{__name__}}", "range": true, "refId": "A" } ], "title": "query_count", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 25 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "{__name__ =~ \"^_vulcanize_bond_v1beta1.*_count$\"}", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + } + ], + "title": "bond_query_count", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "_vulcanize_bond_v1beta1_Query_(.*?)_count", + "renamePattern": "$1" + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 25 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "{__name__ =~ \"^_vulcanize_auction_v1beta1.*_count$\"}", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + } + ], + "title": "auction_query_count", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "_vulcanize_auction_v1beta1_Query_(.*?)_count", + "renamePattern": "$1" + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 25 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "{__name__ =~ \"^_vulcanize_registry_v1beta1.*_count$\"}", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + } + ], + "title": "registry_query_count", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "_vulcanize_registry_v1beta1_Query_(.*?)_count", + "renamePattern": "$1" + } + } + ], + "type": "timeseries" } ], "refresh": "", @@ -756,6 +1056,6 @@ "timezone": "", "title": "Laconicd", "uid": "fc7ea92d-8846-474a-89ed-5256c514f8c6", - "version": 14, + "version": 21, "weekStart": "" } -- 2.45.2 From 17c48c2d20300c5bfd71f90656ce1c7f7c8704cb Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Wed, 10 Apr 2024 11:21:01 +0530 Subject: [PATCH 03/10] Upgrade Grafana version --- stack_orchestrator/data/compose/docker-compose-grafana.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack_orchestrator/data/compose/docker-compose-grafana.yml b/stack_orchestrator/data/compose/docker-compose-grafana.yml index 6d32e37b..d559b246 100644 --- a/stack_orchestrator/data/compose/docker-compose-grafana.yml +++ b/stack_orchestrator/data/compose/docker-compose-grafana.yml @@ -2,7 +2,7 @@ version: "3.7" services: grafana: - image: grafana/grafana:10.2.2 + image: grafana/grafana:10.2.3 restart: always environment: GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL} -- 2.45.2 From 891bc3f4cb3bde6fe16e805f561ac0acc0c47412 Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Wed, 10 Apr 2024 11:57:41 +0530 Subject: [PATCH 04/10] Add panel descriptions --- .../grafana/dashboards/laconicd-dashboard.json | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/laconicd-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/laconicd-dashboard.json index d5ab3e74..540c8da9 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/dashboards/laconicd-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/laconicd-dashboard.json @@ -27,6 +27,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "The tendermint consensus latest block height", "fieldConfig": { "defaults": { "color": { @@ -68,7 +69,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.2", + "pluginVersion": "10.2.3", "targets": [ { "datasource": { @@ -91,6 +92,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Total number of txs processed via DeliverTx", "fieldConfig": { "defaults": { "color": { @@ -183,6 +185,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Total amount of gas used by the last tx", "fieldConfig": { "defaults": { "color": { @@ -275,6 +278,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Total number of successful txs processed via DeliverTx", "fieldConfig": { "defaults": { "color": { @@ -367,6 +371,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Total amount of gas requested by a tx", "fieldConfig": { "defaults": { "color": { @@ -460,6 +465,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Total number of failed txs processed via DeliverTx", "fieldConfig": { "defaults": { "color": { @@ -552,6 +558,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Size of the mempool (number of uncommitted transactions)", "fieldConfig": { "defaults": { "color": { @@ -652,6 +659,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Total query count", "fieldConfig": { "defaults": { "color": { @@ -744,6 +752,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Bond module query count", "fieldConfig": { "defaults": { "color": { @@ -844,6 +853,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Auction module query count", "fieldConfig": { "defaults": { "color": { @@ -944,6 +954,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Registry module query count", "fieldConfig": { "defaults": { "color": { @@ -1041,7 +1052,7 @@ } ], "refresh": "", - "schemaVersion": 38, + "schemaVersion": 39, "tags": [ "registry" ], @@ -1056,6 +1067,6 @@ "timezone": "", "title": "Laconicd", "uid": "fc7ea92d-8846-474a-89ed-5256c514f8c6", - "version": 21, + "version": 25, "weekStart": "" } -- 2.45.2 From 9cd62351098d3d11ba8230e194eef2485eee405d Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Wed, 10 Apr 2024 12:25:59 +0530 Subject: [PATCH 05/10] Resize latest block height panel --- .../grafana/dashboards/laconicd-dashboard.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/laconicd-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/laconicd-dashboard.json index 540c8da9..84d691a9 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/dashboards/laconicd-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/laconicd-dashboard.json @@ -48,8 +48,8 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 12, + "h": 3, + "w": 3, "x": 0, "y": 0 }, @@ -237,10 +237,10 @@ "overrides": [] }, "gridPos": { - "h": 7, + "h": 8, "w": 12, "x": 0, - "y": 4 + "y": 3 }, "id": 4, "options": { @@ -1051,7 +1051,7 @@ "type": "timeseries" } ], - "refresh": "", + "refresh": "auto", "schemaVersion": 39, "tags": [ "registry" @@ -1067,6 +1067,6 @@ "timezone": "", "title": "Laconicd", "uid": "fc7ea92d-8846-474a-89ed-5256c514f8c6", - "version": 25, + "version": 27, "weekStart": "" } -- 2.45.2 From c4b05ac8a06b3e4bcc5d196f8ee0d19665262a88 Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Wed, 10 Apr 2024 14:50:32 +0530 Subject: [PATCH 06/10] Update instructions to add laconicd endpoint to blackbox scrape config --- stack_orchestrator/data/stacks/monitoring/README.md | 1 + stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md | 1 + 2 files changed, 2 insertions(+) diff --git a/stack_orchestrator/data/stacks/monitoring/README.md b/stack_orchestrator/data/stacks/monitoring/README.md index 1c44c34b..1702d59b 100644 --- a/stack_orchestrator/data/stacks/monitoring/README.md +++ b/stack_orchestrator/data/stacks/monitoring/README.md @@ -100,6 +100,7 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --de - targets: - - + - ``` * Postgres (in-stack exporter): diff --git a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md index f7acafc7..ebd67a5b 100644 --- a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md +++ b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md @@ -46,6 +46,7 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers static_configs: - targets: - + - ... - job_name: laconicd static_configs: -- 2.45.2 From 6ee69623a3f454f81705f8dc6504fc61853377c1 Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Wed, 10 Apr 2024 16:03:49 +0530 Subject: [PATCH 07/10] Add alert rule for Ajna watcher --- .../config/monitoring/watcher-alert-rules.yml | 78 +++++++++++++++++++ .../stacks/monitoring/monitoring-watchers.md | 10 +++ 2 files changed, 88 insertions(+) diff --git a/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml b/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml index a190a1fd..c2025029 100644 --- a/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml +++ b/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml @@ -771,3 +771,81 @@ groups: annotations: summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} isPaused: false + + # Ajna + - uid: ajna_diff_external + title: ajna_watcher_head_tracking + condition: condition + data: + - refId: diff + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + disableTextWrap: false + editorMode: code + expr: latest_block_number - on(chain) group_right sync_status_block_number{job="ajna", instance="ajna", kind="latest_indexed"} + fullMetaSearch: false + includeNullMetadata: true + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: diff + useBackend: false + - refId: latest_external + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: latest_block_number{chain="filecoin"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_external + - refId: condition + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: ${diff} >= 16 + intervalMs: 1000 + maxDataPoints: 43200 + refId: condition + type: math + noDataState: Alerting + execErrState: Alerting + for: 15m + annotations: + summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} + isPaused: false diff --git a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md index ebd67a5b..2f057c3c 100644 --- a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md +++ b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md @@ -103,6 +103,16 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers labels: instance: 'merkl_sushiswap' chain: 'filecoin' + + - job_name: ajna + scrape_interval: 20s + metrics_path: /metrics + scheme: http + static_configs: + - targets: ['AJNA_WATCHER_HOST:AJNA_WATCHER_PORT'] + labels: + instance: 'ajna' + chain: 'filecoin' ``` Add scrape config as done above for any additional watcher to add it to the Watchers dashboard. -- 2.45.2 From c696c2608e08ef60deb0db2e14710b6bdec16f45 Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Wed, 10 Apr 2024 17:06:53 +0530 Subject: [PATCH 08/10] Update laconicd fixturenet script to enable prometheus metrics --- .../data/config/fixturenet-laconicd/create-fixturenet.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/stack_orchestrator/data/config/fixturenet-laconicd/create-fixturenet.sh b/stack_orchestrator/data/config/fixturenet-laconicd/create-fixturenet.sh index d444fcad..98ea4dd9 100644 --- a/stack_orchestrator/data/config/fixturenet-laconicd/create-fixturenet.sh +++ b/stack_orchestrator/data/config/fixturenet-laconicd/create-fixturenet.sh @@ -102,6 +102,11 @@ if [ "$1" == "clean" ] || [ ! -d "$HOME/.laconicd/data/blockstore.db" ]; then fi fi + # Enable telemetry (prometheus metrics: http://localhost:1317/metrics?format=prometheus) + sed -i 's/enabled = false/enabled = true/g' $HOME/.laconicd/config/app.toml + sed -i 's/prometheus-retention-time = 0/prometheus-retention-time = 60/g' $HOME/.laconicd/config/app.toml + sed -i 's/prometheus = false/prometheus = true/g' $HOME/.laconicd/config/config.toml + # Allocate genesis accounts (cosmos formatted addresses) laconicd add-genesis-account $KEY 100000000000000000000000000aphoton --keyring-backend $KEYRING -- 2.45.2 From bf9340343c97ca8117f3e76a1e87ef1f91ee84f7 Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Wed, 10 Apr 2024 18:08:05 +0530 Subject: [PATCH 09/10] Use regex to recognize metrics from both old and new laconicd --- .../dashboards/laconicd-dashboard.json | 30 +++++++++---------- .../data/stacks/monitoring/README.md | 4 +-- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/laconicd-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/laconicd-dashboard.json index 84d691a9..0735885c 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/dashboards/laconicd-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/laconicd-dashboard.json @@ -27,7 +27,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "The tendermint consensus latest block height", + "description": "The consensus latest block height", "fieldConfig": { "defaults": { "color": { @@ -77,7 +77,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "tendermint_consensus_latest_block_height", + "expr": "{__name__=~\"^(tendermint|cometbft)_consensus_latest_block_height$\"}", "instant": false, "legendFormat": "__auto", "range": true, @@ -644,14 +644,14 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "tendermint_mempool_size", + "expr": "{__name__=~\"^(tendermint|cometbft)_mempool_size\"}", "instant": false, - "legendFormat": "{{__name__}}", + "legendFormat": "mempool_size", "range": true, "refId": "A" } ], - "title": "tendermint_mempool_size", + "title": "mempool_size", "type": "timeseries" }, { @@ -829,7 +829,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "{__name__ =~ \"^_vulcanize_bond_v1beta1.*_count$\"}", + "expr": "{__name__ =~ \"^_(vulcanize|cerc)_bond_(v1beta1|v1).*_count$\"}", "instant": false, "legendFormat": "{{__name__}}", "range": true, @@ -841,8 +841,8 @@ { "id": "renameByRegex", "options": { - "regex": "_vulcanize_bond_v1beta1_Query_(.*?)_count", - "renamePattern": "$1" + "regex": "_(vulcanize|cerc)_bond_(v1beta1|v1)_Query_(.*?)_count", + "renamePattern": "$3" } } ], @@ -930,7 +930,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "{__name__ =~ \"^_vulcanize_auction_v1beta1.*_count$\"}", + "expr": "{__name__ =~ \"^_(vulcanize|cerc)_auction_(v1beta1|v1).*_count$\"}", "instant": false, "legendFormat": "{{__name__}}", "range": true, @@ -942,8 +942,8 @@ { "id": "renameByRegex", "options": { - "regex": "_vulcanize_auction_v1beta1_Query_(.*?)_count", - "renamePattern": "$1" + "regex": "_(vulcanize|cerc)_auction_(v1beta1|v1)_Query_(.*?)_count", + "renamePattern": "$3" } } ], @@ -1031,7 +1031,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "{__name__ =~ \"^_vulcanize_registry_v1beta1.*_count$\"}", + "expr": "{__name__ =~ \"^_(vulcanize|cerc)_registry_(v1beta1|v1).*_count$\"}", "instant": false, "legendFormat": "{{__name__}}", "range": true, @@ -1043,8 +1043,8 @@ { "id": "renameByRegex", "options": { - "regex": "_vulcanize_registry_v1beta1_Query_(.*?)_count", - "renamePattern": "$1" + "regex": "_(vulcanize|cerc)_registry_(v1beta1|v1)_Query_(.*?)_count", + "renamePattern": "$3" } } ], @@ -1067,6 +1067,6 @@ "timezone": "", "title": "Laconicd", "uid": "fc7ea92d-8846-474a-89ed-5256c514f8c6", - "version": 27, + "version": 1, "weekStart": "" } diff --git a/stack_orchestrator/data/stacks/monitoring/README.md b/stack_orchestrator/data/stacks/monitoring/README.md index 1702d59b..99502902 100644 --- a/stack_orchestrator/data/stacks/monitoring/README.md +++ b/stack_orchestrator/data/stacks/monitoring/README.md @@ -4,7 +4,7 @@ * Comes with the following built-in exporters / dashboards: * Chain Head Exporter - for tracking chain heads given external ETH RPC endpoints * Watchers dashboard - * Laconicd dashboard + * laconicd dashboard * [Prometheus Blackbox](https://grafana.com/grafana/dashboards/7587-prometheus-blackbox-exporter/) - for tracking HTTP endpoints * [NodeJS Application Dashboard](https://grafana.com/grafana/dashboards/11159-nodejs-application-dashboard/) - for default NodeJS metrics * [PostgreSQL Database](https://grafana.com/grafana/dashboards/9628-postgresql-database/) - for monitoring Postgres dbs @@ -118,7 +118,7 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --de ``` * Add database credentials to be used in `auth_modules` in the postgres-exporter config file (`monitoring-deployment/config/monitoring/postgres-exporter.yml`) -* Laconicd: update the `laconicd` job with a Laconicd node's REST endpoint host and port: +* laconicd: update the `laconicd` job with a laconicd node's REST endpoint host and port: ```yml ... -- 2.45.2 From 8d615394a95f31f86ea6e9a1414812235491a6a8 Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Thu, 11 Apr 2024 11:44:39 +0530 Subject: [PATCH 10/10] Handle config update for ostype Darwin --- .../config/fixturenet-laconicd/create-fixturenet.sh | 12 +++++++++--- .../data/config/monitoring/prometheus/prometheus.yml | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/stack_orchestrator/data/config/fixturenet-laconicd/create-fixturenet.sh b/stack_orchestrator/data/config/fixturenet-laconicd/create-fixturenet.sh index 98ea4dd9..de3c6679 100644 --- a/stack_orchestrator/data/config/fixturenet-laconicd/create-fixturenet.sh +++ b/stack_orchestrator/data/config/fixturenet-laconicd/create-fixturenet.sh @@ -103,9 +103,15 @@ if [ "$1" == "clean" ] || [ ! -d "$HOME/.laconicd/data/blockstore.db" ]; then fi # Enable telemetry (prometheus metrics: http://localhost:1317/metrics?format=prometheus) - sed -i 's/enabled = false/enabled = true/g' $HOME/.laconicd/config/app.toml - sed -i 's/prometheus-retention-time = 0/prometheus-retention-time = 60/g' $HOME/.laconicd/config/app.toml - sed -i 's/prometheus = false/prometheus = true/g' $HOME/.laconicd/config/config.toml + if [[ "$OSTYPE" == "darwin"* ]]; then + sed -i '' 's/enabled = false/enabled = true/g' $HOME/.laconicd/config/app.toml + sed -i '' 's/prometheus-retention-time = 0/prometheus-retention-time = 60/g' $HOME/.laconicd/config/app.toml + sed -i '' 's/prometheus = false/prometheus = true/g' $HOME/.laconicd/config/config.toml + else + sed -i 's/enabled = false/enabled = true/g' $HOME/.laconicd/config/app.toml + sed -i 's/prometheus-retention-time = 0/prometheus-retention-time = 60/g' $HOME/.laconicd/config/app.toml + sed -i 's/prometheus = false/prometheus = true/g' $HOME/.laconicd/config/config.toml + fi # Allocate genesis accounts (cosmos formatted addresses) laconicd add-genesis-account $KEY 100000000000000000000000000aphoton --keyring-backend $KEYRING diff --git a/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml b/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml index d95c0373..8229ff53 100644 --- a/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml +++ b/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml @@ -71,6 +71,6 @@ scrape_configs: scrape_interval: 30s static_configs: # Add laconicd REST endpoint target with host and port (1317) - - targets: ['LACONICD_REST_HOST:LACONICD_REST_PORT'] + # - targets: ['example-host:1317'] params: format: ['prometheus'] -- 2.45.2