From ed9c253f61ba78edb80b9646c77c7f51e513a1be Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 19 Sep 2023 20:29:08 -0500 Subject: [PATCH] Simplified, improved default dashboard for mainnet-eth-metrics. (#536) * Simplified, improved default dashboard for mainnet-eth-metrics. * Add missing lighthouse prometheus config. --- .../grafana/etc/dashboards/eth_dashboard.json | 1571 +++++++---------- .../prometheus/etc/prometheus.yml | 7 + 2 files changed, 639 insertions(+), 939 deletions(-) diff --git a/app/data/config/mainnet-eth-metrics/grafana/etc/dashboards/eth_dashboard.json b/app/data/config/mainnet-eth-metrics/grafana/etc/dashboards/eth_dashboard.json index 1946ed37..623c0a56 100644 --- a/app/data/config/mainnet-eth-metrics/grafana/etc/dashboards/eth_dashboard.json +++ b/app/data/config/mainnet-eth-metrics/grafana/etc/dashboards/eth_dashboard.json @@ -24,7 +24,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 1, + "id": 45, "links": [], "liveNow": false, "panels": [ @@ -38,7 +38,7 @@ }, "id": 16, "panels": [], - "title": "Row title", + "title": "Services", "type": "row" }, { @@ -51,47 +51,13 @@ "color": { "mode": "palette-classic" }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, "mappings": [], "thresholds": { - "mode": "absolute", + "mode": "percentage", "steps": [ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] } @@ -100,23 +66,26 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 4, "x": 0, "y": 1 }, - "id": 6, + "id": 12, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "textMode": "auto" }, + "pluginVersion": "10.1.0", "targets": [ { "datasource": { @@ -124,235 +93,9 @@ "uid": "jZUuGao4k" }, "editorMode": "code", - "expr": "rate(nginx_http_requests_total{instance=\"keycloak-nginx-prometheus-exporter:9113\"}[$__rate_interval])", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "nginx Requests/sec", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "builder", - "expr": "rate(ipld_eth_server_http_count{instance=\"ipld-eth-server:8090\"}[$__rate_interval])", - "legendFormat": "{{method}}", - "range": true, - "refId": "A" - } - ], - "title": "ipld-eth-server Requests/sec", - "transformations": [ - { - "id": "seriesToColumns", - "options": { - "byField": "Time" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "{instance=\"ipld-eth-server:8090\", job=\"ipld-eth-server\"}": true - }, - "indexByName": {}, - "renameByName": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 20, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_blockNumber_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "expr": "chain_head_block", "hide": false, - "legendFormat": "eth_blockNumber", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_call_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_call", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_chainId_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_chainId", + "legendFormat": "geth ({{instance}})", "range": true, "refId": "C" }, @@ -362,375 +105,17 @@ "uid": "jZUuGao4k" }, "editorMode": "code", - "expr": "rate(rpc_duration_eth_estimateGas_success_count{instance=\"${node}:6060\"}[$__rate_interval])", + "expr": "beacon_head_slot", "hide": false, - "legendFormat": "eth_estimateGas", + "legendFormat": "lighthouse ({{instance}})", "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_gasPrice_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_gasPrice", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_getBalance_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_getBalance", - "range": true, - "refId": "F" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_getBlockByHash_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_getBlockByHash", - "range": true, - "refId": "G" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_getBlockByNumber_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_getBlockByNumber", - "range": true, - "refId": "H" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_getBlockReceipts_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_getBlockReceipts", - "range": true, - "refId": "I" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_getBlockTransactionCountByHash_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_getBlockTransactionCountByHash", - "range": true, - "refId": "J" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_getBlockTransactionCountByNumber_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_getBlockTransactionCountByNumber", - "range": true, - "refId": "K" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_getCode_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_getCode", - "range": true, - "refId": "L" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_getFilterChanges_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_getFilterChanges", - "range": true, - "refId": "M" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_getFilterLogs_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_getFilterLogs", - "range": true, - "refId": "N" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_getLogs_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_getLogs", - "range": true, - "refId": "O" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_getStorageAt_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_getStorageAt", - "range": true, - "refId": "P" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_getTransactionByBlockHashAndIndex_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_getTransactionByBlockHashAndIndex", - "range": true, - "refId": "Q" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_getTransactionByBlockNumberAndIndex_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_getTransactionByBlockNumberAndIndex", - "range": true, - "refId": "R" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_getTransactionByHash_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_getTransactionByHash", - "range": true, - "refId": "S" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_getTransactionCount_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_getTransactionCount", - "range": true, - "refId": "T" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_getTransactionReceipt_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_getTransactionReceipt", - "range": true, - "refId": "U" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_getUncleCountByBlockHash_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_getUncleCountByBlockHash", - "range": true, - "refId": "V" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_getUncleCountByBlockNumber_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_getUncleCountByBlockNumber", - "range": true, - "refId": "W" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_mining_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_mining_success", - "range": true, - "refId": "X" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_newBlockFilter_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_newBlockFilter", - "range": true, - "refId": "Y" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_newFilter_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_newFilter", - "range": true, - "refId": "Z" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_newPendingTransactionFilter_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_newPendingTransactionFilter", - "range": true, - "refId": "AA" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_sendRawTransaction_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_sendRawTransaction", - "range": true, - "refId": "AB" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_signTransaction_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_signTransaction", - "range": true, - "refId": "AC" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_subscribe_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_subscribe", - "range": true, - "refId": "AD" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_syncing_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_syncing", - "range": true, - "refId": "AE" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_eth_unsubscribe_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "eth_unsubscribe", - "range": true, - "refId": "AF" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_statediff_writeStateDiffFor_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "statediff_writeStateDiffFor", - "range": true, - "refId": "AG" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "editorMode": "code", - "expr": "rate(rpc_duration_statediff_writeStateDiffAt_success_count{instance=\"${node}:6060\"}[$__rate_interval])", - "hide": false, - "legendFormat": "statediff_writeStateDiffAt", - "range": true, - "refId": "AH" + "refId": "A" } ], - "title": "geth API Requests/sec", - "type": "timeseries" + "title": "Head Positions", + "transformations": [ + ], + "type": "stat" }, { "datasource": { @@ -756,6 +141,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -772,7 +158,9 @@ "mode": "off" } }, + "decimals": 2, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -781,15 +169,16 @@ "value": null } ] - } + }, + "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 9 + "w": 8, + "x": 4, + "y": 1 }, "id": 2, "options": { @@ -813,11 +202,11 @@ }, "editorMode": "code", "exemplar": false, - "expr": "rate(keycloak_logins{instance=\"keycloak:8080\"}[90s]) * 60", + "expr": "rate(keycloak_logins[$__rate_interval]) * 60", "format": "time_series", "instant": false, - "interval": "2m", - "legendFormat": "{{client_id}}", + "interval": "5m", + "legendFormat": "{{client_id}} ({{instance}})", "range": true, "refId": "A" } @@ -839,7 +228,11 @@ "13761bf2-2c0e-42fd-acc8-09a677ab179e": true, "79a56c4e-0bfb-4ef0-87d4-244c4ace9b4b": true, "9c41e289-92f0-456f-bd65-8f73024a9e45": true, - "c9718dd1-06d6-4f98-a821-9adbeb4b663b": true + "VVQkSrRksVEKAqUmiNh4aMAVD6BcviyYV4b5LpnCrtUkRwTMoE (jackal.lan.vdb.to:8080)": true, + "c9718dd1-06d6-4f98-a821-9adbeb4b663b": true, + "nYByi9xhWdDHPpStoPLP3hjeJUzyQou8dsjK8aSipOglB3yx75 (jackal.lan.vdb.to:8080)": true, + "oA7XX21gTNMFRJJJdbDzjBVERbjrWamzIGhwHUAeMLrd1sXCWx (jackal.lan.vdb.to:8080)": true, + "oJf5jLHEqRbZg7BvSboYX1DM2T25K7XS4jgwF8NNzl3AHNWHm2 (jackal.lan.vdb.to:8080)": true }, "indexByName": {}, "renameByName": { @@ -867,13 +260,49 @@ "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 28, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, "mappings": [], + "min": 0, "thresholds": { - "mode": "percentage", + "mode": "absolute", "steps": [ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] } @@ -881,27 +310,24 @@ "overrides": [] }, "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 17 + "h": 8, + "w": 12, + "x": 12, + "y": 1 }, - "id": 12, + "id": 20, "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "textMode": "auto" + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "pluginVersion": "9.3.2", "targets": [ { "datasource": { @@ -909,9 +335,240 @@ "uid": "jZUuGao4k" }, "editorMode": "code", - "expr": "chain_head_block{instance=\"${node}:6060\"}", + "expr": "rate(rpc_duration_all_count[$__rate_interval])", "hide": false, - "legendFormat": "geth", + "instant": false, + "legendFormat": "Requests ({{instance}})", + "range": true, + "refId": "A" + } + ], + "title": "geth API Requests/sec", + "transformations": [ + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 67, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "p2p_peers{job=\"geth\"}", + "legendFormat": "geth ({{instance}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "network_inbound_peers{job=\"lighthouse-beacon\"}", + "hide": false, + "legendFormat": "lighthouse ({{instance}})", + "range": true, + "refId": "B" + } + ], + "title": "P2P Peers", + "transformations": [ + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 32, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(system_disk_writebytes[$__rate_interval])", + "interval": "", + "legendFormat": "Process Write ({{instance}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(system_disk_readbytes[$__rate_interval])", + "hide": false, + "legendFormat": "Process Read ({{instance}})", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(eth_db_chaindata_disk_read[$__rate_interval])", + "hide": false, + "legendFormat": "ethdb Read ({{instance}})", "range": true, "refId": "C" }, @@ -921,14 +578,267 @@ "uid": "jZUuGao4k" }, "editorMode": "code", - "expr": "statediff_service_last_statediff_height{instance=\"${node}:6060\"}", - "legendFormat": "statediff", + "expr": "rate(eth_db_chaindata_disk_write[$__rate_interval])", + "hide": false, + "legendFormat": "ethdb Write ({{instance}})", "range": true, - "refId": "A" + "refId": "D" } ], - "title": "Block Positions", - "type": "stat" + "title": "geth Disk Bytes/sec", + "transformations": [ + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(system_disk_readcount[$__rate_interval])", + "legendFormat": "Process Read ({{instance}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(system_disk_writecount[$__rate_interval])", + "hide": false, + "legendFormat": "Process Write ({{instance}})", + "range": true, + "refId": "B" + } + ], + "title": "geth Disk Operations/sec", + "transformations": [ + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 61, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(eth_db_chaindata_db_get_time_count[$__rate_interval])", + "legendFormat": "get ({{instance}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(eth_db_chaindata_db_put_time_count[$__rate_interval])", + "hide": false, + "legendFormat": "put ({{instance}})", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(eth_db_chaindata_db_has_time_count[$__rate_interval])", + "hide": false, + "legendFormat": "has ({{instance}})", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(eth_db_chaindata_db_delete_time_count[$__rate_interval])", + "hide": false, + "legendFormat": "delete ({{instance}})", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(eth_db_chaindata_db_batch_write_time_count[$__rate_interval])", + "hide": false, + "legendFormat": "batch_write ({{instance}})", + "range": true, + "refId": "E" + } + ], + "title": "geth LevelDB Operations/sec", + "transformations": [ + ], + "type": "timeseries" }, { "datasource": { @@ -939,7 +849,7 @@ "fieldConfig": { "defaults": { "color": { - "mode": "continuous-GrYlRd" + "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, @@ -955,6 +865,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -984,132 +895,18 @@ "value": 80 } ] - } - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "Distance" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 17 - }, - "id": 22, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" }, - "editorMode": "code", - "expr": "clamp_min(chain_head_block{instance=\"${node}:6060\"} - on() statediff_service_last_statediff_height{instance=\"${node}:6060\"}, 0)", - "interval": "1m", - "legendFormat": "Distance", - "range": true, - "refId": "A" - } - ], - "title": "Block Distance (geth to statediff)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" + "unit": "percent" }, "overrides": [] }, "gridPos": { - "h": 7, + "h": 8, "w": 12, - "x": 12, - "y": 17 + "x": 0, + "y": 25 }, - "id": 10, + "id": 63, "options": { "legend": { "calcs": [], @@ -1122,7 +919,6 @@ "sort": "none" } }, - "pluginVersion": "9.0.1", "targets": [ { "datasource": { @@ -1130,12 +926,8 @@ "uid": "jZUuGao4k" }, "editorMode": "code", - "exemplar": false, - "expr": "increase(statediff_service_succeeded{instance=\"${node}:6060\"}[1m])", - "format": "time_series", - "instant": false, - "interval": "1m", - "legendFormat": "Succeeded", + "expr": "rate(eth_db_chaindata_db_get_time_total[$__rate_interval]) / 10000000", + "legendFormat": "get ({{instance}})", "range": true, "refId": "A" }, @@ -1145,10 +937,9 @@ "uid": "jZUuGao4k" }, "editorMode": "code", - "expr": "increase(statediff_service_failed{instance=\"${node}:6060\"}[1m])", + "expr": "rate(eth_db_chaindata_db_put_time_total[$__rate_interval]) / 10000000", "hide": false, - "interval": "1m", - "legendFormat": "Failed", + "legendFormat": "put ({{instance}})", "range": true, "refId": "B" }, @@ -1158,119 +949,40 @@ "uid": "jZUuGao4k" }, "editorMode": "code", - "expr": "statediff_service_underway{instance=\"${node}:6060\"}", + "expr": "rate(eth_db_chaindata_db_has_time_total[$__rate_interval]) / 10000000", "hide": false, - "interval": "1s", - "legendFormat": "Underway", + "legendFormat": "has ({{instance}})", "range": true, "refId": "C" - } - ], - "title": "Statediff Operations", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "jZUuGao4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-GrYlRd" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "#EAB839", - "value": 50 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 14, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.0.1", - "targets": [ { "datasource": { "type": "prometheus", "uid": "jZUuGao4k" }, "editorMode": "code", - "exemplar": false, - "expr": "rate(statediff_service_total_processing_time{instance=\"${node}:6060\"}[$__rate_interval])", - "instant": false, - "interval": "1m", - "legendFormat": "__auto", + "expr": "rate(eth_db_chaindata_db_delete_time_total[$__rate_interval]) / 10000000", + "hide": false, + "legendFormat": "delete ({{instance}})", "range": true, - "refId": "A" + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jZUuGao4k" + }, + "editorMode": "code", + "expr": "rate(eth_db_chaindata_db_batch_write_time_total[$__rate_interval]) / 10000000", + "hide": false, + "legendFormat": "batch_write ({{instance}})", + "range": true, + "refId": "E" } ], - "title": "Statediff Duty Cycle", + "title": "geth LevelDB Utilization", + "transformations": [ + ], "type": "timeseries" }, { @@ -1297,6 +1009,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1314,6 +1027,7 @@ } }, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -1334,9 +1048,9 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 25 }, - "id": 4, + "id": 65, "options": { "legend": { "calcs": [], @@ -1356,54 +1070,33 @@ "uid": "jZUuGao4k" }, "editorMode": "code", - "expr": "rate(statediff_service_api_requests{instance=\"${node}:6060\"}[$__rate_interval])", - "legendFormat": "__auto", + "expr": "rate(eth_db_chaindata_db_batch_item_count[$__rate_interval])", + "legendFormat": "batch_items ({{instance}})", "range": true, "refId": "A" } ], - "title": "Statediff API Requests/sec", + "title": "geth LevelDB Batch Items Written/sec", + "transformations": [ + ], "type": "timeseries" } ], - "refresh": "5s", - "schemaVersion": 37, + "refresh": "30s", + "schemaVersion": 38, "style": "dark", "tags": [], "templating": { - "list": [ - { - "current": { - "selected": true, - "text": "mainnet-eth-geth-1", - "value": "mainnet-eth-geth-1" - }, - "hide": 0, - "includeAll": false, - "multi": false, - "name": "node", - "options": [ - { - "selected": true, - "text": "mainnet-eth-geth-1", - "value": "mainnet-eth-geth-1" - } - ], - "query": "mainnet-eth-geth-1", - "queryValue": "", - "skipUrlSync": false, - "type": "custom" - } - ] + "list": [] }, "time": { - "from": "now-15m", + "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "", - "title": "cerc-io overview", - "uid": "nT9VeZoVk", - "version": 2, + "title": "eth API Simple", + "uid": "b4ba9b2e-965d-4c71-8735-78b6af93156e", + "version": 4, "weekStart": "" } diff --git a/app/data/config/mainnet-eth-metrics/prometheus/etc/prometheus.yml b/app/data/config/mainnet-eth-metrics/prometheus/etc/prometheus.yml index e9f23e89..4cc74fc0 100644 --- a/app/data/config/mainnet-eth-metrics/prometheus/etc/prometheus.yml +++ b/app/data/config/mainnet-eth-metrics/prometheus/etc/prometheus.yml @@ -10,6 +10,13 @@ scrape_configs: static_configs: - targets: ['mainnet-eth-geth-1:6060'] + # lighthouse + - job_name: 'lighthouse' + metrics_path: /metrics + scheme: http + static_configs: + - targets: ['mainnet-eth-lighthouse-1:5054'] + # keycloak - job_name: 'keycloak' scrape_interval: 5s