From 27a14737f89e999858be19b445f03f8435484646 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Thu, 14 Dec 2023 09:49:21 -0600 Subject: [PATCH 01/15] Make the container tag based on the deployment path. (#688) --- .../deploy/webapp/deploy_webapp_from_registry.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py index ad439812..02731992 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -1,5 +1,4 @@ # Copyright ©2023 Vulcanize - # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -13,6 +12,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import hashlib import json import os import shlex @@ -51,7 +51,6 @@ def process_app_deployment_request( raise Exception("Only unqualified hostnames allowed at this time.") fqdn = f"{requested_name}.{dns_suffix}" - container_tag = "%s:local" % app.attributes.name.replace("@", "") # 3. check ownership of existing dnsrecord vs this request # TODO: Support foreign DNS @@ -94,13 +93,14 @@ def process_app_deployment_request( deployment_record = laconic.get_record(app_deployment_crn) deployment_dir = os.path.join(deployment_parent_dir, fqdn) deployment_config_file = os.path.join(deployment_dir, "config.env") + deployment_container_tag = "laconic-webapp/%s:local" % hashlib.md5(deployment_dir.encode()).hexdigest() # b. check for deployment directory (create if necessary) if not os.path.exists(deployment_dir): if deployment_record: - raise ("Deployment record %s exists, but not deployment dir %s. Please remove name." % - (app_deployment_crn, deployment_dir)) + raise Exception("Deployment record %s exists, but not deployment dir %s. Please remove name." % + (app_deployment_crn, deployment_dir)) print("deploy_webapp", deployment_dir) - deploy_webapp.create_deployment(ctx, deployment_dir, container_tag, + deploy_webapp.create_deployment(ctx, deployment_dir, deployment_container_tag, f"https://{fqdn}", kube_config, image_registry, env_filename) elif env_filename: shutil.copyfile(env_filename, deployment_config_file) @@ -108,7 +108,7 @@ def process_app_deployment_request( needs_k8s_deploy = False # 6. build container (if needed) if not deployment_record or deployment_record.attributes.application != app.id: - build_container_image(app, container_tag) + build_container_image(app, deployment_container_tag) push_container_image(deployment_dir) needs_k8s_deploy = True @@ -262,7 +262,7 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_ record_namespace_deployments, record_namespace_dns, dns_suffix, - deployment_parent_dir, + os.path.abspath(deployment_parent_dir), kube_config, image_registry ) -- 2.45.2 From 61f288450594a833b54bdb88d96fb8230829f70b Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Thu, 14 Dec 2023 17:46:03 -0600 Subject: [PATCH 02/15] Reduce base image size (first round of improvements) (#690) --- .../data/container-build/cerc-nextjs-base/Dockerfile | 4 ++-- .../container-build/cerc-nextjs-base/scripts/build-app.sh | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile index d3ff3f1b..8949c4e9 100644 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile @@ -1,6 +1,6 @@ # Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile # [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster -ARG VARIANT=20-bullseye +ARG VARIANT=20-bullseye-slim FROM node:${VARIANT} ARG USERNAME=node @@ -30,7 +30,7 @@ RUN \ # [Optional] Uncomment this section to install additional OS packages. RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get -y install --no-install-recommends jq gettext-base moreutils + && apt-get -y install --no-install-recommends jq gettext-base # [Optional] Uncomment if you want to install more global node modules # RUN su node -c "npm install -g " diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh index ef6244cf..759ba255 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh @@ -104,7 +104,8 @@ CUR_NEXT_VERSION="`jq -r '.dependencies.next' package.json`" if [ "$CERC_NEXT_VERSION" != "keep" ] && [ "$CUR_NEXT_VERSION" != "$CERC_NEXT_VERSION" ]; then echo "Changing 'next' version specifier from '$CUR_NEXT_VERSION' to '$CERC_NEXT_VERSION' (set with '--extra-build-args \"--build-arg CERC_NEXT_VERSION=$CERC_NEXT_VERSION\"')" - cat package.json | jq ".dependencies.next = \"$CERC_NEXT_VERSION\"" | sponge package.json + cat package.json | jq ".dependencies.next = \"$CERC_NEXT_VERSION\"" > package.json.$$ + mv package.json.$$ package.json fi $CERC_BUILD_TOOL install || exit 1 @@ -128,7 +129,8 @@ to use for the build with: ############################################################################### EOF - cat package.json | jq ".dependencies.next = \"^$CERC_MIN_NEXTVER\"" | sponge package.json + cat package.json | jq ".dependencies.next = \"^$CERC_MIN_NEXTVER\"" > package.json.$$ + mv package.json.$$ package.json $CERC_BUILD_TOOL install || exit 1 fi -- 2.45.2 From 997496b8a52057ecfac1f31b77eb924565c8e2b3 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Thu, 14 Dec 2023 19:47:30 -0600 Subject: [PATCH 03/15] Update script for new nextjs build output. (#691) --- .../cerc-nextjs-base/scripts/start-serving-app.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh index bf35bcdb..3692f2f6 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh @@ -42,7 +42,7 @@ if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then while [ $count -lt $CERC_MAX_GENERATE_TIME ] && [ "$generate_done" == "false" ]; do sleep 1 count=$((count + 1)) - grep 'rendered as static HTML' gen.out > /dev/null + grep 'rendered as static' gen.out > /dev/null if [ $? -eq 0 ]; then generate_done="true" fi -- 2.45.2 From d9ecd65522f0b581c8418a074bd72e74c9bacc1f Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Fri, 15 Dec 2023 11:04:12 +0530 Subject: [PATCH 04/15] Include retry jobs and update default refresh intervals --- .../monitoring/grafana/watcher-dashboard-template.json | 6 +++--- .../azimuth/azimuth-watcher-dashboard.json | 6 +++--- .../azimuth/censures-watcher-dashboard.json | 6 +++--- .../azimuth/claims-watcher-dashboard.json | 6 +++--- .../azimuth/conditional-star-release-watcher-dashboard.json | 6 +++--- .../delegated-sending-watcher-watcher-dashboard.json | 6 +++--- .../azimuth/ecliptic-watcher-dashboard.json | 6 +++--- .../azimuth/linear-star-release-watcher-dashboard.json | 6 +++--- .../watcher-dashboards/azimuth/polls-watcher-dashboard.json | 6 +++--- .../sushi/merkl-sushiswap-watcher-dashboard.json | 6 +++--- .../sushi/sushiswap-watcher-dashboard.json | 6 +++--- 11 files changed, 33 insertions(+), 33 deletions(-) diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json index 03144776..640b7a16 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json @@ -749,7 +749,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "expr": "pgboss_jobs_total{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -851,7 +851,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "expr": "pgboss_jobs_total{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -866,7 +866,7 @@ "type": "timeseries" } ], - "refresh": "5s", + "refresh": "10s", "schemaVersion": 38, "tags": [ "REPLACE_WITH_WATCHER_CHAIN", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json index 6f924b12..42e0855c 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json @@ -749,7 +749,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"azimuth\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"azimuth\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -851,7 +851,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"azimuth\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"azimuth\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -866,7 +866,7 @@ "type": "timeseries" } ], - "refresh": "5s", + "refresh": "10s", "schemaVersion": 38, "tags": [ "ethereum", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json index cbeb07ae..14bf1be5 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json @@ -749,7 +749,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"censures\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"censures\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -851,7 +851,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"censures\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"censures\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -866,7 +866,7 @@ "type": "timeseries" } ], - "refresh": "5s", + "refresh": "10s", "schemaVersion": 38, "tags": [ "ethereum", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json index 29f18fe1..e738ba78 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json @@ -749,7 +749,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"claims\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"claims\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -851,7 +851,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"claims\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"claims\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -866,7 +866,7 @@ "type": "timeseries" } ], - "refresh": "5s", + "refresh": "10s", "schemaVersion": 38, "tags": [ "ethereum", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json index 28ec488d..f089b41c 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json @@ -749,7 +749,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"conditional_star_release\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"conditional_star_release\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -851,7 +851,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"conditional_star_release\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"conditional_star_release\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -866,7 +866,7 @@ "type": "timeseries" } ], - "refresh": "5s", + "refresh": "10s", "schemaVersion": 38, "tags": [ "ethereum", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json index b7411c45..015d5243 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json @@ -749,7 +749,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"delegated_sending_watcher\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"delegated_sending_watcher\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -851,7 +851,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"delegated_sending_watcher\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"delegated_sending_watcher\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -866,7 +866,7 @@ "type": "timeseries" } ], - "refresh": "5s", + "refresh": "10s", "schemaVersion": 38, "tags": [ "ethereum", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json index 3ee7d151..6aca6504 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json @@ -749,7 +749,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"ecliptic\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"ecliptic\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -851,7 +851,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"ecliptic\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"ecliptic\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -866,7 +866,7 @@ "type": "timeseries" } ], - "refresh": "5s", + "refresh": "10s", "schemaVersion": 38, "tags": [ "ethereum", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json index be6ef9ec..eb8e62de 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json @@ -749,7 +749,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"linear_star_release\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"linear_star_release\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -851,7 +851,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"linear_star_release\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"linear_star_release\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -866,7 +866,7 @@ "type": "timeseries" } ], - "refresh": "5s", + "refresh": "10s", "schemaVersion": 38, "tags": [ "ethereum", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json index 01910d9a..fe894c5d 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json @@ -749,7 +749,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"polls\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"polls\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -851,7 +851,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"polls\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"polls\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -866,7 +866,7 @@ "type": "timeseries" } ], - "refresh": "5s", + "refresh": "10s", "schemaVersion": 38, "tags": [ "ethereum", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json index f41b3298..f76815e7 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json @@ -749,7 +749,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"sushi\", instance=\"merkl_sushiswap\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "expr": "pgboss_jobs_total{job=\"sushi\", instance=\"merkl_sushiswap\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -851,7 +851,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"sushi\", instance=\"merkl_sushiswap\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "expr": "pgboss_jobs_total{job=\"sushi\", instance=\"merkl_sushiswap\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -1158,7 +1158,7 @@ "type": "timeseries" } ], - "refresh": "5s", + "refresh": "20s", "schemaVersion": 38, "tags": [ "filecoin", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json index e602000f..bcaa97f6 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json @@ -749,7 +749,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"sushi\", instance=\"sushiswap\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "expr": "pgboss_jobs_total{job=\"sushi\", instance=\"sushiswap\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -851,7 +851,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"sushi\", instance=\"sushiswap\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "expr": "pgboss_jobs_total{job=\"sushi\", instance=\"sushiswap\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -1158,7 +1158,7 @@ "type": "timeseries" } ], - "refresh": "5s", + "refresh": "20s", "schemaVersion": 38, "tags": [ "filecoin", -- 2.45.2 From 96d4d1e2bf43dbbd35e0739e1cf20a53b79122ae Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Fri, 15 Dec 2023 11:19:25 +0530 Subject: [PATCH 05/15] Add prometheus blackbox exporter and it's dashboard --- .../compose/docker-compose-prom-server.yml | 10 + .../data/config/monitoring/blackbox.yml | 7 + .../prometheus-blackbox-exporter.json | 1097 +++++++++++++++++ .../monitoring/prometheus/prometheus.yml | 21 + .../stacks/monitoring/monitoring-watchers.md | 5 + 5 files changed, 1140 insertions(+) create mode 100644 stack_orchestrator/data/config/monitoring/blackbox.yml create mode 100644 stack_orchestrator/data/config/monitoring/grafana/dashboards/prometheus-blackbox-exporter.json diff --git a/stack_orchestrator/data/compose/docker-compose-prom-server.yml b/stack_orchestrator/data/compose/docker-compose-prom-server.yml index 26a61e88..e3faecc8 100644 --- a/stack_orchestrator/data/compose/docker-compose-prom-server.yml +++ b/stack_orchestrator/data/compose/docker-compose-prom-server.yml @@ -18,5 +18,15 @@ services: extra_hosts: - "host.docker.internal:host-gateway" + blackbox: + image: prom/blackbox-exporter:latest + restart: always + volumes: + - ../config/monitoring/blackbox.yml:/etc/blackbox_exporter/config.yml + ports: + - '9115' + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: prometheus_data: diff --git a/stack_orchestrator/data/config/monitoring/blackbox.yml b/stack_orchestrator/data/config/monitoring/blackbox.yml new file mode 100644 index 00000000..4a9d72eb --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/blackbox.yml @@ -0,0 +1,7 @@ +modules: + http_2xx: + prober: http + timeout: 5s + http: + valid_status_codes: [] #default to 2xx + method: GET diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/prometheus-blackbox-exporter.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/prometheus-blackbox-exporter.json new file mode 100644 index 00000000..638c4d5b --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/prometheus-blackbox-exporter.json @@ -0,0 +1,1097 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Prometheus Blackbox Exporter Overview", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 7587, + "graphTooltip": 0, + "id": 29, + "links": [], + "liveNow": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 138, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "probe_duration_seconds{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Global Probe Duration", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 15, + "panels": [], + "repeat": "target", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "$target status", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "text": "DOWN" + }, + "1": { + "text": "UP" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "#299c46", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 9 + }, + "id": 2, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "probe_success{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Status", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 10, + "x": 4, + "y": 9 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "probe_http_duration_seconds{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ phase }}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "HTTP Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 10, + "x": 14, + "y": 9 + }, + "hiddenSeries": false, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "probe_duration_seconds{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "seconds", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Probe Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "0": { + "text": "N/A" + }, + "1": { + "text": "YES" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 201 + }, + { + "color": "#d44a3a", + "value": 399 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 11 + }, + "id": 20, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "probe_http_status_code{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "HTTP Status Code", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 13 + }, + "id": 27, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "probe_http_version{instance=~\"$target\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "HTTP Version", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "text": "NO" + }, + "1": { + "text": "YES" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 + }, + { + "color": "#299c46", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 15 + }, + "id": 18, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "probe_http_ssl{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "SSL", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "0": { + "text": "NO" + }, + "1": { + "text": "YES" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 + }, + { + "color": "#299c46", + "value": 1209600 + } + ] + }, + "unit": "dtdurations" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 10, + "x": 4, + "y": 15 + }, + "id": 19, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "probe_ssl_earliest_cert_expiry{instance=~\"$target\"} - time()", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "SSL Expiry", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 5, + "x": 14, + "y": 15 + }, + "id": 23, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "avg(probe_duration_seconds{instance=~\"$target\"})", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Average Probe Duration", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 5, + "x": 19, + "y": 15 + }, + "id": 24, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "avg(probe_dns_lookup_time_seconds{instance=~\"$target\"})", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Average DNS Lookup", + "type": "stat" + } + ], + "refresh": "10s", + "schemaVersion": 38, + "tags": [ + "blackbox", + "prometheus" + ], + "templating": { + "list": [ + { + "auto": true, + "auto_count": 10, + "auto_min": "10s", + "current": { + "selected": false, + "text": "10s", + "value": "10s" + }, + "hide": 0, + "label": "Interval", + "name": "interval", + "options": [ + { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "5s", + "value": "5s" + }, + { + "selected": true, + "text": "10s", + "value": "10s" + }, + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "5s,10s,30s,1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "target", + "options": [], + "query": "label_values(probe_success, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Prometheus Blackbox Exporter", + "uid": "xtkCtBkiz", + "version": 2, + "weekStart": "" +} diff --git a/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml b/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml index e8e7b8b8..6b2e380e 100644 --- a/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml +++ b/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml @@ -10,3 +10,24 @@ scrape_configs: - job_name: prometheus static_configs: - targets: ['localhost:9090'] + + - job_name: 'blackbox' + metrics_path: /probe + params: + module: [http_2xx] + static_configs: + # Add URLs to be monitored below + - targets: + # - https://github.com + relabel_configs: + - source_labels: [__address__] + regex: (.*)(:80)? + target_label: __param_target + - source_labels: [__param_target] + regex: (.*) + target_label: instance + replacement: ${1} + - source_labels: [] + regex: .* + target_label: __address__ + replacement: blackbox:9115 diff --git a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md index f49bdcda..711b3c37 100644 --- a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md +++ b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md @@ -40,6 +40,11 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-watchers-spec Add the following scrape configs to prometheus config file (`monitoring-watchers-deployment/config/monitoring/prometheus/prometheus.yml`) in the deployment folder: ```yml + - job_name: 'blackbox' + ... + static_configs: + - targets: + - ... - job_name: azimuth metrics_path: /metrics -- 2.45.2 From ed50870e49bcef318cae7ac75a6766dd7c47e948 Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Fri, 15 Dec 2023 13:52:49 +0530 Subject: [PATCH 06/15] Add NodeJS application dashboard --- .../dashboards/nodejs-app-dashboard.json | 943 ++++++++++++++++++ .../stacks/monitoring/monitoring-watchers.md | 1 + 2 files changed, 944 insertions(+) create mode 100644 stack_orchestrator/data/config/monitoring/grafana/dashboards/nodejs-app-dashboard.json diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/nodejs-app-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/nodejs-app-dashboard.json new file mode 100644 index 00000000..5a95578a --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/nodejs-app-dashboard.json @@ -0,0 +1,943 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "node.js prometheus client basic metrics", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 11159, + "graphTooltip": 0, + "id": 15, + "links": [], + "liveNow": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 10, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(process_cpu_user_seconds_total{instance=~\"$instance\"}[2m]) * 100", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "User CPU - {{instance}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(process_cpu_system_seconds_total{instance=~\"$instance\"}[2m]) * 100", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Sys CPU - {{instance}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Process CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 9, + "x": 10, + "y": 0 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_eventloop_lag_seconds{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Event Loop Lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 19, + "y": 0 + }, + "id": 2, + "interval": "", + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum(nodejs_version_info{instance=~\"$instance\"}) by (version)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{version}}", + "refId": "A" + } + ], + "title": "Node.js Version", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#F2495C", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 19, + "y": 3 + }, + "id": 4, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum(changes(process_start_time_seconds{instance=~\"$instance\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Process Restart Times", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 16, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "process_resident_memory_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Process Memory - {{instance}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_heap_size_total_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Heap Total - {{instance}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_heap_size_used_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Heap Used - {{instance}}", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_external_memory_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "External Memory - {{instance}}", + "refId": "D" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Process Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_active_handles_total{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Active Handler - {{instance}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_active_requests_total{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Active Request - {{instance}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Active Handlers/Requests Total", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_heap_space_size_total_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Heap Total - {{instance}} - {{space}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Heap Total Detail", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 14 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_heap_space_size_used_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Heap Used - {{instance}} - {{space}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Heap Used Detail", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 14 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_heap_space_size_available_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Heap Used - {{instance}} - {{space}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Heap Available Detail", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": "10s", + "schemaVersion": 38, + "tags": [ + "nodejs" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(nodejs_version_info, instance)", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [], + "query": "label_values(nodejs_version_info, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "NodeJS Application Dashboard", + "uid": "PTSqcpJWk", + "version": 3, + "weekStart": "" +} diff --git a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md index 711b3c37..a209d6d7 100644 --- a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md +++ b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md @@ -40,6 +40,7 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-watchers-spec Add the following scrape configs to prometheus config file (`monitoring-watchers-deployment/config/monitoring/prometheus/prometheus.yml`) in the deployment folder: ```yml + ... - job_name: 'blackbox' ... static_configs: -- 2.45.2 From d8357df3458b5783e70381f0a968819b6a012bbd Mon Sep 17 00:00:00 2001 From: David Boreham Date: Fri, 15 Dec 2023 14:27:45 -0700 Subject: [PATCH 07/15] Add image pull secret to pods (#692) --- stack_orchestrator/deploy/k8s/cluster_info.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 24fe15a0..7718c777 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -200,9 +200,10 @@ class ClusterInfo: ) containers.append(container) volumes = volumes_for_pod_files(self.parsed_pod_yaml_map) + image_pull_secrets = [client.V1LocalObjectReference(name="laconic-registry")] template = client.V1PodTemplateSpec( metadata=client.V1ObjectMeta(labels={"app": self.app_name}), - spec=client.V1PodSpec(containers=containers, volumes=volumes), + spec=client.V1PodSpec(containers=containers, image_pull_secrets=image_pull_secrets, volumes=volumes), ) spec = client.V1DeploymentSpec( replicas=1, template=template, selector={ -- 2.45.2 From 743dc6ea839405cfa68492f736e52c6396dd9d74 Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Mon, 18 Dec 2023 14:56:29 +0530 Subject: [PATCH 08/15] Allow UI updates --- .../monitoring/grafana/provisioning/dashboards/dashboards.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/stack_orchestrator/data/config/monitoring/grafana/provisioning/dashboards/dashboards.yml b/stack_orchestrator/data/config/monitoring/grafana/provisioning/dashboards/dashboards.yml index 4705688e..9166cbff 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/provisioning/dashboards/dashboards.yml +++ b/stack_orchestrator/data/config/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -4,6 +4,7 @@ providers: - name: dashboards type: file updateIntervalSeconds: 10 + allowUiUpdates: true options: path: /etc/grafana/dashboards foldersFromFilesStructure: true -- 2.45.2 From d4e42d2aec76b55f1d6cd7df70144a99c67216bd Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Mon, 18 Dec 2023 15:12:20 +0530 Subject: [PATCH 09/15] Update watcher dashboards for upstream and external chain heads --- .../compose/docker-compose-prom-server.yml | 15 +++++++ .../grafana/watcher-dashboard-template.json | 40 +++++++++++++++++- .../azimuth/azimuth-watcher-dashboard.json | 40 +++++++++++++++++- .../azimuth/censures-watcher-dashboard.json | 40 +++++++++++++++++- .../azimuth/claims-watcher-dashboard.json | 40 +++++++++++++++++- ...tional-star-release-watcher-dashboard.json | 40 +++++++++++++++++- ...ted-sending-watcher-watcher-dashboard.json | 40 +++++++++++++++++- .../azimuth/ecliptic-watcher-dashboard.json | 40 +++++++++++++++++- ...linear-star-release-watcher-dashboard.json | 40 +++++++++++++++++- .../azimuth/polls-watcher-dashboard.json | 40 +++++++++++++++++- .../merkl-sushiswap-watcher-dashboard.json | 40 +++++++++++++++++- .../sushi/sushiswap-watcher-dashboard.json | 42 +++++++++++++++++-- .../monitoring/prometheus/prometheus.yml | 9 +++- .../data/stacks/monitoring/stack.yml | 2 + 14 files changed, 444 insertions(+), 24 deletions(-) diff --git a/stack_orchestrator/data/compose/docker-compose-prom-server.yml b/stack_orchestrator/data/compose/docker-compose-prom-server.yml index e3faecc8..27b8c18c 100644 --- a/stack_orchestrator/data/compose/docker-compose-prom-server.yml +++ b/stack_orchestrator/data/compose/docker-compose-prom-server.yml @@ -28,5 +28,20 @@ services: extra_hosts: - "host.docker.internal:host-gateway" + chain-head-exporter: + image: cerc/watcher-ts:local + restart: always + working_dir: /app/packages/cli + environment: + ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT} + FIL_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT} + ETH_RPC_API_KEY: ${CERC_INFURA_KEY} + PORT: ${CERC_METRICS_PORT} + command: ["sh", "-c", "yarn export-metrics:chain-heads"] + ports: + - '5000' + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: prometheus_data: diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json index 640b7a16..1abb276a 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json @@ -95,7 +95,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head block number of upstream endpoint", + "description": "Chain head block number of the upstream endpoint", "fieldConfig": { "defaults": { "color": { @@ -138,6 +138,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_upstream_block_number{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "Upstream chain head", "type": "stat" }, @@ -146,7 +164,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head from a public endpoint (etherscan)", + "description": "Chain head from an external public endpoint", "fieldConfig": { "defaults": { "color": { @@ -189,6 +207,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"REPLACE_WITH_WATCHER_CHAIN\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "External chain head", "type": "stat" }, diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json index 42e0855c..f7cb673f 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json @@ -95,7 +95,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head block number of upstream endpoint", + "description": "Chain head block number of the upstream endpoint", "fieldConfig": { "defaults": { "color": { @@ -138,6 +138,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"azimuth\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "Upstream chain head", "type": "stat" }, @@ -146,7 +164,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head from a public endpoint (etherscan)", + "description": "Chain head from an external public endpoint", "fieldConfig": { "defaults": { "color": { @@ -189,6 +207,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "External chain head", "type": "stat" }, diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json index 14bf1be5..8f926c18 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json @@ -95,7 +95,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head block number of upstream endpoint", + "description": "Chain head block number of the upstream endpoint", "fieldConfig": { "defaults": { "color": { @@ -138,6 +138,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"censures\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "Upstream chain head", "type": "stat" }, @@ -146,7 +164,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head from a public endpoint (etherscan)", + "description": "Chain head from an external public endpoint", "fieldConfig": { "defaults": { "color": { @@ -189,6 +207,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "External chain head", "type": "stat" }, diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json index e738ba78..a11fffc6 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json @@ -95,7 +95,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head block number of upstream endpoint", + "description": "Chain head block number of the upstream endpoint", "fieldConfig": { "defaults": { "color": { @@ -138,6 +138,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"claims\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "Upstream chain head", "type": "stat" }, @@ -146,7 +164,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head from a public endpoint (etherscan)", + "description": "Chain head from an external public endpoint", "fieldConfig": { "defaults": { "color": { @@ -189,6 +207,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "External chain head", "type": "stat" }, diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json index f089b41c..bba12b3e 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json @@ -95,7 +95,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head block number of upstream endpoint", + "description": "Chain head block number of the upstream endpoint", "fieldConfig": { "defaults": { "color": { @@ -138,6 +138,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"conditional_star_release\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "Upstream chain head", "type": "stat" }, @@ -146,7 +164,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head from a public endpoint (etherscan)", + "description": "Chain head from an external public endpoint", "fieldConfig": { "defaults": { "color": { @@ -189,6 +207,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "External chain head", "type": "stat" }, diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json index 015d5243..dcfc9ddd 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json @@ -95,7 +95,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head block number of upstream endpoint", + "description": "Chain head block number of the upstream endpoint", "fieldConfig": { "defaults": { "color": { @@ -138,6 +138,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"delegated_sending_watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "Upstream chain head", "type": "stat" }, @@ -146,7 +164,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head from a public endpoint (etherscan)", + "description": "Chain head from an external public endpoint", "fieldConfig": { "defaults": { "color": { @@ -189,6 +207,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "External chain head", "type": "stat" }, diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json index 6aca6504..ddfe18eb 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json @@ -95,7 +95,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head block number of upstream endpoint", + "description": "Chain head block number of the upstream endpoint", "fieldConfig": { "defaults": { "color": { @@ -138,6 +138,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"ecliptic\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "Upstream chain head", "type": "stat" }, @@ -146,7 +164,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head from a public endpoint (etherscan)", + "description": "Chain head from an external public endpoint", "fieldConfig": { "defaults": { "color": { @@ -189,6 +207,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "External chain head", "type": "stat" }, diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json index eb8e62de..a2a2bf93 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json @@ -95,7 +95,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head block number of upstream endpoint", + "description": "Chain head block number of the upstream endpoint", "fieldConfig": { "defaults": { "color": { @@ -138,6 +138,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"linear_star_release\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "Upstream chain head", "type": "stat" }, @@ -146,7 +164,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head from a public endpoint (etherscan)", + "description": "Chain head from an external public endpoint", "fieldConfig": { "defaults": { "color": { @@ -189,6 +207,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "External chain head", "type": "stat" }, diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json index fe894c5d..c68a1826 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json @@ -95,7 +95,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head block number of upstream endpoint", + "description": "Chain head block number of the upstream endpoint", "fieldConfig": { "defaults": { "color": { @@ -138,6 +138,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"polls\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "Upstream chain head", "type": "stat" }, @@ -146,7 +164,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head from a public endpoint (etherscan)", + "description": "Chain head from an external public endpoint", "fieldConfig": { "defaults": { "color": { @@ -189,6 +207,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "External chain head", "type": "stat" }, diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json index f76815e7..7a16e332 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json @@ -95,7 +95,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head block number of upstream endpoint", + "description": "Chain head block number of the upstream endpoint", "fieldConfig": { "defaults": { "color": { @@ -138,6 +138,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_upstream_block_number{job=\"sushi\", instance=\"merkl_sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "Upstream chain head", "type": "stat" }, @@ -146,7 +164,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head from a public endpoint (filfox)", + "description": "Chain head from an external public endpoint", "fieldConfig": { "defaults": { "color": { @@ -189,6 +207,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"filecoin\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "External chain head", "type": "stat" }, diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json index bcaa97f6..fffd722b 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json @@ -95,7 +95,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head block number of upstream endpoint", + "description": "Chain head block number of the upstream endpoint", "fieldConfig": { "defaults": { "color": { @@ -138,6 +138,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_upstream_block_number{job=\"sushi\", instance=\"sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "Upstream chain head", "type": "stat" }, @@ -146,7 +164,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head from a public endpoint (filfox)", + "description": "Chain head from an external public endpoint", "fieldConfig": { "defaults": { "color": { @@ -189,6 +207,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"filecoin\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "External chain head", "type": "stat" }, @@ -1174,6 +1210,6 @@ "timepicker": {}, "timezone": "", "title": "sushiswap watcher", - "version": 3, + "version": 1, "weekStart": "" } diff --git a/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml b/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml index 6b2e380e..f7631198 100644 --- a/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml +++ b/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml @@ -1,5 +1,5 @@ global: - scrape_interval: 15s + scrape_interval: 10s evaluation_interval: 15s rule_files: @@ -31,3 +31,10 @@ scrape_configs: regex: .* target_label: __address__ replacement: blackbox:9115 + + - job_name: chain_heads + metrics_path: /metrics + scheme: http + static_configs: + - targets: ['chain-head-exporter:5000'] + diff --git a/stack_orchestrator/data/stacks/monitoring/stack.yml b/stack_orchestrator/data/stacks/monitoring/stack.yml index c3532024..6a1e3d7e 100644 --- a/stack_orchestrator/data/stacks/monitoring/stack.yml +++ b/stack_orchestrator/data/stacks/monitoring/stack.yml @@ -1,7 +1,9 @@ version: "0.1" name: monitoring repos: + - github.com/cerc-io/watcher-ts # @v0.2.79 containers: + - cerc/watcher-ts pods: - prom-server - grafana -- 2.45.2 From 8c7aef121d11da9738f0ea62a1bc364c9e54451d Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Tue, 19 Dec 2023 16:44:48 +0530 Subject: [PATCH 10/15] Update watcher dashboards with watcher config metrics --- .../monitoring/grafana/dashboards/.gitkeep | 0 .../grafana/watcher-dashboard-template.json | 877 ++++++++++++++++- .../azimuth/azimuth-watcher-dashboard.json | 877 ++++++++++++++++- .../azimuth/censures-watcher-dashboard.json | 877 ++++++++++++++++- .../azimuth/claims-watcher-dashboard.json | 877 ++++++++++++++++- ...tional-star-release-watcher-dashboard.json | 877 ++++++++++++++++- ...ted-sending-watcher-watcher-dashboard.json | 877 ++++++++++++++++- .../azimuth/ecliptic-watcher-dashboard.json | 877 ++++++++++++++++- ...linear-star-release-watcher-dashboard.json | 877 ++++++++++++++++- .../azimuth/polls-watcher-dashboard.json | 877 ++++++++++++++++- .../merkl-sushiswap-watcher-dashboard.json | 886 +++++++++++++++++- .../sushi/sushiswap-watcher-dashboard.json | 886 +++++++++++++++++- .../monitoring/prometheus/prometheus.yml | 2 + .../stacks/monitoring/monitoring-watchers.md | 2 + 14 files changed, 9536 insertions(+), 133 deletions(-) delete mode 100644 stack_orchestrator/data/config/monitoring/grafana/dashboards/.gitkeep diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/.gitkeep b/stack_orchestrator/data/config/monitoring/grafana/dashboards/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json index 1abb276a..f38aa90d 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json @@ -29,7 +29,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -99,7 +99,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "orange", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -168,7 +168,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "red", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -330,6 +330,322 @@ "title": "Last block process duration (s)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "orange", + "index": 2, + "text": "LAZY" + }, + "1": { + "color": "semi-dark-green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"server\", field=\"is_active\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher kind", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 3 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "latest_upstream_block_number{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_upstream", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_upstream_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"REPLACE_WITH_WATCHER_CHAIN\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_external", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -356,10 +672,10 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 3 + "h": 3, + "w": 8, + "x": 4, + "y": 5 }, "id": 13, "options": { @@ -374,6 +690,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -429,6 +748,100 @@ "title": "Sync status", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-green", + "index": 1, + "text": "REAL TIME" + }, + "1": { + "color": "orange", + "index": 2, + "text": "HISTORICAL" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 5.5 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "is_syncing_historical_blocks{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher sync mode", + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -556,10 +969,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 0, - "y": 7 + "y": 8 }, "id": 7, "options": { @@ -574,6 +987,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -641,10 +1057,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 6, - "y": 7 + "y": 8 }, "id": 4, "options": { @@ -659,6 +1075,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -900,6 +1319,442 @@ ], "title": "Num event-processing jobs", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"upstream\", field=\"is_using_rpc_client\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"upstream\", field=\"is_fevm\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"server\", field=\"rpc_supports_block_hash\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"upstream\", field=\"filter_logs_by_topics\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + } + ], + "title": "Watcher config (upstream eth server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "watcher_config_info" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"server\", field=\"is_subgraph_watcher\"}", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"server\", field=\"max_events_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"server\", field=\"clear_entities_cache_interval\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"server\", field=\"max_simultaneous_requests\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"server\", field=\"max_request_queue_limit\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "F" + } + ], + "title": "Watcher config (server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "is_subgraph_watcher" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"jobqueue\", field=\"num_events_in_batch\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"jobqueue\", field=\"block_delay_seconds\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"jobqueue\", field=\"use_block_ranges\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + } + ], + "title": "Watcher config (job queue)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "use_block_ranges" + } + ], + "fields": {} + } + } + ], + "type": "stat" } ], "refresh": "10s", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json index f7cb673f..791c4a68 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json @@ -29,7 +29,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -99,7 +99,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "orange", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -168,7 +168,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "red", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -330,6 +330,322 @@ "title": "Last block process duration (s)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "orange", + "index": 2, + "text": "LAZY" + }, + "1": { + "color": "semi-dark-green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"server\", field=\"is_active\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher kind", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 3 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"azimuth\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"azimuth\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_upstream", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_upstream_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"azimuth\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_external", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -356,10 +672,10 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 3 + "h": 3, + "w": 8, + "x": 4, + "y": 5 }, "id": 13, "options": { @@ -374,6 +690,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -429,6 +748,100 @@ "title": "Sync status", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-green", + "index": 1, + "text": "REAL TIME" + }, + "1": { + "color": "orange", + "index": 2, + "text": "HISTORICAL" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 5.5 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "is_syncing_historical_blocks{job=\"azimuth\", instance=\"azimuth\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher sync mode", + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -556,10 +969,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 0, - "y": 7 + "y": 8 }, "id": 7, "options": { @@ -574,6 +987,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -641,10 +1057,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 6, - "y": 7 + "y": 8 }, "id": 4, "options": { @@ -659,6 +1075,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -900,6 +1319,442 @@ ], "title": "Num event-processing jobs", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"upstream\", field=\"is_using_rpc_client\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"upstream\", field=\"is_fevm\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"server\", field=\"rpc_supports_block_hash\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"upstream\", field=\"filter_logs_by_topics\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + } + ], + "title": "Watcher config (upstream eth server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "watcher_config_info" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"server\", field=\"is_subgraph_watcher\"}", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"server\", field=\"max_events_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"server\", field=\"clear_entities_cache_interval\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"server\", field=\"max_simultaneous_requests\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"server\", field=\"max_request_queue_limit\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "F" + } + ], + "title": "Watcher config (server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "is_subgraph_watcher" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"jobqueue\", field=\"num_events_in_batch\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"jobqueue\", field=\"block_delay_seconds\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"jobqueue\", field=\"use_block_ranges\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + } + ], + "title": "Watcher config (job queue)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "use_block_ranges" + } + ], + "fields": {} + } + } + ], + "type": "stat" } ], "refresh": "10s", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json index 8f926c18..b6cd04d0 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json @@ -29,7 +29,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -99,7 +99,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "orange", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -168,7 +168,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "red", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -330,6 +330,322 @@ "title": "Last block process duration (s)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "orange", + "index": 2, + "text": "LAZY" + }, + "1": { + "color": "semi-dark-green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"server\", field=\"is_active\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher kind", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 3 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"censures\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"censures\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_upstream", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_upstream_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"censures\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_external", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -356,10 +672,10 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 3 + "h": 3, + "w": 8, + "x": 4, + "y": 5 }, "id": 13, "options": { @@ -374,6 +690,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -429,6 +748,100 @@ "title": "Sync status", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-green", + "index": 1, + "text": "REAL TIME" + }, + "1": { + "color": "orange", + "index": 2, + "text": "HISTORICAL" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 5.5 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "is_syncing_historical_blocks{job=\"azimuth\", instance=\"censures\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher sync mode", + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -556,10 +969,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 0, - "y": 7 + "y": 8 }, "id": 7, "options": { @@ -574,6 +987,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -641,10 +1057,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 6, - "y": 7 + "y": 8 }, "id": 4, "options": { @@ -659,6 +1075,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -900,6 +1319,442 @@ ], "title": "Num event-processing jobs", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"upstream\", field=\"is_using_rpc_client\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"upstream\", field=\"is_fevm\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"server\", field=\"rpc_supports_block_hash\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"upstream\", field=\"filter_logs_by_topics\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + } + ], + "title": "Watcher config (upstream eth server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "watcher_config_info" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"server\", field=\"is_subgraph_watcher\"}", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"server\", field=\"max_events_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"server\", field=\"clear_entities_cache_interval\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"server\", field=\"max_simultaneous_requests\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"server\", field=\"max_request_queue_limit\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "F" + } + ], + "title": "Watcher config (server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "is_subgraph_watcher" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"jobqueue\", field=\"num_events_in_batch\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"jobqueue\", field=\"block_delay_seconds\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"jobqueue\", field=\"use_block_ranges\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + } + ], + "title": "Watcher config (job queue)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "use_block_ranges" + } + ], + "fields": {} + } + } + ], + "type": "stat" } ], "refresh": "10s", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json index a11fffc6..4903fbc9 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json @@ -29,7 +29,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -99,7 +99,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "orange", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -168,7 +168,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "red", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -330,6 +330,322 @@ "title": "Last block process duration (s)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "orange", + "index": 2, + "text": "LAZY" + }, + "1": { + "color": "semi-dark-green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"server\", field=\"is_active\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher kind", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 3 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"claims\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"claims\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_upstream", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_upstream_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"claims\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_external", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -356,10 +672,10 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 3 + "h": 3, + "w": 8, + "x": 4, + "y": 5 }, "id": 13, "options": { @@ -374,6 +690,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -429,6 +748,100 @@ "title": "Sync status", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-green", + "index": 1, + "text": "REAL TIME" + }, + "1": { + "color": "orange", + "index": 2, + "text": "HISTORICAL" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 5.5 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "is_syncing_historical_blocks{job=\"azimuth\", instance=\"claims\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher sync mode", + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -556,10 +969,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 0, - "y": 7 + "y": 8 }, "id": 7, "options": { @@ -574,6 +987,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -641,10 +1057,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 6, - "y": 7 + "y": 8 }, "id": 4, "options": { @@ -659,6 +1075,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -900,6 +1319,442 @@ ], "title": "Num event-processing jobs", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"upstream\", field=\"is_using_rpc_client\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"upstream\", field=\"is_fevm\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"server\", field=\"rpc_supports_block_hash\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"upstream\", field=\"filter_logs_by_topics\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + } + ], + "title": "Watcher config (upstream eth server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "watcher_config_info" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"server\", field=\"is_subgraph_watcher\"}", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"server\", field=\"max_events_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"server\", field=\"clear_entities_cache_interval\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"server\", field=\"max_simultaneous_requests\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"server\", field=\"max_request_queue_limit\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "F" + } + ], + "title": "Watcher config (server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "is_subgraph_watcher" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"jobqueue\", field=\"num_events_in_batch\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"jobqueue\", field=\"block_delay_seconds\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"jobqueue\", field=\"use_block_ranges\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + } + ], + "title": "Watcher config (job queue)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "use_block_ranges" + } + ], + "fields": {} + } + } + ], + "type": "stat" } ], "refresh": "10s", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json index bba12b3e..ba6fdb2d 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json @@ -29,7 +29,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -99,7 +99,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "orange", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -168,7 +168,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "red", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -330,6 +330,322 @@ "title": "Last block process duration (s)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "orange", + "index": 2, + "text": "LAZY" + }, + "1": { + "color": "semi-dark-green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"server\", field=\"is_active\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher kind", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 3 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"conditional_star_release\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"conditional_star_release\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_upstream", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_upstream_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"conditional_star_release\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_external", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -356,10 +672,10 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 3 + "h": 3, + "w": 8, + "x": 4, + "y": 5 }, "id": 13, "options": { @@ -374,6 +690,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -429,6 +748,100 @@ "title": "Sync status", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-green", + "index": 1, + "text": "REAL TIME" + }, + "1": { + "color": "orange", + "index": 2, + "text": "HISTORICAL" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 5.5 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "is_syncing_historical_blocks{job=\"azimuth\", instance=\"conditional_star_release\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher sync mode", + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -556,10 +969,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 0, - "y": 7 + "y": 8 }, "id": 7, "options": { @@ -574,6 +987,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -641,10 +1057,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 6, - "y": 7 + "y": 8 }, "id": 4, "options": { @@ -659,6 +1075,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -900,6 +1319,442 @@ ], "title": "Num event-processing jobs", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"upstream\", field=\"is_using_rpc_client\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"upstream\", field=\"is_fevm\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"server\", field=\"rpc_supports_block_hash\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"upstream\", field=\"filter_logs_by_topics\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + } + ], + "title": "Watcher config (upstream eth server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "watcher_config_info" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"server\", field=\"is_subgraph_watcher\"}", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"server\", field=\"max_events_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"server\", field=\"clear_entities_cache_interval\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"server\", field=\"max_simultaneous_requests\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"server\", field=\"max_request_queue_limit\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "F" + } + ], + "title": "Watcher config (server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "is_subgraph_watcher" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"jobqueue\", field=\"num_events_in_batch\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"jobqueue\", field=\"block_delay_seconds\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"jobqueue\", field=\"use_block_ranges\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + } + ], + "title": "Watcher config (job queue)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "use_block_ranges" + } + ], + "fields": {} + } + } + ], + "type": "stat" } ], "refresh": "10s", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json index dcfc9ddd..36ef4edd 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json @@ -29,7 +29,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -99,7 +99,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "orange", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -168,7 +168,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "red", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -330,6 +330,322 @@ "title": "Last block process duration (s)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "orange", + "index": 2, + "text": "LAZY" + }, + "1": { + "color": "semi-dark-green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"server\", field=\"is_active\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher kind", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 3 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"delegated_sending_watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"delegated_sending_watcher\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_upstream", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_upstream_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"delegated_sending_watcher\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_external", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -356,10 +672,10 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 3 + "h": 3, + "w": 8, + "x": 4, + "y": 5 }, "id": 13, "options": { @@ -374,6 +690,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -429,6 +748,100 @@ "title": "Sync status", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-green", + "index": 1, + "text": "REAL TIME" + }, + "1": { + "color": "orange", + "index": 2, + "text": "HISTORICAL" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 5.5 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "is_syncing_historical_blocks{job=\"azimuth\", instance=\"delegated_sending_watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher sync mode", + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -556,10 +969,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 0, - "y": 7 + "y": 8 }, "id": 7, "options": { @@ -574,6 +987,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -641,10 +1057,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 6, - "y": 7 + "y": 8 }, "id": 4, "options": { @@ -659,6 +1075,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -900,6 +1319,442 @@ ], "title": "Num event-processing jobs", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"upstream\", field=\"is_using_rpc_client\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"upstream\", field=\"is_fevm\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"server\", field=\"rpc_supports_block_hash\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"upstream\", field=\"filter_logs_by_topics\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + } + ], + "title": "Watcher config (upstream eth server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "watcher_config_info" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"server\", field=\"is_subgraph_watcher\"}", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"server\", field=\"max_events_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"server\", field=\"clear_entities_cache_interval\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"server\", field=\"max_simultaneous_requests\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"server\", field=\"max_request_queue_limit\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "F" + } + ], + "title": "Watcher config (server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "is_subgraph_watcher" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"jobqueue\", field=\"num_events_in_batch\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"jobqueue\", field=\"block_delay_seconds\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"jobqueue\", field=\"use_block_ranges\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + } + ], + "title": "Watcher config (job queue)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "use_block_ranges" + } + ], + "fields": {} + } + } + ], + "type": "stat" } ], "refresh": "10s", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json index ddfe18eb..4ee284a9 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json @@ -29,7 +29,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -99,7 +99,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "orange", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -168,7 +168,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "red", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -330,6 +330,322 @@ "title": "Last block process duration (s)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "orange", + "index": 2, + "text": "LAZY" + }, + "1": { + "color": "semi-dark-green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"server\", field=\"is_active\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher kind", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 3 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"ecliptic\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"ecliptic\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_upstream", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_upstream_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"ecliptic\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_external", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -356,10 +672,10 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 3 + "h": 3, + "w": 8, + "x": 4, + "y": 5 }, "id": 13, "options": { @@ -374,6 +690,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -429,6 +748,100 @@ "title": "Sync status", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-green", + "index": 1, + "text": "REAL TIME" + }, + "1": { + "color": "orange", + "index": 2, + "text": "HISTORICAL" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 5.5 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "is_syncing_historical_blocks{job=\"azimuth\", instance=\"ecliptic\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher sync mode", + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -556,10 +969,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 0, - "y": 7 + "y": 8 }, "id": 7, "options": { @@ -574,6 +987,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -641,10 +1057,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 6, - "y": 7 + "y": 8 }, "id": 4, "options": { @@ -659,6 +1075,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -900,6 +1319,442 @@ ], "title": "Num event-processing jobs", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"upstream\", field=\"is_using_rpc_client\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"upstream\", field=\"is_fevm\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"server\", field=\"rpc_supports_block_hash\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"upstream\", field=\"filter_logs_by_topics\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + } + ], + "title": "Watcher config (upstream eth server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "watcher_config_info" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"server\", field=\"is_subgraph_watcher\"}", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"server\", field=\"max_events_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"server\", field=\"clear_entities_cache_interval\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"server\", field=\"max_simultaneous_requests\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"server\", field=\"max_request_queue_limit\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "F" + } + ], + "title": "Watcher config (server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "is_subgraph_watcher" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"jobqueue\", field=\"num_events_in_batch\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"jobqueue\", field=\"block_delay_seconds\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"jobqueue\", field=\"use_block_ranges\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + } + ], + "title": "Watcher config (job queue)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "use_block_ranges" + } + ], + "fields": {} + } + } + ], + "type": "stat" } ], "refresh": "10s", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json index a2a2bf93..c5e16fb8 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json @@ -29,7 +29,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -99,7 +99,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "orange", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -168,7 +168,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "red", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -330,6 +330,322 @@ "title": "Last block process duration (s)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "orange", + "index": 2, + "text": "LAZY" + }, + "1": { + "color": "semi-dark-green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"server\", field=\"is_active\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher kind", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 3 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"linear_star_release\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"linear_star_release\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_upstream", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_upstream_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"linear_star_release\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_external", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -356,10 +672,10 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 3 + "h": 3, + "w": 8, + "x": 4, + "y": 5 }, "id": 13, "options": { @@ -374,6 +690,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -429,6 +748,100 @@ "title": "Sync status", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-green", + "index": 1, + "text": "REAL TIME" + }, + "1": { + "color": "orange", + "index": 2, + "text": "HISTORICAL" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 5.5 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "is_syncing_historical_blocks{job=\"azimuth\", instance=\"linear_star_release\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher sync mode", + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -556,10 +969,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 0, - "y": 7 + "y": 8 }, "id": 7, "options": { @@ -574,6 +987,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -641,10 +1057,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 6, - "y": 7 + "y": 8 }, "id": 4, "options": { @@ -659,6 +1075,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -900,6 +1319,442 @@ ], "title": "Num event-processing jobs", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"upstream\", field=\"is_using_rpc_client\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"upstream\", field=\"is_fevm\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"server\", field=\"rpc_supports_block_hash\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"upstream\", field=\"filter_logs_by_topics\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + } + ], + "title": "Watcher config (upstream eth server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "watcher_config_info" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"server\", field=\"is_subgraph_watcher\"}", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"server\", field=\"max_events_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"server\", field=\"clear_entities_cache_interval\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"server\", field=\"max_simultaneous_requests\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"server\", field=\"max_request_queue_limit\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "F" + } + ], + "title": "Watcher config (server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "is_subgraph_watcher" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"jobqueue\", field=\"num_events_in_batch\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"jobqueue\", field=\"block_delay_seconds\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"jobqueue\", field=\"use_block_ranges\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + } + ], + "title": "Watcher config (job queue)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "use_block_ranges" + } + ], + "fields": {} + } + } + ], + "type": "stat" } ], "refresh": "10s", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json index c68a1826..5d35f868 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json @@ -29,7 +29,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -99,7 +99,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "orange", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -168,7 +168,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "red", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -330,6 +330,322 @@ "title": "Last block process duration (s)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "orange", + "index": 2, + "text": "LAZY" + }, + "1": { + "color": "semi-dark-green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"server\", field=\"is_active\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher kind", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 3 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"polls\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"polls\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_upstream", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_upstream_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"polls\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_external", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -356,10 +672,10 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 3 + "h": 3, + "w": 8, + "x": 4, + "y": 5 }, "id": 13, "options": { @@ -374,6 +690,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -429,6 +748,100 @@ "title": "Sync status", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-green", + "index": 1, + "text": "REAL TIME" + }, + "1": { + "color": "orange", + "index": 2, + "text": "HISTORICAL" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 5.5 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "is_syncing_historical_blocks{job=\"azimuth\", instance=\"polls\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher sync mode", + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -556,10 +969,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 0, - "y": 7 + "y": 8 }, "id": 7, "options": { @@ -574,6 +987,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -641,10 +1057,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 6, - "y": 7 + "y": 8 }, "id": 4, "options": { @@ -659,6 +1075,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -900,6 +1319,442 @@ ], "title": "Num event-processing jobs", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"upstream\", field=\"is_using_rpc_client\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"upstream\", field=\"is_fevm\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"server\", field=\"rpc_supports_block_hash\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"upstream\", field=\"filter_logs_by_topics\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + } + ], + "title": "Watcher config (upstream eth server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "watcher_config_info" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"server\", field=\"is_subgraph_watcher\"}", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"server\", field=\"max_events_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"server\", field=\"clear_entities_cache_interval\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"server\", field=\"max_simultaneous_requests\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"server\", field=\"max_request_queue_limit\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "F" + } + ], + "title": "Watcher config (server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "is_subgraph_watcher" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"jobqueue\", field=\"num_events_in_batch\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"jobqueue\", field=\"block_delay_seconds\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"jobqueue\", field=\"use_block_ranges\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + } + ], + "title": "Watcher config (job queue)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "use_block_ranges" + } + ], + "fields": {} + } + } + ], + "type": "stat" } ], "refresh": "10s", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json index 7a16e332..2cb41527 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json @@ -29,7 +29,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -99,7 +99,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "orange", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -168,7 +168,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "red", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -330,6 +330,322 @@ "title": "Last block process duration (s)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "orange", + "index": 2, + "text": "LAZY" + }, + "1": { + "color": "semi-dark-green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"server\", field=\"is_active\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher kind", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 3 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "latest_upstream_block_number{job=\"sushi\", instance=\"merkl_sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"sushi\", instance=\"merkl_sushiswap\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_upstream", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_upstream_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"filecoin\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"sushi\", instance=\"merkl_sushiswap\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_external", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -356,10 +672,10 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 3 + "h": 3, + "w": 8, + "x": 4, + "y": 5 }, "id": 13, "options": { @@ -374,6 +690,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -429,6 +748,100 @@ "title": "Sync status", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-green", + "index": 1, + "text": "REAL TIME" + }, + "1": { + "color": "orange", + "index": 2, + "text": "HISTORICAL" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 5.5 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "is_syncing_historical_blocks{job=\"sushi\", instance=\"merkl_sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher sync mode", + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -556,10 +969,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 0, - "y": 7 + "y": 8 }, "id": 7, "options": { @@ -574,6 +987,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -641,10 +1057,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 6, - "y": 7 + "y": 8 }, "id": 4, "options": { @@ -659,6 +1075,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -950,8 +1369,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -1047,8 +1465,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -1144,8 +1561,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -1192,6 +1608,442 @@ ], "title": "Num subgraph entities cached (pruned region)", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 23 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"upstream\", field=\"is_using_rpc_client\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"upstream\", field=\"is_fevm\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"server\", field=\"rpc_supports_block_hash\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"upstream\", field=\"filter_logs_by_topics\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + } + ], + "title": "Watcher config (upstream eth server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "watcher_config_info" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 23 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"server\", field=\"is_subgraph_watcher\"}", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"server\", field=\"max_events_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"server\", field=\"clear_entities_cache_interval\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"server\", field=\"max_simultaneous_requests\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"server\", field=\"max_request_queue_limit\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "F" + } + ], + "title": "Watcher config (server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "is_subgraph_watcher" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 23 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"jobqueue\", field=\"num_events_in_batch\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"jobqueue\", field=\"block_delay_seconds\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"jobqueue\", field=\"use_block_ranges\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + } + ], + "title": "Watcher config (job queue)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "use_block_ranges" + } + ], + "fields": {} + } + } + ], + "type": "stat" } ], "refresh": "20s", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json index fffd722b..674568bd 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json @@ -29,7 +29,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -99,7 +99,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "orange", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -168,7 +168,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "red", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -330,6 +330,322 @@ "title": "Last block process duration (s)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "orange", + "index": 2, + "text": "LAZY" + }, + "1": { + "color": "semi-dark-green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"server\", field=\"is_active\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher kind", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 3 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "latest_upstream_block_number{job=\"sushi\", instance=\"sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"sushi\", instance=\"sushiswap\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_upstream", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_upstream_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"filecoin\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"sushi\", instance=\"sushiswap\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_external", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -356,10 +672,10 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 3 + "h": 3, + "w": 8, + "x": 4, + "y": 5 }, "id": 13, "options": { @@ -374,6 +690,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -429,6 +748,100 @@ "title": "Sync status", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-green", + "index": 1, + "text": "REAL TIME" + }, + "1": { + "color": "orange", + "index": 2, + "text": "HISTORICAL" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 5.5 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "is_syncing_historical_blocks{job=\"sushi\", instance=\"sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher sync mode", + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -556,10 +969,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 0, - "y": 7 + "y": 8 }, "id": 7, "options": { @@ -574,6 +987,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -641,10 +1057,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 6, - "y": 7 + "y": 8 }, "id": 4, "options": { @@ -659,6 +1075,9 @@ "fields": "", "values": false }, + "text": { + "titleSize": 15 + }, "textMode": "auto", "wideLayout": true }, @@ -950,8 +1369,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -1047,8 +1465,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -1144,8 +1561,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -1192,6 +1608,442 @@ ], "title": "Num subgraph entities cached (pruned region)", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 23 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"upstream\", field=\"is_using_rpc_client\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"upstream\", field=\"is_fevm\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"server\", field=\"rpc_supports_block_hash\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"upstream\", field=\"filter_logs_by_topics\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + } + ], + "title": "Watcher config (upstream eth server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "watcher_config_info" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 23 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"server\", field=\"is_subgraph_watcher\"}", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"server\", field=\"max_events_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"server\", field=\"clear_entities_cache_interval\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"server\", field=\"max_simultaneous_requests\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"server\", field=\"max_request_queue_limit\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "F" + } + ], + "title": "Watcher config (server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "is_subgraph_watcher" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 23 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"jobqueue\", field=\"num_events_in_batch\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"jobqueue\", field=\"block_delay_seconds\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"jobqueue\", field=\"use_block_ranges\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + } + ], + "title": "Watcher config (job queue)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "use_block_ranges" + } + ], + "fields": {} + } + } + ], + "type": "stat" } ], "refresh": "20s", diff --git a/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml b/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml index f7631198..40068287 100644 --- a/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml +++ b/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml @@ -12,6 +12,7 @@ scrape_configs: - targets: ['localhost:9090'] - job_name: 'blackbox' + scrape_interval: 10s metrics_path: /probe params: module: [http_2xx] @@ -33,6 +34,7 @@ scrape_configs: replacement: blackbox:9115 - job_name: chain_heads + scrape_interval: 10s metrics_path: /metrics scheme: http static_configs: diff --git a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md index a209d6d7..81410872 100644 --- a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md +++ b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md @@ -48,6 +48,7 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers - ... - job_name: azimuth + scrape_interval: 10s metrics_path: /metrics scheme: http static_configs: @@ -77,6 +78,7 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers instance: 'polls' - job_name: sushi + scrape_interval: 20s metrics_path: /metrics scheme: http static_configs: -- 2.45.2 From 2608e3bad9803db2e31a60f60b283b3d8bae7a4e Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Tue, 19 Dec 2023 17:42:30 +0530 Subject: [PATCH 11/15] Upgrade sushiswap and azimuth watchers --- .../watcher-config-template.toml | 2 -- .../watcher-config-template.toml | 2 -- stack_orchestrator/data/stacks/azimuth/README.md | 16 ++++++++++++++++ stack_orchestrator/data/stacks/azimuth/stack.yml | 2 +- .../data/stacks/monitoring/stack.yml | 2 +- .../data/stacks/sushiswap-v3/stack.yml | 2 +- 6 files changed, 19 insertions(+), 7 deletions(-) diff --git a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml index 8d3b3f4d..57494ce3 100644 --- a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml +++ b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml @@ -84,8 +84,6 @@ subgraphEventsOrder = true # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime blockDelayInMilliSecs = 30000 - prefetchBlocksInMem = false - prefetchBlockCount = 10 # Boolean to switch between modes of processing events when starting the server. # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them. diff --git a/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml index e45cf0ea..24449d4f 100644 --- a/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml +++ b/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml @@ -84,8 +84,6 @@ subgraphEventsOrder = true # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime blockDelayInMilliSecs = 30000 - prefetchBlocksInMem = false - prefetchBlockCount = 10 # Boolean to switch between modes of processing events when starting the server. # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them. diff --git a/stack_orchestrator/data/stacks/azimuth/README.md b/stack_orchestrator/data/stacks/azimuth/README.md index f7d93f33..21200369 100644 --- a/stack_orchestrator/data/stacks/azimuth/README.md +++ b/stack_orchestrator/data/stacks/azimuth/README.md @@ -40,20 +40,36 @@ network: ports: watcher-db: - 0.0.0.0:15432:5432 + azimuth-watcher-job-runner: + - 0.0.0.0:9000:9000 azimuth-watcher-server: - 0.0.0.0:3001:3001 + censures-watcher-job-runner: + - 0.0.0.0:9002:9002 censures-watcher-server: - 0.0.0.0:3002:3002 + claims-watcher-job-runner: + - 0.0.0.0:9004:9004 claims-watcher-server: - 0.0.0.0:3003:3003 + conditional-star-release-watcher-job-runner: + - 0.0.0.0:9006:9006 conditional-star-release-watcher-server: - 0.0.0.0:3004:3004 + delegated-sending-watcher-job-runner: + - 0.0.0.0:9008:9008 delegated-sending-watcher-server: - 0.0.0.0:3005:3005 + ecliptic-watcher-job-runner: + - 0.0.0.0:9010:9010 ecliptic-watcher-server: - 0.0.0.0:3006:3006 + linear-star-release-watcher-job-runner: + - 0.0.0.0:9012:9012 linear-star-release-watcher-server: - 0.0.0.0:3007:3007 + polls-watcher-job-runner: + - 0.0.0.0:9014:9014 polls-watcher-server: - 0.0.0.0:3008:3008 gateway-server: diff --git a/stack_orchestrator/data/stacks/azimuth/stack.yml b/stack_orchestrator/data/stacks/azimuth/stack.yml index 7adbe663..374f2af0 100644 --- a/stack_orchestrator/data/stacks/azimuth/stack.yml +++ b/stack_orchestrator/data/stacks/azimuth/stack.yml @@ -1,7 +1,7 @@ version: "1.0" name: azimuth repos: - - github.com/cerc-io/azimuth-watcher-ts@v0.1.2 + - github.com/cerc-io/azimuth-watcher-ts@v0.1.3 containers: - cerc/watcher-azimuth pods: diff --git a/stack_orchestrator/data/stacks/monitoring/stack.yml b/stack_orchestrator/data/stacks/monitoring/stack.yml index 6a1e3d7e..058ca971 100644 --- a/stack_orchestrator/data/stacks/monitoring/stack.yml +++ b/stack_orchestrator/data/stacks/monitoring/stack.yml @@ -1,7 +1,7 @@ version: "0.1" name: monitoring repos: - - github.com/cerc-io/watcher-ts # @v0.2.79 + - github.com/cerc-io/watcher-ts@v0.2.79 containers: - cerc/watcher-ts pods: diff --git a/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml b/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml index 49c604bf..90766012 100644 --- a/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml +++ b/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml @@ -2,7 +2,7 @@ version: "1.0" name: sushiswap-v3 description: "SushiSwap v3 watcher stack" repos: - - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.4 + - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.5 containers: - cerc/watcher-sushiswap-v3 pods: -- 2.45.2 From 70a12407d51d3a9c3d7528c079ad5aa44b2467e0 Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Wed, 20 Dec 2023 10:38:38 +0530 Subject: [PATCH 12/15] Removed fixed title size values --- .../grafana/watcher-dashboard-template.json | 18 ++++++------------ .../azimuth/azimuth-watcher-dashboard.json | 18 ++++++------------ .../azimuth/censures-watcher-dashboard.json | 18 ++++++------------ .../azimuth/claims-watcher-dashboard.json | 18 ++++++------------ ...itional-star-release-watcher-dashboard.json | 18 ++++++------------ ...ated-sending-watcher-watcher-dashboard.json | 18 ++++++------------ .../azimuth/ecliptic-watcher-dashboard.json | 18 ++++++------------ .../linear-star-release-watcher-dashboard.json | 18 ++++++------------ .../azimuth/polls-watcher-dashboard.json | 18 ++++++------------ .../merkl-sushiswap-watcher-dashboard.json | 18 ++++++------------ .../sushi/sushiswap-watcher-dashboard.json | 18 ++++++------------ 11 files changed, 66 insertions(+), 132 deletions(-) diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json index f38aa90d..1e0f4e17 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json @@ -53,7 +53,7 @@ }, "id": 1, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -123,7 +123,7 @@ }, "id": 11, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -192,7 +192,7 @@ }, "id": 12, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -690,9 +690,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -987,9 +985,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -1075,9 +1071,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json index 791c4a68..3a67d7f4 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json @@ -53,7 +53,7 @@ }, "id": 1, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -123,7 +123,7 @@ }, "id": 11, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -192,7 +192,7 @@ }, "id": 12, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -690,9 +690,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -987,9 +985,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -1075,9 +1071,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json index b6cd04d0..04c93969 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json @@ -53,7 +53,7 @@ }, "id": 1, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -123,7 +123,7 @@ }, "id": 11, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -192,7 +192,7 @@ }, "id": 12, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -690,9 +690,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -987,9 +985,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -1075,9 +1071,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json index 4903fbc9..9c5c5510 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json @@ -53,7 +53,7 @@ }, "id": 1, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -123,7 +123,7 @@ }, "id": 11, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -192,7 +192,7 @@ }, "id": 12, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -690,9 +690,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -987,9 +985,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -1075,9 +1071,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json index ba6fdb2d..6f739555 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json @@ -53,7 +53,7 @@ }, "id": 1, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -123,7 +123,7 @@ }, "id": 11, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -192,7 +192,7 @@ }, "id": 12, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -690,9 +690,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -987,9 +985,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -1075,9 +1071,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json index 36ef4edd..9c169174 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json @@ -53,7 +53,7 @@ }, "id": 1, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -123,7 +123,7 @@ }, "id": 11, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -192,7 +192,7 @@ }, "id": 12, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -690,9 +690,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -987,9 +985,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -1075,9 +1071,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json index 4ee284a9..f46408f0 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json @@ -53,7 +53,7 @@ }, "id": 1, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -123,7 +123,7 @@ }, "id": 11, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -192,7 +192,7 @@ }, "id": 12, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -690,9 +690,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -987,9 +985,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -1075,9 +1071,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json index c5e16fb8..e2187a43 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json @@ -53,7 +53,7 @@ }, "id": 1, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -123,7 +123,7 @@ }, "id": 11, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -192,7 +192,7 @@ }, "id": 12, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -690,9 +690,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -987,9 +985,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -1075,9 +1071,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json index 5d35f868..5e653ee3 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json @@ -53,7 +53,7 @@ }, "id": 1, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -123,7 +123,7 @@ }, "id": 11, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -192,7 +192,7 @@ }, "id": 12, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -690,9 +690,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -987,9 +985,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -1075,9 +1071,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json index 2cb41527..8e495464 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json @@ -53,7 +53,7 @@ }, "id": 1, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -123,7 +123,7 @@ }, "id": 11, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -192,7 +192,7 @@ }, "id": 12, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -690,9 +690,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -987,9 +985,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -1075,9 +1071,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json index 674568bd..b611f04d 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json @@ -53,7 +53,7 @@ }, "id": 1, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -123,7 +123,7 @@ }, "id": 11, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -192,7 +192,7 @@ }, "id": 12, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -690,9 +690,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -987,9 +985,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -1075,9 +1071,7 @@ "fields": "", "values": false }, - "text": { - "titleSize": 15 - }, + "text": {}, "textMode": "auto", "wideLayout": true }, -- 2.45.2 From a65a9ae390f9334e454a29eb82a20b8878be527b Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Wed, 20 Dec 2023 14:55:06 +0530 Subject: [PATCH 13/15] Update instructions --- .../data/stacks/monitoring/README.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/stack_orchestrator/data/stacks/monitoring/README.md b/stack_orchestrator/data/stacks/monitoring/README.md index 6f1fc227..f8149f96 100644 --- a/stack_orchestrator/data/stacks/monitoring/README.md +++ b/stack_orchestrator/data/stacks/monitoring/README.md @@ -1,6 +1,9 @@ # monitoring * Instructions to setup and run a Prometheus server and Grafana dashboard +* Comes with the following built-in exporters / dashboards: + * [Prometheus Blackbox Exporter](https://grafana.com/grafana/dashboards/7587-prometheus-blackbox-exporter/) - for tracking HTTP endpoints + * [NodeJS Application Dashboard](https://grafana.com/grafana/dashboards/11159-nodejs-application-dashboard/) - for default NodeJS metrics * See [monitoring-watchers.md](./monitoring-watchers.md) for an example usage of the stack with pre-configured dashboards for watchers ## Create a deployment @@ -43,7 +46,7 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --de ### Prometheus Config -Add desired scrape configs to prometheus config file (`monitoring-deployment/config/monitoring/prometheus/prometheus.yml`) in the deployment folder; for example: +* Add desired scrape configs to prometheus config file (`monitoring-deployment/config/monitoring/prometheus/prometheus.yml`) in the deployment folder; for example: ```yml ... @@ -54,6 +57,19 @@ Add desired scrape configs to prometheus config file (`monitoring-deployment/con - targets: [':'] ``` +* Also update the `blackbox` job to add any endpoints to be monitored on the Blackbox dashboard: + + ```yml + ... + - job_name: 'blackbox' + ... + static_configs: + # Add URLs to be monitored below + - targets: + - + - + ``` + Note: Use `host.docker.internal` as host to access ports on the host machine ### Grafana Config -- 2.45.2 From 324a32a0ae3bed7080f587bd68e7fb4af47e9e72 Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Wed, 20 Dec 2023 15:06:57 +0530 Subject: [PATCH 14/15] Update instructions for env config --- .../data/stacks/monitoring/README.md | 19 +++++++++++++++++++ .../stacks/monitoring/monitoring-watchers.md | 9 +++++++++ 2 files changed, 28 insertions(+) diff --git a/stack_orchestrator/data/stacks/monitoring/README.md b/stack_orchestrator/data/stacks/monitoring/README.md index f8149f96..95dfcca8 100644 --- a/stack_orchestrator/data/stacks/monitoring/README.md +++ b/stack_orchestrator/data/stacks/monitoring/README.md @@ -4,6 +4,7 @@ * Comes with the following built-in exporters / dashboards: * [Prometheus Blackbox Exporter](https://grafana.com/grafana/dashboards/7587-prometheus-blackbox-exporter/) - for tracking HTTP endpoints * [NodeJS Application Dashboard](https://grafana.com/grafana/dashboards/11159-nodejs-application-dashboard/) - for default NodeJS metrics + * Chain Head Exporter - for tracking chain heads given external ETH RPC endpoints * See [monitoring-watchers.md](./monitoring-watchers.md) for an example usage of the stack with pre-configured dashboards for watchers ## Create a deployment @@ -76,6 +77,24 @@ Note: Use `host.docker.internal` as host to access ports on the host machine Place the dashboard json files in grafana dashboards config directory (`monitoring-deployment/config/monitoring/grafana/dashboards`) in the deployment folder +### Env + +Set the following env variables in the deployment env config file (`monitoring-deployment/config.env`): + + ```bash + # External ETH RPC endpoint (ethereum) + # (Optional, default: https://mainnet.infura.io/v3) + CERC_ETH_RPC_ENDPOINT= + + # Infura key to be used + # (Optional, used with ETH_RPC_ENDPOINT if provided) + CERC_INFURA_KEY= + + # External ETH RPC endpoint (filecoin) + # (Optional, default: https://api.node.glif.io/rpc/v1) + CERC_FIL_RPC_ENDPOINT= + ``` + ## Start the stack Start the deployment: diff --git a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md index 81410872..41ba2e07 100644 --- a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md +++ b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md @@ -98,6 +98,15 @@ In the deployment folder, copy over the pre-configured watcher dashboard JSON fi cp -r monitoring-watchers-deployment/config/monitoring/grafana/watcher-dashboards/* monitoring-watchers-deployment/config/monitoring/grafana/dashboards/ ``` +### Env + +Set the following env variables in the deployment env config file (`monitoring-watchers-deployment/config.env`): + + ```bash + # Infura key to be used + CERC_INFURA_KEY= + ``` + ## Start the stack Start the deployment: -- 2.45.2 From 54b30ce2ea6982e8f3e184efcf93d77198c0728d Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Wed, 20 Dec 2023 15:15:35 +0530 Subject: [PATCH 15/15] Update instructions with setup --- .../data/stacks/monitoring/README.md | 14 ++++++++++++++ .../data/stacks/monitoring/monitoring-watchers.md | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/stack_orchestrator/data/stacks/monitoring/README.md b/stack_orchestrator/data/stacks/monitoring/README.md index 95dfcca8..803da548 100644 --- a/stack_orchestrator/data/stacks/monitoring/README.md +++ b/stack_orchestrator/data/stacks/monitoring/README.md @@ -7,6 +7,20 @@ * Chain Head Exporter - for tracking chain heads given external ETH RPC endpoints * See [monitoring-watchers.md](./monitoring-watchers.md) for an example usage of the stack with pre-configured dashboards for watchers +## Setup + +Clone required repositories: + +```bash +laconic-so --stack monitoring setup-repositories --git-ssh --pull +``` + +Build the container images: + +```bash +laconic-so --stack monitoring build-containers +``` + ## Create a deployment First, create a spec file for the deployment, which will map the stack's ports and volumes to the host: diff --git a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md index 41ba2e07..4ef43b36 100644 --- a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md +++ b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md @@ -4,7 +4,7 @@ Instructions to setup and run monitoring stack with pre-configured watcher dashb ## Create a deployment -First, create a spec file for the deployment, which will map the stack's ports and volumes to the host: +After completing [setup](./README.md#setup), create a spec file for the deployment, which will map the stack's ports and volumes to the host: ```bash laconic-so --stack monitoring deploy init --output monitoring-watchers-spec.yml -- 2.45.2