From 042b413598df5dd691853be6ac12b5c90f336f87 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 8 Nov 2023 23:44:48 -0600 Subject: [PATCH 01/75] Support the case where webpack config is already present next.config.js (#631) * Support the case where webpack config is already present next.config.js * Update scripts for experimental-compile/experimental-generate --- stack_orchestrator/build/build_webapp.py | 1 + .../cerc-nextjs-base/Dockerfile | 2 +- .../scripts/apply-runtime-env.sh | 4 +- .../cerc-nextjs-base/scripts/build-app.sh | 73 +++++++++++++------ .../cerc-nextjs-base/scripts/find-env.sh | 5 ++ .../scripts/start-serving-app.sh | 25 ++++++- 6 files changed, 84 insertions(+), 26 deletions(-) diff --git a/stack_orchestrator/build/build_webapp.py b/stack_orchestrator/build/build_webapp.py index f4668c5d..7a656ae7 100644 --- a/stack_orchestrator/build/build_webapp.py +++ b/stack_orchestrator/build/build_webapp.py @@ -65,6 +65,7 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args): # Now build the target webapp. We use the same build script, but with a different Dockerfile and work dir. + container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true" container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"] = os.path.abspath(source_repo) container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = os.path.join(container_build_dir, base_container.replace("/", "-"), diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile index 147cec29..435d8c73 100644 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile @@ -1,6 +1,6 @@ # Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile # [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster -ARG VARIANT=18-bullseye +ARG VARIANT=20-bullseye FROM node:${VARIANT} ARG USERNAME=node diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh index ba1cd17d..793333c3 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh @@ -25,12 +25,12 @@ if [ -f ".env" ]; then fi for f in $(find "$TRG_DIR" -regex ".*.[tj]sx?$" -type f | grep -v 'node_modules'); do - for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '[{},()]' '\n' | egrep -o '^"CERC_RUNTIME_ENV[^\"]+"$'); do + for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '[{},();]' '\n' | egrep -o '^"CERC_RUNTIME_ENV_[^\"]+"'); do orig_name=$(echo -n "${e}" | sed 's/"//g') cur_name=$(echo -n "${orig_name}" | sed 's/CERC_RUNTIME_ENV_//g') cur_val=$(echo -n "\$${cur_name}" | envsubst) esc_val=$(sed 's/[&/\]/\\&/g' <<< "$cur_val") - echo "$cur_name=$cur_val" + echo "$f: $cur_name=$cur_val" sed -i "s/$orig_name/$esc_val/g" $f done done diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh index 9277abc6..c2115f6a 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh @@ -10,21 +10,25 @@ WORK_DIR="${1:-/app}" cd "${WORK_DIR}" || exit 1 -cp next.config.js next.config.dist +if [ ! -f "next.config.dist" ]; then + cp next.config.js next.config.dist +fi + +which js-beautify >/dev/null +if [ $? -ne 0 ]; then + npm i -g js-beautify +fi -npm i -g js-beautify js-beautify next.config.dist > next.config.js -npm install - -CONFIG_LINES=$(wc -l next.config.js | awk '{ print $1 }') -MOD_EXPORTS_LINE=$(grep -n 'module.exports' next.config.js | cut -d':' -f1) - -head -$(( ${MOD_EXPORTS_LINE} - 1 )) next.config.js > next.config.js.1 - -cat > next.config.js.2 < next.config.js.0 < next.config.js.1 < next.config.js.3 +CONFIG_LINES=$(wc -l next.config.js | awk '{ print $1 }') +ENV_LINE=$(grep -n 'env:' next.config.js | cut -d':' -f1) +WEBPACK_CONF_LINE=$(egrep -n 'webpack:\s+\([^,]+,' next.config.js | cut -d':' -f1) +NEXT_SECTION_ADJUSTMENT=0 -cat > next.config.js.4 < { - config.plugins.push(new webpack.DefinePlugin(envMap)); - return config; - }, +if [ -n "$WEBPACK_CONF_LINE" ]; then + WEBPACK_CONF_VAR=$(egrep -n 'webpack:\s+\([^,]+,' next.config.js | cut -d',' -f1 | cut -d'(' -f2) + head -$(( ${WEBPACK_CONF_LINE} )) next.config.js > next.config.js.2 + cat > next.config.js.3 < next.config.js.2 + cat > next.config.js.3 < { + config.plugins.push(new webpack.DefinePlugin(envMap)); + return config; + }, +EOF + NEXT_SECTION_ADJUSTMENT=2 + NEXT_SECTION_LINE=$ENV_LINE +else + echo "WARNING: Cannot find location to insert environment variable map in next.config.js" 1>&2 + rm -f next.config.js.* + NEXT_SECTION_LINE=0 +fi -tail -$(( ${CONFIG_LINES} - ${MOD_EXPORTS_LINE} + 1 )) next.config.js | grep -v 'process\.env\.' > next.config.js.5 +tail -$(( ${CONFIG_LINES} - ${NEXT_SECTION_LINE} + ${NEXT_SECTION_ADJUSTMENT} )) next.config.js > next.config.js.5 -cat next.config.js.* | js-beautify > next.config.js +cat next.config.js.* | sed 's/^ *//g' | js-beautify | grep -v 'process\.\env\.' | js-beautify > next.config.js rm next.config.js.* "${SCRIPT_DIR}/find-env.sh" "$(pwd)" > .env-list.json -npm run build -rm .env-list.json \ No newline at end of file +if [ ! -f "package.dist" ]; then + cp package.json package.dist +fi + +cat package.dist | jq '.scripts.cerc_compile = "next experimental-compile"' | jq '.scripts.cerc_generate = "next experimental-generate"' > package.json + +npm install || exit 1 +npm run cerc_compile || exit 1 + +exit 0 diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/find-env.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/find-env.sh index 0c0e87c9..59cb3d49 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/find-env.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/find-env.sh @@ -20,5 +20,10 @@ for d in $(find . -maxdepth 1 -type d | grep -v '\./\.' | grep '/' | cut -d'/' - done done +NEXT_CONF="next.config.js next.config.dist" +for f in $NEXT_CONF; do + cat "$f" | tr -s '[:blank:]' '\n' | tr -s '[{},()]' '\n' | egrep -o 'process.env.[A-Za-z0-9_]+' >> $TMPF +done + cat $TMPF | sort -u | jq --raw-input . | jq --slurp . rm -f $TMPF diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh index abe72935..61664c68 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh @@ -8,6 +8,27 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/app}" cd "$CERC_WEBAPP_FILES_DIR" -rm -rf .next-r "$SCRIPT_DIR/apply-runtime-env.sh" "`pwd`" .next .next-r -npm start .next-r -p ${CERC_LISTEN_PORT:-3000} +mv .next .next.old +mv .next-r/.next . + +if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then + jq -e '.scripts.cerc_generate' package.json >/dev/null + if [ $? -eq 0 ]; then + npm run cerc_generate > gen.out 2>&1 & + tail -n0 -f gen.out | sed '/rendered as static HTML/ q' + count=0 + while [ $count -lt 10 ]; do + sleep 1 + ps -ef | grep 'node' | grep 'next' | grep 'generate' >/dev/null + if [ $? -ne 0 ]; then + break + else + count=$((count + 1)) + fi + done + kill $(ps -ef |grep node | grep next | grep generate | awk '{print $2}') 2>/dev/null + fi +fi + +npm start . -p ${CERC_LISTEN_PORT:-3000} From 1072fc98c3d6d1a82f3a8ff464356dc6af705d97 Mon Sep 17 00:00:00 2001 From: iskay Date: Fri, 10 Nov 2023 20:05:22 +0000 Subject: [PATCH 02/75] update fixturenet-optimism --- .../docker-compose-fixturenet-optimism.yml | 117 ++++++------ .../fixturenet-optimism/generate-l2-config.sh | 37 ---- .../optimism-contracts/deploy-contracts.sh | 172 ++++++++++++++++++ .../optimism-contracts/run.sh | 131 ------------- .../optimism-contracts/update-config.js | 36 ---- .../config/fixturenet-optimism/run-geth.sh | 155 ++++++++++++++++ .../fixturenet-optimism/run-op-batcher.sh | 26 +-- .../config/fixturenet-optimism/run-op-geth.sh | 84 +++------ .../config/fixturenet-optimism/run-op-node.sh | 51 ++++-- .../fixturenet-optimism/run-op-proposer.sh | 38 ++-- .../cerc-optimism-contracts/Dockerfile | 2 +- .../cerc-optimism-op-batcher/Dockerfile | 6 +- .../cerc-optimism-op-node/Dockerfile | 4 +- .../cerc-optimism-op-proposer/Dockerfile | 6 +- .../data/stacks/fixturenet-optimism/README.md | 113 ++++++++++-- .../fixturenet-optimism/deploy/commands.py | 37 ++++ .../stacks/fixturenet-optimism/l2-only.md | 79 +++++--- .../data/stacks/fixturenet-optimism/stack.yml | 4 +- 18 files changed, 656 insertions(+), 442 deletions(-) delete mode 100755 stack_orchestrator/data/config/fixturenet-optimism/generate-l2-config.sh create mode 100755 stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/deploy-contracts.sh delete mode 100755 stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/run.sh delete mode 100644 stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/update-config.js create mode 100755 stack_orchestrator/data/config/fixturenet-optimism/run-geth.sh create mode 100644 stack_orchestrator/data/stacks/fixturenet-optimism/deploy/commands.py diff --git a/stack_orchestrator/data/compose/docker-compose-fixturenet-optimism.yml b/stack_orchestrator/data/compose/docker-compose-fixturenet-optimism.yml index ddf7e290..fe1eac50 100644 --- a/stack_orchestrator/data/compose/docker-compose-fixturenet-optimism.yml +++ b/stack_orchestrator/data/compose/docker-compose-fixturenet-optimism.yml @@ -6,8 +6,8 @@ services: # Deploys the L1 smart contracts (outputs to volume l1_deployment) fixturenet-optimism-contracts: restart: on-failure - hostname: fixturenet-optimism-contracts image: cerc/optimism-contracts:local + hostname: fixturenet-optimism-contracts env_file: - ../config/fixturenet-optimism/l1-params.env environment: @@ -17,27 +17,49 @@ services: CERC_L1_ACCOUNTS_CSV_URL: ${CERC_L1_ACCOUNTS_CSV_URL} CERC_L1_ADDRESS: ${CERC_L1_ADDRESS} CERC_L1_PRIV_KEY: ${CERC_L1_PRIV_KEY} - CERC_L1_ADDRESS_2: ${CERC_L1_ADDRESS_2} - CERC_L1_PRIV_KEY_2: ${CERC_L1_PRIV_KEY_2} - # Waits for L1 endpoint to be up before running the script - command: | - "./wait-for-it.sh -h ${CERC_L1_HOST:-$${DEFAULT_CERC_L1_HOST}} -p ${CERC_L1_PORT:-$${DEFAULT_CERC_L1_PORT}} -s -t 60 -- ./run.sh" volumes: - ../config/network/wait-for-it.sh:/app/packages/contracts-bedrock/wait-for-it.sh - - ../config/optimism-contracts/hardhat-tasks/verify-contract-deployment.ts:/app/packages/contracts-bedrock/tasks/verify-contract-deployment.ts - - ../config/optimism-contracts/hardhat-tasks/rekey-json.ts:/app/packages/contracts-bedrock/tasks/rekey-json.ts - - ../config/optimism-contracts/hardhat-tasks/send-balance.ts:/app/packages/contracts-bedrock/tasks/send-balance.ts - - ../config/fixturenet-optimism/optimism-contracts/update-config.js:/app/packages/contracts-bedrock/update-config.js - - ../config/fixturenet-optimism/optimism-contracts/run.sh:/app/packages/contracts-bedrock/run.sh + - ../config/fixturenet-optimism/optimism-contracts/deploy-contracts.sh:/app/packages/contracts-bedrock/deploy-contracts.sh - l2_accounts:/l2-accounts - - l1_deployment:/app/packages/contracts-bedrock + - l1_deployment:/l1-deployment + - l2_config:/l2-config + # Waits for L1 endpoint to be up before running the contract deploy script + command: | + "./wait-for-it.sh -h ${CERC_L1_HOST:-$${DEFAULT_CERC_L1_HOST}} -p ${CERC_L1_PORT:-$${DEFAULT_CERC_L1_PORT}} -s -t 60 -- ./deploy-contracts.sh" + + # Initializes and runs the L2 execution client (outputs to volume l2_geth_data) + op-geth: + restart: always + image: cerc/optimism-l2geth:local + hostname: op-geth + depends_on: + op-node: + condition: service_started + volumes: + - ../config/fixturenet-optimism/run-op-geth.sh:/run-op-geth.sh + - l2_config:/l2-config:ro + - l2_accounts:/l2-accounts:ro + - l2_geth_data:/datadir + entrypoint: "sh" + command: "/run-op-geth.sh" + ports: + - "8545" + - "8546" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost:8545"] + interval: 30s + timeout: 10s + retries: 100 + start_period: 10s extra_hosts: - "host.docker.internal:host-gateway" - # Generates the config files required for L2 (outputs to volume l2_config) - op-node-l2-config-gen: - restart: on-failure + # Runs the L2 consensus client (Sequencer node) + # Generates the L2 config files if not already present (outputs to volume l2_config) + op-node: + restart: always image: cerc/optimism-op-node:local + hostname: op-node depends_on: fixturenet-optimism-contracts: condition: service_completed_successfully @@ -47,61 +69,19 @@ services: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_L1_RPC: ${CERC_L1_RPC} volumes: - - ../config/fixturenet-optimism/generate-l2-config.sh:/app/generate-l2-config.sh - - l1_deployment:/contracts-bedrock:ro - - l2_config:/app - command: ["sh", "/app/generate-l2-config.sh"] - extra_hosts: - - "host.docker.internal:host-gateway" - - # Initializes and runs the L2 execution client (outputs to volume l2_geth_data) - op-geth: - restart: always - image: cerc/optimism-l2geth:local - depends_on: - op-node-l2-config-gen: - condition: service_started - volumes: - - ../config/fixturenet-optimism/run-op-geth.sh:/run-op-geth.sh - - l2_config:/op-node:ro + - ../config/fixturenet-optimism/run-op-node.sh:/run-op-node.sh + - l1_deployment:/l1-deployment:ro + - l2_config:/l2-config - l2_accounts:/l2-accounts:ro - - l2_geth_data:/datadir entrypoint: "sh" - command: "/run-op-geth.sh" + command: "/run-op-node.sh" ports: - - "0.0.0.0:8545:8545" - - "0.0.0.0:8546:8546" - healthcheck: - test: ["CMD", "nc", "-vz", "localhost:8545"] - interval: 30s - timeout: 10s - retries: 10 - start_period: 10s - - # Runs the L2 consensus client (Sequencer node) - op-node: - restart: always - image: cerc/optimism-op-node:local - depends_on: - op-geth: - condition: service_healthy - env_file: - - ../config/fixturenet-optimism/l1-params.env - environment: - CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} - CERC_L1_RPC: ${CERC_L1_RPC} - volumes: - - ../config/fixturenet-optimism/run-op-node.sh:/app/run-op-node.sh - - l2_config:/op-node-data:ro - - l2_accounts:/l2-accounts:ro - command: ["sh", "/app/run-op-node.sh"] - ports: - - "0.0.0.0:8547:8547" + - "8547" healthcheck: test: ["CMD", "nc", "-vz", "localhost:8547"] interval: 30s timeout: 10s - retries: 10 + retries: 100 start_period: 10s extra_hosts: - "host.docker.internal:host-gateway" @@ -110,6 +90,7 @@ services: op-batcher: restart: always image: cerc/optimism-op-batcher:local + hostname: op-batcher depends_on: op-node: condition: service_healthy @@ -129,7 +110,7 @@ services: command: | "/wait-for-it.sh -h ${CERC_L1_HOST:-$${DEFAULT_CERC_L1_HOST}} -p ${CERC_L1_PORT:-$${DEFAULT_CERC_L1_PORT}} -s -t 60 -- /run-op-batcher.sh" ports: - - "127.0.0.1:8548:8548" + - "8548" extra_hosts: - "host.docker.internal:host-gateway" @@ -137,25 +118,29 @@ services: op-proposer: restart: always image: cerc/optimism-op-proposer:local + hostname: op-proposer depends_on: op-node: condition: service_healthy + op-geth: + condition: service_healthy env_file: - ../config/fixturenet-optimism/l1-params.env environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_L1_RPC: ${CERC_L1_RPC} + CERC_L1_CHAIN_ID: ${CERC_L1_CHAIN_ID} volumes: - ../config/network/wait-for-it.sh:/wait-for-it.sh - ../config/fixturenet-optimism/run-op-proposer.sh:/run-op-proposer.sh - - l1_deployment:/contracts-bedrock:ro + - l1_deployment:/l1-deployment:ro - l2_accounts:/l2-accounts:ro entrypoint: ["sh", "-c"] # Waits for L1 endpoint to be up before running the proposer command: | "/wait-for-it.sh -h ${CERC_L1_HOST:-$${DEFAULT_CERC_L1_HOST}} -p ${CERC_L1_PORT:-$${DEFAULT_CERC_L1_PORT}} -s -t 60 -- /run-op-proposer.sh" ports: - - "127.0.0.1:8560:8560" + - "8560" extra_hosts: - "host.docker.internal:host-gateway" diff --git a/stack_orchestrator/data/config/fixturenet-optimism/generate-l2-config.sh b/stack_orchestrator/data/config/fixturenet-optimism/generate-l2-config.sh deleted file mode 100755 index b10048d2..00000000 --- a/stack_orchestrator/data/config/fixturenet-optimism/generate-l2-config.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -set -e -if [ -n "$CERC_SCRIPT_DEBUG" ]; then - set -x -fi - -CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}" - -# Check existing config if it exists -if [ -f /app/jwt.txt ] && [ -f /app/rollup.json ]; then - echo "Found existing L2 config, cross-checking with L1 deployment config" - - SOURCE_L1_CONF=$(cat /contracts-bedrock/deploy-config/getting-started.json) - EXP_L1_BLOCKHASH=$(echo "$SOURCE_L1_CONF" | jq -r '.l1StartingBlockTag') - EXP_BATCHER=$(echo "$SOURCE_L1_CONF" | jq -r '.batchSenderAddress') - - GEN_L2_CONF=$(cat /app/rollup.json) - GEN_L1_BLOCKHASH=$(echo "$GEN_L2_CONF" | jq -r '.genesis.l1.hash') - GEN_BATCHER=$(echo "$GEN_L2_CONF" | jq -r '.genesis.system_config.batcherAddr') - - if [ "$EXP_L1_BLOCKHASH" = "$GEN_L1_BLOCKHASH" ] && [ "$EXP_BATCHER" = "$GEN_BATCHER" ]; then - echo "Config cross-checked, exiting" - exit 0 - fi - - echo "Existing L2 config doesn't match the L1 deployment config, please clear L2 config volume before starting" - exit 1 -fi - -op-node genesis l2 \ - --deploy-config /contracts-bedrock/deploy-config/getting-started.json \ - --deployment-dir /contracts-bedrock/deployments/getting-started/ \ - --outfile.l2 /app/genesis.json \ - --outfile.rollup /app/rollup.json \ - --l1-rpc $CERC_L1_RPC - -openssl rand -hex 32 > /app/jwt.txt diff --git a/stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/deploy-contracts.sh b/stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/deploy-contracts.sh new file mode 100755 index 00000000..23a2bc30 --- /dev/null +++ b/stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/deploy-contracts.sh @@ -0,0 +1,172 @@ +#!/bin/bash +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +CERC_L1_CHAIN_ID="${CERC_L1_CHAIN_ID:-${DEFAULT_CERC_L1_CHAIN_ID}}" +CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}" + +CERC_L1_ACCOUNTS_CSV_URL="${CERC_L1_ACCOUNTS_CSV_URL:-${DEFAULT_CERC_L1_ACCOUNTS_CSV_URL}}" + +export DEPLOYMENT_CONTEXT="$CERC_L1_CHAIN_ID" +# Optional create2 salt for deterministic deployment of contract implementations +export IMPL_SALT=$(openssl rand -hex 32) + +echo "Using L1 RPC endpoint ${CERC_L1_RPC}" + +# Exit if a deployment already exists (on restarts) +if [ -d "/l1-deployment/$DEPLOYMENT_CONTEXT" ]; then + echo "Deployment directory /l1-deployment/$DEPLOYMENT_CONTEXT, checking OptimismPortal deployment" + + OPTIMISM_PORTAL_ADDRESS=$(cat /l1-deployment/$DEPLOYMENT_CONTEXT/OptimismPortal.json | jq -r .address) + contract_code=$(cast code $OPTIMISM_PORTAL_ADDRESS --rpc-url $CERC_L1_RPC) + + if [ -z "${contract_code#0x}" ]; then + echo "Error: A deployment directory was found in the volume, but no contract code was found on-chain at the associated address. Please clear L1 deployment volume before restarting." + exit 1 + else + echo "Deployment found, exiting (successfully)." + exit 0 + fi +fi + +wait_for_block() { + local block="$1" # Block to wait for + local timeout="$2" # Max time to wait in seconds + + echo "Waiting for block $block." + i=0 + loops=$(($timeout/10)) + while [ -z "$block_result" ] && [[ "$i" -lt "$loops" ]]; do + sleep 10 + echo "Checking..." + block_result=$(cast block $block --rpc-url $CERC_L1_RPC | grep -E "(timestamp|hash|number)" || true) + i=$(($i + 1)) + done +} + +# We need four accounts and their private keys for the deployment: Admin, Proposer, Batcher, and Sequencer +# If $CERC_L1_ADDRESS and $CERC_L1_PRIV_KEY have been set, we'll assign it to Admin and generate/fund the remaining three accounts from it +# If not, we'll assume the L1 is the stack's own fixturenet-eth and use the pre-funded accounts/keys from $CERC_L1_ACCOUNTS_CSV_URL +if [ -n "$CERC_L1_ADDRESS" ] && [ -n "$CERC_L1_PRIV_KEY" ]; then + wallet1=$(cast wallet new) + wallet2=$(cast wallet new) + wallet3=$(cast wallet new) + # Admin + ADMIN=$CERC_L1_ADDRESS + ADMIN_KEY=$CERC_L1_PRIV_KEY + # Proposer + PROPOSER=$(echo "$wallet1" | awk '/Address:/{print $2}') + PROPOSER_KEY=$(echo "$wallet1" | awk '/Private key:/{print $3}') + # Batcher + BATCHER=$(echo "$wallet2" | awk '/Address:/{print $2}') + BATCHER_KEY=$(echo "$wallet2" | awk '/Private key:/{print $3}') + # Sequencer + SEQ=$(echo "$wallet3" | awk '/Address:/{print $2}') + SEQ_KEY=$(echo "$wallet3" | awk '/Private key:/{print $3}') + + echo "Funding accounts." + wait_for_block 1 300 + cast send --from $ADMIN --rpc-url $CERC_L1_RPC --value 5ether $PROPOSER --private-key $ADMIN_KEY + cast send --from $ADMIN --rpc-url $CERC_L1_RPC --value 10ether $BATCHER --private-key $ADMIN_KEY + cast send --from $ADMIN --rpc-url $CERC_L1_RPC --value 2ether $SEQ --private-key $ADMIN_KEY +else + curl -o accounts.csv $CERC_L1_ACCOUNTS_CSV_URL + # Admin + ADMIN=$(awk -F ',' 'NR == 1 {print $2}' accounts.csv) + ADMIN_KEY=$(awk -F ',' 'NR == 1 {print $3}' accounts.csv) + # Proposer + PROPOSER=$(awk -F ',' 'NR == 2 {print $2}' accounts.csv) + PROPOSER_KEY=$(awk -F ',' 'NR == 2 {print $3}' accounts.csv) + # Batcher + BATCHER=$(awk -F ',' 'NR == 3 {print $2}' accounts.csv) + BATCHER_KEY=$(awk -F ',' 'NR == 3 {print $3}' accounts.csv) + # Sequencer + SEQ=$(awk -F ',' 'NR == 4 {print $2}' accounts.csv) + SEQ_KEY=$(awk -F ',' 'NR == 4 {print $3}' accounts.csv) +fi + +echo "Using accounts:" +echo -e "Admin: $ADMIN\nProposer: $PROPOSER\nBatcher: $BATCHER\nSequencer: $SEQ" + +# These accounts will be needed by other containers, so write them to a shared volume +echo "Writing accounts/private keys to volume l2_accounts." +accounts_json=$(jq -n \ + --arg Admin "$ADMIN" --arg AdminKey "$ADMIN_KEY" \ + --arg Proposer "$PROPOSER" --arg ProposerKey "$PROPOSER_KEY" \ + --arg Batcher "$BATCHER" --arg BatcherKey "$BATCHER_KEY" \ + --arg Seq "$SEQ" --arg SeqKey "$SEQ_KEY" \ + '{Admin: $Admin, AdminKey: $AdminKey, Proposer: $Proposer, ProposerKey: $ProposerKey, Batcher: $Batcher, BatcherKey: $BatcherKey, Seq: $Seq, SeqKey: $SeqKey}') +echo "$accounts_json" > "/l2-accounts/accounts.json" + +# Get a finalized L1 block to set as the starting point for the L2 deployment +# If the chain is a freshly created fixturenet-eth, a finalized block won't be available for many minutes; rather than wait, we can use block 1 +echo "Checking L1 for finalized block..." +finalized=$(cast block finalized --rpc-url $CERC_L1_RPC | grep -E "(timestamp|hash|number)" || true) + +if [ -n "$finalized" ]; then + # finalized block was found + start_block=$finalized +else + # assume fresh chain and use block 1 instead + echo "No finalized block. Using block 1 instead." + # wait for 20 or so blocks to be safe + wait_for_block 24 300 + start_block=$(cast block 1 --rpc-url $CERC_L1_RPC | grep -E "(timestamp|hash|number)" || true) +fi + +if [ -z "$start_block" ]; then + echo "Unable to query chain for starting block. Exiting..." + exit 1 +fi + +BLOCKHASH=$(echo $start_block | awk -F ' ' '{print $2}') +HEIGHT=$(echo $start_block | awk -F ' ' '{print $4}') +TIMESTAMP=$(echo $start_block | awk -F ' ' '{print $6}') + +echo "Using block as deployment point:" +echo "Height: $HEIGHT" +echo "Hash: $BLOCKHASH" +echo "Timestamp: $TIMESTAMP" + +# Fill out the deployment template (./deploy-config/getting-started.json) with our values: +echo "Writing deployment config." +deploy_config_file="deploy-config/$DEPLOYMENT_CONTEXT.json" +cp deploy-config/getting-started.json $deploy_config_file +sed -i "s/\"l1ChainID\": .*/\"l1ChainID\": $DEPLOYMENT_CONTEXT,/g" $deploy_config_file +sed -i "s/ADMIN/$ADMIN/g" $deploy_config_file +sed -i "s/PROPOSER/$PROPOSER/g" $deploy_config_file +sed -i "s/BATCHER/$BATCHER/g" $deploy_config_file +sed -i "s/SEQUENCER/$SEQ/g" $deploy_config_file +sed -i "s/BLOCKHASH/$BLOCKHASH/g" $deploy_config_file +sed -i "s/TIMESTAMP/$TIMESTAMP/g" $deploy_config_file + +mkdir -p deployments/$DEPLOYMENT_CONTEXT + +# Deployment requires the create2 deterministic proxy contract be published on L1 at address 0x4e59b44847b379578588920ca78fbf26c0b4956c +# See: https://github.com/Arachnid/deterministic-deployment-proxy +echo "Deploying create2 proxy contract..." +echo "Funding deployment signer address" +deployment_signer="0x3fab184622dc19b6109349b94811493bf2a45362" +cast send --from $ADMIN --rpc-url $CERC_L1_RPC --value 0.5ether $deployment_signer --private-key $ADMIN_KEY +echo "Deploying contract..." +raw_bytes="0xf8a58085174876e800830186a08080b853604580600e600039806000f350fe7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe03601600081602082378035828234f58015156039578182fd5b8082525050506014600cf31ba02222222222222222222222222222222222222222222222222222222222222222a02222222222222222222222222222222222222222222222222222222222222222" + +cast publish --rpc-url $CERC_L1_RPC $raw_bytes + +# Create the L2 deployment +echo "Deploying L1 Optimism contracts..." +forge script scripts/Deploy.s.sol:Deploy --private-key $ADMIN_KEY --broadcast --rpc-url $CERC_L1_RPC +forge script scripts/Deploy.s.sol:Deploy --sig 'sync()' --private-key $ADMIN_KEY --broadcast --rpc-url $CERC_L1_RPC + +echo "*************************************" +echo "Done deploying contracts." + +# Copy files needed by other containers to the appropriate shared volumes +echo "Copying deployment artifacts volume l1_deployment and deploy-config to volume l2_config" +cp -a /app/packages/contracts-bedrock/deployments/$DEPLOYMENT_CONTEXT /l1-deployment +cp /app/packages/contracts-bedrock/deploy-config/$DEPLOYMENT_CONTEXT.json /l2-config +openssl rand -hex 32 > /l2-config/l2-jwt.txt + +echo "Deployment successful. Exiting" diff --git a/stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/run.sh b/stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/run.sh deleted file mode 100755 index d878c03f..00000000 --- a/stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/run.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/bin/bash -set -e -if [ -n "$CERC_SCRIPT_DEBUG" ]; then - set -x -fi - -CERC_L1_CHAIN_ID="${CERC_L1_CHAIN_ID:-${DEFAULT_CERC_L1_CHAIN_ID}}" -CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}" - -CERC_L1_ACCOUNTS_CSV_URL="${CERC_L1_ACCOUNTS_CSV_URL:-${DEFAULT_CERC_L1_ACCOUNTS_CSV_URL}}" - -echo "Using L1 RPC endpoint ${CERC_L1_RPC}" - -IMPORT_1="import './verify-contract-deployment'" -IMPORT_2="import './rekey-json'" -IMPORT_3="import './send-balance'" - -# Append mounted tasks to tasks/index.ts file if not present -if ! grep -Fxq "$IMPORT_1" tasks/index.ts; then - echo "$IMPORT_1" >> tasks/index.ts - echo "$IMPORT_2" >> tasks/index.ts - echo "$IMPORT_3" >> tasks/index.ts -fi - -# Update the chainId in the hardhat config -sed -i "/getting-started/ {n; s/.*chainId.*/ chainId: $CERC_L1_CHAIN_ID,/}" hardhat.config.ts - -# Exit if a deployment already exists (on restarts) -# Note: fixturenet-eth-geth currently starts fresh on a restart -if [ -d "deployments/getting-started" ]; then - echo "Deployment directory deployments/getting-started found, checking SystemDictator deployment" - - # Read JSON file into variable - SYSTEM_DICTATOR_DETAILS=$(cat deployments/getting-started/SystemDictator.json) - - # Parse JSON into variables - SYSTEM_DICTATOR_ADDRESS=$(echo "$SYSTEM_DICTATOR_DETAILS" | jq -r '.address') - SYSTEM_DICTATOR_TXHASH=$(echo "$SYSTEM_DICTATOR_DETAILS" | jq -r '.transactionHash') - - if yarn hardhat verify-contract-deployment --contract "${SYSTEM_DICTATOR_ADDRESS}" --transaction-hash "${SYSTEM_DICTATOR_TXHASH}"; then - echo "Deployment verfication successful, exiting" - exit 0 - else - echo "Deployment verfication failed, please clear L1 deployment volume before starting" - exit 1 - fi -fi - -# Generate the L2 account addresses -yarn hardhat rekey-json --output /l2-accounts/keys.json - -# Read JSON file into variable -KEYS_JSON=$(cat /l2-accounts/keys.json) - -# Parse JSON into variables -ADMIN_ADDRESS=$(echo "$KEYS_JSON" | jq -r '.Admin.address') -ADMIN_PRIV_KEY=$(echo "$KEYS_JSON" | jq -r '.Admin.privateKey') -PROPOSER_ADDRESS=$(echo "$KEYS_JSON" | jq -r '.Proposer.address') -BATCHER_ADDRESS=$(echo "$KEYS_JSON" | jq -r '.Batcher.address') -SEQUENCER_ADDRESS=$(echo "$KEYS_JSON" | jq -r '.Sequencer.address') - -# Get the private keys of L1 accounts -if [ -n "$CERC_L1_ACCOUNTS_CSV_URL" ] && \ - l1_accounts_response=$(curl -L --write-out '%{http_code}' --silent --output /dev/null "$CERC_L1_ACCOUNTS_CSV_URL") && \ - [ "$l1_accounts_response" -eq 200 ]; -then - echo "Fetching L1 account credentials using provided URL" - mkdir -p /geth-accounts - wget -O /geth-accounts/accounts.csv "$CERC_L1_ACCOUNTS_CSV_URL" - - CERC_L1_ADDRESS=$(head -n 1 /geth-accounts/accounts.csv | cut -d ',' -f 2) - CERC_L1_PRIV_KEY=$(head -n 1 /geth-accounts/accounts.csv | cut -d ',' -f 3) - CERC_L1_ADDRESS_2=$(awk -F, 'NR==2{print $(NF-1)}' /geth-accounts/accounts.csv) - CERC_L1_PRIV_KEY_2=$(awk -F, 'NR==2{print $NF}' /geth-accounts/accounts.csv) -else - echo "Couldn't fetch L1 account credentials, using them from env" -fi - -# Send balances to the above L2 addresses -yarn hardhat send-balance --to "${ADMIN_ADDRESS}" --amount 2 --private-key "${CERC_L1_PRIV_KEY}" --network getting-started -yarn hardhat send-balance --to "${PROPOSER_ADDRESS}" --amount 5 --private-key "${CERC_L1_PRIV_KEY}" --network getting-started -yarn hardhat send-balance --to "${BATCHER_ADDRESS}" --amount 1000 --private-key "${CERC_L1_PRIV_KEY}" --network getting-started - -echo "Balances sent to L2 accounts" - -# Select a finalized L1 block as the starting point for roll ups -until FINALIZED_BLOCK=$(cast block finalized --rpc-url "$CERC_L1_RPC"); do - echo "Waiting for a finalized L1 block to exist, retrying after 10s" - sleep 10 -done - -L1_BLOCKNUMBER=$(echo "$FINALIZED_BLOCK" | awk '/number/{print $2}') -L1_BLOCKHASH=$(echo "$FINALIZED_BLOCK" | awk '/hash/{print $2}') -L1_BLOCKTIMESTAMP=$(echo "$FINALIZED_BLOCK" | awk '/timestamp/{print $2}') - -echo "Selected L1 block ${L1_BLOCKNUMBER} as the starting block for roll ups" - -# Update the deployment config -sed -i 's/"l2OutputOracleStartingTimestamp": TIMESTAMP/"l2OutputOracleStartingTimestamp": '"$L1_BLOCKTIMESTAMP"'/g' deploy-config/getting-started.json -jq --arg chainid "$CERC_L1_CHAIN_ID" '.l1ChainID = ($chainid | tonumber)' deploy-config/getting-started.json > tmp.json && mv tmp.json deploy-config/getting-started.json - -node update-config.js deploy-config/getting-started.json "$ADMIN_ADDRESS" "$PROPOSER_ADDRESS" "$BATCHER_ADDRESS" "$SEQUENCER_ADDRESS" "$L1_BLOCKHASH" - -echo "Updated the deployment config" - -# Create a .env file -echo "L1_RPC=$CERC_L1_RPC" > .env -echo "PRIVATE_KEY_DEPLOYER=$ADMIN_PRIV_KEY" >> .env - -echo "Deploying the L1 smart contracts, this will take a while..." - -# Deploy the L1 smart contracts -yarn hardhat deploy --network getting-started --tags l1 - -echo "Deployed the L1 smart contracts" - -# Read Proxy contract's JSON and get the address -PROXY_JSON=$(cat deployments/getting-started/Proxy__OVM_L1StandardBridge.json) -PROXY_ADDRESS=$(echo "$PROXY_JSON" | jq -r '.address') - -# Send balance to the above Proxy contract in L1 for reflecting balance in L2 -# First account -yarn hardhat send-balance --to "${PROXY_ADDRESS}" --amount 1 --private-key "${CERC_L1_PRIV_KEY}" --network getting-started -# Second account -yarn hardhat send-balance --to "${PROXY_ADDRESS}" --amount 1 --private-key "${CERC_L1_PRIV_KEY_2}" --network getting-started - -echo "Balance sent to Proxy L2 contract" -echo "Use following accounts for transactions in L2:" -echo "${CERC_L1_ADDRESS}" -echo "${CERC_L1_ADDRESS_2}" -echo "Done" diff --git a/stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/update-config.js b/stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/update-config.js deleted file mode 100644 index 8a6c09d4..00000000 --- a/stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/update-config.js +++ /dev/null @@ -1,36 +0,0 @@ -const fs = require('fs') - -// Get the command-line argument -const configFile = process.argv[2] -const adminAddress = process.argv[3] -const proposerAddress = process.argv[4] -const batcherAddress = process.argv[5] -const sequencerAddress = process.argv[6] -const blockHash = process.argv[7] - -// Read the JSON file -const configData = fs.readFileSync(configFile) -const configObj = JSON.parse(configData) - -// Update the finalSystemOwner property with the ADMIN_ADDRESS value -configObj.finalSystemOwner = - configObj.portalGuardian = - configObj.controller = - configObj.l2OutputOracleChallenger = - configObj.proxyAdminOwner = - configObj.baseFeeVaultRecipient = - configObj.l1FeeVaultRecipient = - configObj.sequencerFeeVaultRecipient = - configObj.governanceTokenOwner = - adminAddress - -configObj.l2OutputOracleProposer = proposerAddress - -configObj.batchSenderAddress = batcherAddress - -configObj.p2pSequencerAddress = sequencerAddress - -configObj.l1StartingBlockTag = blockHash - -// Write the updated JSON object back to the file -fs.writeFileSync(configFile, JSON.stringify(configObj, null, 2)) diff --git a/stack_orchestrator/data/config/fixturenet-optimism/run-geth.sh b/stack_orchestrator/data/config/fixturenet-optimism/run-geth.sh new file mode 100755 index 00000000..b24fe867 --- /dev/null +++ b/stack_orchestrator/data/config/fixturenet-optimism/run-geth.sh @@ -0,0 +1,155 @@ +#!/bin/bash +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +# To facilitate deploying the Optimism contracts, a few additional arguments have been added to the geth start command +# Otherwise this script is unchanged from the image's default startup script + +ETHERBASE=`cat /opt/testnet/build/el/accounts.csv | head -1 | cut -d',' -f2` +NETWORK_ID=`cat /opt/testnet/el/el-config.yaml | grep 'chain_id' | awk '{ print $2 }'` +NETRESTRICT=`ip addr | grep inet | grep -v '127.0' | awk '{print $2}'` +CERC_ETH_DATADIR="${CERC_ETH_DATADIR:-$HOME/ethdata}" +CERC_PLUGINS_DIR="${CERC_PLUGINS_DIR:-/usr/local/lib/plugeth}" + +cd /opt/testnet/build/el +python3 -m http.server 9898 & +cd $HOME + +START_CMD="geth" +if [ "true" == "$CERC_REMOTE_DEBUG" ] && [ -x "/usr/local/bin/dlv" ]; then + START_CMD="/usr/local/bin/dlv --listen=:40000 --headless=true --api-version=2 --accept-multiclient exec /usr/local/bin/geth --continue --" +fi + +# See https://linuxconfig.org/how-to-propagate-a-signal-to-child-processes-from-a-bash-script +cleanup() { + echo "Signal received, cleaning up..." + + # Kill the child process first (CERC_REMOTE_DEBUG=true uses dlv which starts geth as a child process) + pkill -P ${geth_pid} + sleep 2 + kill $(jobs -p) + + wait + echo "Done" +} +trap 'cleanup' SIGINT SIGTERM + +if [ "true" == "$RUN_BOOTNODE" ]; then + $START_CMD \ + --datadir="${CERC_ETH_DATADIR}" \ + --nodekeyhex="${BOOTNODE_KEY}" \ + --nodiscover \ + --ipcdisable \ + --networkid=${NETWORK_ID} \ + --netrestrict="${NETRESTRICT}" \ + & + + geth_pid=$! +else + cd /opt/testnet/accounts + ./import_keys.sh + + echo -n "$JWT" > /opt/testnet/build/el/jwtsecret + + if [ "$CERC_RUN_STATEDIFF" == "detect" ] && [ -n "$CERC_STATEDIFF_DB_HOST" ]; then + dig_result=$(dig $CERC_STATEDIFF_DB_HOST +short) + dig_status_code=$? + if [[ $dig_status_code = 0 && -n $dig_result ]]; then + echo "Statediff DB at $CERC_STATEDIFF_DB_HOST" + CERC_RUN_STATEDIFF="true" + else + echo "No statediff DB available." + CERC_RUN_STATEDIFF="false" + fi + fi + + STATEDIFF_OPTS="" + if [ "$CERC_RUN_STATEDIFF" == "true" ]; then + ready=0 + echo "Waiting for statediff DB..." + while [ $ready -eq 0 ]; do + sleep 1 + export PGPASSWORD="$CERC_STATEDIFF_DB_PASSWORD" + result=$(psql -h "$CERC_STATEDIFF_DB_HOST" \ + -p "$CERC_STATEDIFF_DB_PORT" \ + -U "$CERC_STATEDIFF_DB_USER" \ + -d "$CERC_STATEDIFF_DB_NAME" \ + -t -c 'select max(version_id) from goose_db_version;' 2>/dev/null | awk '{ print $1 }') + if [ -n "$result" ]; then + echo "DB ready..." + if [ $result -ge $CERC_STATEDIFF_DB_GOOSE_MIN_VER ]; then + ready=1 + else + echo "DB not at required version (want $CERC_STATEDIFF_DB_GOOSE_MIN_VER, have $result)" + fi + fi + done + STATEDIFF_OPTS="--statediff \ + --statediff.db.host=$CERC_STATEDIFF_DB_HOST \ + --statediff.db.name=$CERC_STATEDIFF_DB_NAME \ + --statediff.db.nodeid=$CERC_STATEDIFF_DB_NODE_ID \ + --statediff.db.password=$CERC_STATEDIFF_DB_PASSWORD \ + --statediff.db.port=$CERC_STATEDIFF_DB_PORT \ + --statediff.db.user=$CERC_STATEDIFF_DB_USER \ + --statediff.db.logstatements=${CERC_STATEDIFF_DB_LOG_STATEMENTS:-false} \ + --statediff.db.copyfrom=${CERC_STATEDIFF_DB_COPY_FROM:-true} \ + --statediff.waitforsync=true \ + --statediff.workers=${CERC_STATEDIFF_WORKERS:-1} \ + --statediff.writing=true" + + if [ -d "${CERC_PLUGINS_DIR}" ]; then + # With plugeth, we separate the statediff options by prefixing with ' -- ' + STATEDIFF_OPTS="--pluginsdir "${CERC_PLUGINS_DIR}" -- ${STATEDIFF_OPTS}" + fi + fi + + # unlock account[0] + echo $ACCOUNT_PASSWORD > "$CERC_ETH_DATADIR/password" + + $START_CMD \ + --datadir="${CERC_ETH_DATADIR}" \ + --bootnodes="${ENODE}" \ + --allow-insecure-unlock \ + --password="${CERC_ETH_DATADIR}/password" \ + --unlock="$ETHERBASE" \ + --rpc.allow-unprotected-txs \ + --http \ + --http.addr="0.0.0.0" \ + --http.vhosts="*" \ + --http.api="${CERC_GETH_HTTP_APIS:-eth,web3,net,admin,personal,debug,statediff}" \ + --http.corsdomain="*" \ + --authrpc.addr="0.0.0.0" \ + --authrpc.vhosts="*" \ + --authrpc.jwtsecret="/opt/testnet/build/el/jwtsecret" \ + --ws \ + --ws.addr="0.0.0.0" \ + --ws.origins="*" \ + --ws.api="${CERC_GETH_WS_APIS:-eth,web3,net,admin,personal,debug,statediff}" \ + --http.corsdomain="*" \ + --networkid="${NETWORK_ID}" \ + --netrestrict="${NETRESTRICT}" \ + --gcmode archive \ + --txlookuplimit=0 \ + --cache.preimages \ + --syncmode=full \ + --mine \ + --miner.threads=1 \ + --metrics \ + --metrics.addr="0.0.0.0" \ + --verbosity=${CERC_GETH_VERBOSITY:-3} \ + --log.vmodule="${CERC_GETH_VMODULE:-statediff/*=5}" \ + --miner.etherbase="${ETHERBASE}" \ + ${STATEDIFF_OPTS} \ + & + + geth_pid=$! +fi + +wait $geth_pid + +if [ "true" == "$CERC_KEEP_RUNNING_AFTER_GETH_EXIT" ]; then + while [ 1 -eq 1 ]; do + sleep 60 + done +fi diff --git a/stack_orchestrator/data/config/fixturenet-optimism/run-op-batcher.sh b/stack_orchestrator/data/config/fixturenet-optimism/run-op-batcher.sh index 18955545..29a65d5d 100755 --- a/stack_orchestrator/data/config/fixturenet-optimism/run-op-batcher.sh +++ b/stack_orchestrator/data/config/fixturenet-optimism/run-op-batcher.sh @@ -6,22 +6,14 @@ fi CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}" -# Get Batcher key from keys.json -BATCHER_KEY=$(jq -r '.Batcher.privateKey' /l2-accounts/keys.json | tr -d '"') +# Start op-batcher +L2_RPC="http://op-geth:8545" +ROLLUP_RPC="http://op-node:8547" +BATCHER_KEY=$(cat /l2-accounts/accounts.json | jq -r .BatcherKey) -cleanup() { - echo "Signal received, cleaning up..." - kill ${batcher_pid} - - wait - echo "Done" -} -trap 'cleanup' INT TERM - -# Run op-batcher op-batcher \ - --l2-eth-rpc=http://op-geth:8545 \ - --rollup-rpc=http://op-node:8547 \ + --l2-eth-rpc=$L2_RPC \ + --rollup-rpc=$ROLLUP_RPC \ --poll-interval=1s \ --sub-safety-margin=6 \ --num-confirmations=1 \ @@ -32,8 +24,4 @@ op-batcher \ --rpc.enable-admin \ --max-channel-duration=1 \ --l1-eth-rpc=$CERC_L1_RPC \ - --private-key=$BATCHER_KEY \ - & - -batcher_pid=$! -wait $batcher_pid + --private-key="${BATCHER_KEY#0x}" diff --git a/stack_orchestrator/data/config/fixturenet-optimism/run-op-geth.sh b/stack_orchestrator/data/config/fixturenet-optimism/run-op-geth.sh index 8b521f85..9b06cedc 100755 --- a/stack_orchestrator/data/config/fixturenet-optimism/run-op-geth.sh +++ b/stack_orchestrator/data/config/fixturenet-optimism/run-op-geth.sh @@ -4,61 +4,36 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi -# TODO: Add in container build or use other tool -echo "Installing jq" -apk update && apk add jq +l2_genesis_file="/l2-config/genesis.json" -# Get Sequencer key from keys.json -SEQUENCER_KEY=$(jq -r '.Sequencer.privateKey' /l2-accounts/keys.json | tr -d '"') +# Check for genesis file; if necessary, wait on op-node to generate +timeout=300 # 5 minutes +start_time=$(date +%s) +elapsed_time=0 +echo "Checking for L2 genesis file at location $l2_genesis_file" +while [ ! -f "$l2_genesis_file" ] && [ $elapsed_time -lt $timeout ]; do + echo "Waiting for L2 genesis file to be generated..." + sleep 10 + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) +done -# Initialize op-geth if datadir/geth not found -if [ -f /op-node/jwt.txt ] && [ -d datadir/geth ]; then - echo "Found existing datadir, checking block signer key" - - BLOCK_SIGNER_KEY=$(cat datadir/block-signer-key) - - if [ "$SEQUENCER_KEY" = "$BLOCK_SIGNER_KEY" ]; then - echo "Sequencer and block signer keys match, skipping initialization" - else - echo "Sequencer and block signer keys don't match, please clear L2 geth data volume before starting" - exit 1 - fi -else - echo "Initializing op-geth" - - mkdir -p datadir - echo "pwd" > datadir/password - echo $SEQUENCER_KEY > datadir/block-signer-key - - geth account import --datadir=datadir --password=datadir/password datadir/block-signer-key - - while [ ! -f "/op-node/jwt.txt" ] - do - echo "Config files not created. Checking after 5 seconds." - sleep 5 - done - - echo "Config files created by op-node, proceeding with the initialization..." - - geth init --datadir=datadir /op-node/genesis.json - echo "Node Initialized" +if [ ! -f "$l2_genesis_file" ]; then + echo "L2 genesis file not found after timeout of $timeout seconds. Exiting..." + exit 1 fi -SEQUENCER_ADDRESS=$(jq -r '.Sequencer.address' /l2-accounts/keys.json | tr -d '"') -echo "SEQUENCER_ADDRESS: ${SEQUENCER_ADDRESS}" +# Initialize geth from our generated L2 genesis file (if not already initialized) +data_dir="/datadir" +if [ ! -d "$datadir/geth" ]; then + geth init --datadir=$data_dir $l2_genesis_file +fi -cleanup() { - echo "Signal received, cleaning up..." - kill ${geth_pid} +# Start op-geth +jwt_file="/l2-config/l2-jwt.txt" - wait - echo "Done" -} -trap 'cleanup' INT TERM - -# Run op-geth geth \ - --datadir ./datadir \ + --datadir=$data_dir \ --http \ --http.corsdomain="*" \ --http.vhosts="*" \ @@ -77,14 +52,5 @@ geth \ --authrpc.vhosts="*" \ --authrpc.addr=0.0.0.0 \ --authrpc.port=8551 \ - --authrpc.jwtsecret=/op-node/jwt.txt \ - --rollup.disabletxpoolgossip=true \ - --password=./datadir/password \ - --allow-insecure-unlock \ - --mine \ - --miner.etherbase=$SEQUENCER_ADDRESS \ - --unlock=$SEQUENCER_ADDRESS \ - & - -geth_pid=$! -wait $geth_pid + --authrpc.jwtsecret=$jwt_file \ + --rollup.disabletxpoolgossip=true diff --git a/stack_orchestrator/data/config/fixturenet-optimism/run-op-node.sh b/stack_orchestrator/data/config/fixturenet-optimism/run-op-node.sh index 516cf0a5..60a96855 100755 --- a/stack_orchestrator/data/config/fixturenet-optimism/run-op-node.sh +++ b/stack_orchestrator/data/config/fixturenet-optimism/run-op-node.sh @@ -4,23 +4,42 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi +CERC_L1_CHAIN_ID="${CERC_L1_CHAIN_ID:-${DEFAULT_CERC_L1_CHAIN_ID}}" CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}" +DEPLOYMENT_CONTEXT="$CERC_L1_CHAIN_ID" -# Get Sequencer key from keys.json -SEQUENCER_KEY=$(jq -r '.Sequencer.privateKey' /l2-accounts/keys.json | tr -d '"') +deploy_config_file="/l2-config/$DEPLOYMENT_CONTEXT.json" +deployment_dir="/l1-deployment/$DEPLOYMENT_CONTEXT" +genesis_outfile="/l2-config/genesis.json" +rollup_outfile="/l2-config/rollup.json" + +# Generate L2 genesis (if not already done) +if [ ! -f "$genesis_outfile" ] || [ ! -f "$rollup_outfile" ]; then + op-node genesis l2 \ + --deploy-config $deploy_config_file \ + --deployment-dir $deployment_dir \ + --outfile.l2 $genesis_outfile \ + --outfile.rollup $rollup_outfile \ + --l1-rpc $CERC_L1_RPC +fi + +# Start op-node +SEQ_KEY=$(cat /l2-accounts/accounts.json | jq -r .SeqKey) +jwt_file=/l2-config/l2-jwt.txt +L2_AUTH="http://op-geth:8551" +RPC_KIND=any # this can optionally be set to a preset for common node providers like Infura, Alchemy, etc. -# Run op-node op-node \ - --l2=http://op-geth:8551 \ - --l2.jwt-secret=/op-node-data/jwt.txt \ - --sequencer.enabled \ - --sequencer.l1-confs=3 \ - --verifier.l1-confs=3 \ - --rollup.config=/op-node-data/rollup.json \ - --rpc.addr=0.0.0.0 \ - --rpc.port=8547 \ - --p2p.disable \ - --rpc.enable-admin \ - --p2p.sequencer.key=$SEQUENCER_KEY \ - --l1=$CERC_L1_RPC \ - --l1.rpckind=any + --l2=$L2_AUTH \ + --l2.jwt-secret=$jwt_file \ + --sequencer.enabled \ + --sequencer.l1-confs=5 \ + --verifier.l1-confs=4 \ + --rollup.config=$rollup_outfile \ + --rpc.addr=0.0.0.0 \ + --rpc.port=8547 \ + --p2p.disable \ + --rpc.enable-admin \ + --p2p.sequencer.key="${SEQ_KEY#0x}" \ + --l1=$CERC_L1_RPC \ + --l1.rpckind=$RPC_KIND diff --git a/stack_orchestrator/data/config/fixturenet-optimism/run-op-proposer.sh b/stack_orchestrator/data/config/fixturenet-optimism/run-op-proposer.sh index 09746760..092705ca 100755 --- a/stack_orchestrator/data/config/fixturenet-optimism/run-op-proposer.sh +++ b/stack_orchestrator/data/config/fixturenet-optimism/run-op-proposer.sh @@ -5,32 +5,18 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then fi CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}" +CERC_L1_CHAIN_ID="${CERC_L1_CHAIN_ID:-${DEFAULT_CERC_L1_CHAIN_ID}}" +DEPLOYMENT_CONTEXT="$CERC_L1_CHAIN_ID" -# Read the L2OutputOracle contract address from the deployment -L2OO_DEPLOYMENT=$(cat /contracts-bedrock/deployments/getting-started/L2OutputOracle.json) -L2OO_ADDR=$(echo "$L2OO_DEPLOYMENT" | jq -r '.address') +# Start op-proposer +ROLLUP_RPC="http://op-node:8547" +PROPOSER_KEY=$(cat /l2-accounts/accounts.json | jq -r .ProposerKey) +L2OO_ADDR=$(cat /l1-deployment/$DEPLOYMENT_CONTEXT/L2OutputOracleProxy.json | jq -r .address) -# Get Proposer key from keys.json -PROPOSER_KEY=$(jq -r '.Proposer.privateKey' /l2-accounts/keys.json | tr -d '"') - -cleanup() { - echo "Signal received, cleaning up..." - kill ${proposer_pid} - - wait - echo "Done" -} -trap 'cleanup' INT TERM - -# Run op-proposer op-proposer \ - --poll-interval 12s \ - --rpc.port 8560 \ - --rollup-rpc http://op-node:8547 \ - --l2oo-address $L2OO_ADDR \ - --private-key $PROPOSER_KEY \ - --l1-eth-rpc $CERC_L1_RPC \ - & - -proposer_pid=$! -wait $proposer_pid + --poll-interval=12s \ + --rpc.port=8560 \ + --rollup-rpc=$ROLLUP_RPC \ + --l2oo-address="${L2OO_ADDR#0x}" \ + --private-key="${PROPOSER_KEY#0x}" \ + --l1-eth-rpc=$CERC_L1_RPC diff --git a/stack_orchestrator/data/container-build/cerc-optimism-contracts/Dockerfile b/stack_orchestrator/data/container-build/cerc-optimism-contracts/Dockerfile index ed9c4b22..2499df0a 100644 --- a/stack_orchestrator/data/container-build/cerc-optimism-contracts/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-optimism-contracts/Dockerfile @@ -17,6 +17,6 @@ WORKDIR /app COPY . . RUN echo "Building optimism" && \ - yarn && yarn build + pnpm install && pnpm build WORKDIR /app/packages/contracts-bedrock diff --git a/stack_orchestrator/data/container-build/cerc-optimism-op-batcher/Dockerfile b/stack_orchestrator/data/container-build/cerc-optimism-op-batcher/Dockerfile index 23d6b629..f52e75b9 100644 --- a/stack_orchestrator/data/container-build/cerc-optimism-op-batcher/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-optimism-op-batcher/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.19.0-alpine3.15 as builder +FROM golang:1.21.0-alpine3.18 as builder ARG VERSION=v0.0.0 @@ -9,7 +9,7 @@ COPY ./op-batcher /app/op-batcher COPY ./op-bindings /app/op-bindings COPY ./op-node /app/op-node COPY ./op-service /app/op-service -COPY ./op-signer /app/op-signer +#COPY ./op-signer /app/op-signer COPY ./go.mod /app/go.mod COPY ./go.sum /app/go.sum @@ -23,7 +23,7 @@ ARG TARGETOS TARGETARCH RUN make op-batcher VERSION="$VERSION" GOOS=$TARGETOS GOARCH=$TARGETARCH -FROM alpine:3.15 +FROM alpine:3.18 RUN apk add --no-cache jq bash diff --git a/stack_orchestrator/data/container-build/cerc-optimism-op-node/Dockerfile b/stack_orchestrator/data/container-build/cerc-optimism-op-node/Dockerfile index 17d273b6..ad63bb2c 100644 --- a/stack_orchestrator/data/container-build/cerc-optimism-op-node/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-optimism-op-node/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.19.0-alpine3.15 as builder +FROM golang:1.21.0-alpine3.18 as builder ARG VERSION=v0.0.0 @@ -21,7 +21,7 @@ ARG TARGETOS TARGETARCH RUN make op-node VERSION="$VERSION" GOOS=$TARGETOS GOARCH=$TARGETARCH -FROM alpine:3.15 +FROM alpine:3.18 RUN apk add --no-cache openssl jq diff --git a/stack_orchestrator/data/container-build/cerc-optimism-op-proposer/Dockerfile b/stack_orchestrator/data/container-build/cerc-optimism-op-proposer/Dockerfile index e91aa4bb..9032a7ff 100644 --- a/stack_orchestrator/data/container-build/cerc-optimism-op-proposer/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-optimism-op-proposer/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.19.0-alpine3.15 as builder +FROM golang:1.21.0-alpine3.18 as builder ARG VERSION=v0.0.0 @@ -9,7 +9,7 @@ COPY ./op-proposer /app/op-proposer COPY ./op-bindings /app/op-bindings COPY ./op-node /app/op-node COPY ./op-service /app/op-service -COPY ./op-signer /app/op-signer +#COPY ./op-signer /app/op-signer COPY ./go.mod /app/go.mod COPY ./go.sum /app/go.sum COPY ./.git /app/.git @@ -22,7 +22,7 @@ ARG TARGETOS TARGETARCH RUN make op-proposer VERSION="$VERSION" GOOS=$TARGETOS GOARCH=$TARGETARCH -FROM alpine:3.15 +FROM alpine:3.18 RUN apk add --no-cache jq bash diff --git a/stack_orchestrator/data/stacks/fixturenet-optimism/README.md b/stack_orchestrator/data/stacks/fixturenet-optimism/README.md index 4d933f83..dd681aa5 100644 --- a/stack_orchestrator/data/stacks/fixturenet-optimism/README.md +++ b/stack_orchestrator/data/stacks/fixturenet-optimism/README.md @@ -12,6 +12,7 @@ Clone required repositories: laconic-so --stack fixturenet-optimism setup-repositories # If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories mentioned below and re-run the command +# The repositories are located in $HOME/cerc by default ``` Build the container images: @@ -39,20 +40,53 @@ This should create the required docker images in the local image registry: * `cerc/optimism-op-batcher` * `cerc/optimism-op-proposer` -## Deploy -Deploy the stack: +## Create a deployment +First, create a spec file for the deployment, which will map the stack's ports and volumes to the host: ```bash -laconic-so --stack fixturenet-optimism deploy up +laconic-so --stack fixturenet-optimism deploy init --map-ports-to-host any-fixed-random --output fixturenet-optimism-spec.yml ``` -The `fixturenet-optimism-contracts` service takes a while to complete running as it: -1. waits for the 'Merge' to happen on L1 -2. waits for a finalized block to exist on L1 (so that it can be taken as a starting block for roll ups) -3. deploys the L1 contracts -It may restart a few times after running into errors. +### Ports +It is usually necessary to expose certain container ports on one or more the host's addresses to allow incoming connections. +Any ports defined in the Docker compose file are exposed by default with random port assignments, bound to "any" interface (IP address 0.0.0.0), but the port mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`. +In addition, a stack-wide port mapping "recipe" can be applied at the time the +`laconic-so deploy init` command is run, by supplying the desired recipe with the `--map-ports-to-host` option. The following recipes are supported: +| Recipe | Host Port Mapping | +|--------|-------------------| +| any-variable-random | Bind to 0.0.0.0 using a random port assigned at start time (default) | +| localhost-same | Bind to 127.0.0.1 using the same port number as exposed by the containers | +| any-same | Bind to 0.0.0.0 using the same port number as exposed by the containers | +| localhost-fixed-random | Bind to 127.0.0.1 using a random port number selected at the time the command is run (not checked for already in use)| +| any-fixed-random | Bind to 0.0.0.0 using a random port number selected at the time the command is run (not checked for already in use) | + +For example, you may wish to use `any-fixed-random` to generate the initial mappings and then edit the spec file to set the `fixturenet-eth-geth-1` RPC to port 8545 and the `op-geth` RPC to port 9545 on the host. + +Or, you may wish to use `any-same` for the initial mappings -- in which case you'll have to edit the spec to file to ensure the various geth instances aren't all trying to publish to host ports 8545/8546 at once. + +### Data volumes +Container data volumes are bind-mounted to specified paths in the host filesystem. +The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory. The default mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`. + +--- +Once you've made any needed changes to the spec file, create a deployment from it: +```bash +laconic-so --stack fixturenet-optimism deploy create --spec-file fixturenet-optimism-spec.yml --deployment-dir fixturenet-optimism-deployment +``` + +## Start the stack +Start the deployment: +```bash +laconic-so deployment --dir fixturenet-optimism-deployment start +``` +1. The `fixturenet-eth` L1 chain will start up first and begin producing blocks. +2. The `fixturenet-optimism-contracts` service will configure and deploy the Optimism contracts to L1, exiting when complete. This may take several minutes; you can follow the progress by following the container's logs (see below). +3. The `op-node` and `op-geth` services will initialize themselves (if not already initialized) and start +4. The remaining services, `op-batcher` and `op-proposer` will start + +### Logs To list and monitor the running containers: ```bash @@ -65,22 +99,69 @@ docker ps docker logs -f ``` -## Clean up +## Example: bridge some ETH from L1 to L2 -Stop all services running in the background: +Send some ETH from the desired account to the `L1StandardBridgeProxy` contract on L1 to test bridging to L2. +We can use the testing account `0xe6CE22afe802CAf5fF7d3845cec8c736ecc8d61F` which is pre-funded and unlocked, and the `cerc/foundry:local` container to make use of the `cast` cli. + +1. Note the docker network the stack is running on: ```bash -laconic-so --stack fixturenet-optimism deploy down 30 +docker network ls +# The network name will be something like laconic-[some_hash]_default +``` +2. Set some variables: +```bash +L1_RPC=http://fixturenet-eth-geth-1:8545 +L2_RPC=http://op-geth:8545 +NETWORK= +DEPLOYMENT_CONTEXT= +ACCOUNT=0xe6CE22afe802CAf5fF7d3845cec8c736ecc8d61F ``` -Clear volumes created by this stack: +If you need to check the L1 chain-id, you can use: +```bash +docker run --rm --network $NETWORK cerc/foundry:local "cast chain-id --rpc-url $L1_RPC" +``` + +3. Check the account starting balance on L2 (it should be 0): +```bash +docker run --rm --network $NETWORK cerc/foundry:local "cast balance $ACCOUNT --rpc-url $L2_RPC" +# 0 +``` + +4. Read the bridge contract address from the L1 deployment records in the `op-node` container: +```bash +# get the container id for op-node +NODE_CONTAINER=$(docker ps --filter "name=op-node" -q) +BRIDGE=$(docker exec $NODE_CONTAINER cat /l1-deployment/$DEPLOYMENT_CONTEXT/L1StandardBridgeProxy.json | jq -r .address) +``` + +5. Use cast to send some ETH to the bridge contract: +```bash +docker run --rm --network $NETWORK cerc/foundry:local "cast send --from $ACCOUNT --value 1ether $BRIDGE --rpc-url $L1_RPC" +``` + +6. Allow a couple minutes for the bridge to complete + +7. Check the L2 balance again (it should show the bridged funds): +```bash +docker run --rm --network $NETWORK cerc/foundry:local "cast balance $ACCOUNT --rpc-url $L2_RPC" +# 1000000000000000000 +``` + +## Clean up + +To stop all services running in the background, while preserving chain data: ```bash -# List all relevant volumes -docker volume ls -q --filter "name=.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data" +laconic-so deployment --dir fixturenet-optimism-deployment stop +``` -# Remove all the listed volumes -docker volume rm $(docker volume ls -q --filter "name=.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data") +To stop all services and also delete chain data: + +```bash +laconic-so deployment --dir fixturenet-optimism-deployment stop --delete-volumes ``` ## Troubleshooting diff --git a/stack_orchestrator/data/stacks/fixturenet-optimism/deploy/commands.py b/stack_orchestrator/data/stacks/fixturenet-optimism/deploy/commands.py new file mode 100644 index 00000000..43aa6ca7 --- /dev/null +++ b/stack_orchestrator/data/stacks/fixturenet-optimism/deploy/commands.py @@ -0,0 +1,37 @@ +# Copyright © 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from app.deploy.deploy_types import DeployCommandContext, LaconicStackSetupCommand, DeploymentContext +from ruamel.yaml import YAML + +def create(context: DeploymentContext, extra_args): + # Slightly modify the base fixturenet-eth compose file to replace the startup script for fixturenet-eth-geth-1 + # We need to start geth with the flag to allow non eip-155 compliant transactions in order to publish the + # deterministic-deployment-proxy contract, which itself is a prereq for Optimism contract deployment + fixturenet_eth_compose_file = context.deployment_dir.joinpath('compose', 'docker-compose-fixturenet-eth.yml') + + with open(fixturenet_eth_compose_file, 'r') as yaml_file: + yaml=YAML() + yaml_data = yaml.load(yaml_file) + + new_script = '../config/fixturenet-optimism/run-geth.sh:/opt/testnet/run.sh' + + if new_script not in yaml_data['services']['fixturenet-eth-geth-1']['volumes']: + yaml_data['services']['fixturenet-eth-geth-1']['volumes'].append(new_script) + + with open(fixturenet_eth_compose_file, 'w') as yaml_file: + yaml=YAML() + yaml.dump(yaml_data, yaml_file) + + return None diff --git a/stack_orchestrator/data/stacks/fixturenet-optimism/l2-only.md b/stack_orchestrator/data/stacks/fixturenet-optimism/l2-only.md index 4e9daf43..4299ca8d 100644 --- a/stack_orchestrator/data/stacks/fixturenet-optimism/l2-only.md +++ b/stack_orchestrator/data/stacks/fixturenet-optimism/l2-only.md @@ -1,4 +1,4 @@ -# fixturenet-optimism +# fixturenet-optimism (L2-only) Instructions to setup and deploy L2 fixturenet using [Optimism](https://stack.optimism.io) @@ -28,9 +28,43 @@ This should create the required docker images in the local image registry: * `cerc/optimism-op-batcher` * `cerc/optimism-op-proposer` -## Deploy +## Create a deployment -Create and update an env file to be used in the next step ([defaults](../../config/fixturenet-optimism/l1-params.env)): +First, create a spec file for the deployment, which will map the stack's ports and volumes to the host: +```bash +laconic-so --stack fixturenet-optimism deploy init --map-ports-to-host any-fixed-random --output fixturenet-optimism-spec.yml +``` +### Ports +It is usually necessary to expose certain container ports on one or more the host's addresses to allow incoming connections. +Any ports defined in the Docker compose file are exposed by default with random port assignments, bound to "any" interface (IP address 0.0.0.0), but the port mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`. + +In addition, a stack-wide port mapping "recipe" can be applied at the time the +`laconic-so deploy init` command is run, by supplying the desired recipe with the `--map-ports-to-host` option. The following recipes are supported: +| Recipe | Host Port Mapping | +|--------|-------------------| +| any-variable-random | Bind to 0.0.0.0 using a random port assigned at start time (default) | +| localhost-same | Bind to 127.0.0.1 using the same port number as exposed by the containers | +| any-same | Bind to 0.0.0.0 using the same port number as exposed by the containers | +| localhost-fixed-random | Bind to 127.0.0.1 using a random port number selected at the time the command is run (not checked for already in use)| +| any-fixed-random | Bind to 0.0.0.0 using a random port number selected at the time the command is run (not checked for already in use) | + +For example, you may wish to use `any-fixed-random` to generate the initial mappings and then edit the spec file to set the `op-geth` RPC to an easy to remember port like 8545 or 9545 on the host. + +### Data volumes +Container data volumes are bind-mounted to specified paths in the host filesystem. +The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory. The default mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`. + +--- +Once you've made any needed changes to the spec file, create a deployment from it: +```bash +laconic-so --stack fixturenet-optimism deploy create --spec-file fixturenet-optimism-spec.yml --deployment-dir fixturenet-optimism-deployment +``` + +Finally, open the `stack.yml` file inside your deployment directory and, under the `pods:` section, remove (or comment out) the entry for `fixturenet-eth`. This will prevent the deployment from trying to spin up a new L1 chain when starting the stack. + +## Set chain env variables + +Inside the deployment directory, open the file `config.env` and add the following variables to point the stack at your L1 rpc and provide account credentials ([defaults](../../config/fixturenet-optimism/l1-params.env)): ```bash # External L1 endpoint @@ -45,30 +79,29 @@ Create and update an env file to be used in the next step ([defaults](../../conf CERC_L1_ACCOUNTS_CSV_URL= # OR - # Specify the required account credentials + # Specify the required account credentials for the Admin account + # Other generated accounts will be funded from this account, so it should contain ~20 Eth CERC_L1_ADDRESS= CERC_L1_PRIV_KEY= - CERC_L1_ADDRESS_2= - CERC_L1_PRIV_KEY_2= ``` -* NOTE: If L1 is running on the host machine, use `host.docker.internal` as the hostname to access the host port - -Deploy the stack: +* NOTE: If L1 is running on the host machine, use `host.docker.internal` as the hostname to access the host port, or use the `ip a` command to find the IP address of the `docker0` interface (this will usually be something like `172.17.0.1` or `172.18.0.1`) +## Start the stack +Start the deployment: ```bash -laconic-so --stack fixturenet-optimism deploy --include fixturenet-optimism --env-file up +laconic-so deployment --dir fixturenet-optimism-deployment start ``` +1. The stack will check for a response from the L1 endpoint specified in your env file. +2. The `fixturenet-optimism-contracts` service will configure and deploy the Optimism contracts to L1, exiting when complete. This may take several minutes; you can follow the progress by following the container's logs (see below). +3. The `op-node` and `op-geth` services will initialize themselves (if not already initialized) and start +4. The remaining services, `op-batcher` and `op-proposer` will start -The `fixturenet-optimism-contracts` service may take a while (`~15 mins`) to complete running as it: -1. waits for the 'Merge' to happen on L1 -2. waits for a finalized block to exist on L1 (so that it can be taken as a starting block for roll ups) -3. deploys the L1 contracts - -To list down and monitor the running containers: +### Logs +To list and monitor the running containers: ```bash -laconic-so --stack fixturenet-optimism deploy --include fixturenet-optimism ps +laconic-so --stack fixturenet-optimism deploy ps # With status docker ps @@ -79,20 +112,16 @@ docker logs -f ## Clean up -Stop all services running in the background: +To stop all L2 services running in the background, while preserving chain data: ```bash -laconic-so --stack fixturenet-optimism deploy --include fixturenet-optimism down 30 +laconic-so deployment --dir fixturenet-optimism-deployment stop ``` -Clear volumes created by this stack: +To stop all L2 services and also delete chain data: ```bash -# List all relevant volumes -docker volume ls -q --filter "name=.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data" - -# Remove all the listed volumes -docker volume rm $(docker volume ls -q --filter "name=.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data") +laconic-so deployment --dir fixturenet-optimism-deployment stop --delete-volumes ``` ## Troubleshooting diff --git a/stack_orchestrator/data/stacks/fixturenet-optimism/stack.yml b/stack_orchestrator/data/stacks/fixturenet-optimism/stack.yml index 75c7620b..bca34b16 100644 --- a/stack_orchestrator/data/stacks/fixturenet-optimism/stack.yml +++ b/stack_orchestrator/data/stacks/fixturenet-optimism/stack.yml @@ -5,8 +5,8 @@ repos: - git.vdb.to/cerc-io/go-ethereum@v1.11.6-statediff-v5 - git.vdb.to/cerc-io/lighthouse - github.com/dboreham/foundry - - github.com/ethereum-optimism/optimism@v1.0.4 - - github.com/ethereum-optimism/op-geth@v1.101105.2 + - github.com/ethereum-optimism/optimism@op-node/v1.3.0 + - github.com/ethereum-optimism/op-geth@v1.101304.0 containers: - cerc/go-ethereum - cerc/lighthouse From 3db443b2bbcc26342e9b7fa63518efef8754b036 Mon Sep 17 00:00:00 2001 From: iskay Date: Fri, 10 Nov 2023 20:42:02 +0000 Subject: [PATCH 03/75] fix commands import --- .../data/stacks/fixturenet-optimism/deploy/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack_orchestrator/data/stacks/fixturenet-optimism/deploy/commands.py b/stack_orchestrator/data/stacks/fixturenet-optimism/deploy/commands.py index 43aa6ca7..76668135 100644 --- a/stack_orchestrator/data/stacks/fixturenet-optimism/deploy/commands.py +++ b/stack_orchestrator/data/stacks/fixturenet-optimism/deploy/commands.py @@ -12,7 +12,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from app.deploy.deploy_types import DeployCommandContext, LaconicStackSetupCommand, DeploymentContext +from stack_orchestrator.deploy.deployment_context import DeploymentContext from ruamel.yaml import YAML def create(context: DeploymentContext, extra_args): From 414b8870363f183cc719ac75726f845929137c13 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Fri, 10 Nov 2023 17:44:25 -0600 Subject: [PATCH 04/75] Allow setting build tool (npm/yarn) and next.js version. (#639) * Allow setting build tool (npm/yarn) and next.js version. --- .../cerc-nextjs-base/Dockerfile | 19 +++---------- .../cerc-nextjs-base/Dockerfile.webapp | 4 +++ .../scripts/apply-runtime-env.sh | 9 +++++++ .../cerc-nextjs-base/scripts/build-app.sh | 27 +++++++++++++++---- .../scripts/start-serving-app.sh | 11 +++++++- tests/webapp-test/run-webapp-test.sh | 8 +++--- 6 files changed, 52 insertions(+), 26 deletions(-) diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile index 435d8c73..69e38932 100644 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile @@ -28,28 +28,15 @@ RUN \ # [Optional] Uncomment this section to install additional OS packages. RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get -y install --no-install-recommends jq gettext-base - -# [Optional] Uncomment if you want to install an additional version of node using nvm -# ARG EXTRA_NODE_VERSION=10 -# RUN su node -c "source /usr/local/share/nvm/nvm.sh && nvm install ${EXTRA_NODE_VERSION}" - -# We do this to get a yq binary from the published container, for the correct architecture we're building here -# COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq - -COPY /scripts /scripts + && apt-get -y install --no-install-recommends jq gettext-base moreutils # [Optional] Uncomment if you want to install more global node modules # RUN su node -c "npm install -g " -# RUN mkdir -p /config -# COPY ./config.yml /config - -# Install simple web server for now (use nginx perhaps later) -# RUN yarn global add http-server - # Expose port for http EXPOSE 3000 +COPY /scripts /scripts + # Default command sleeps forever so docker doesn't kill it CMD ["/scripts/start-serving-app.sh"] diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile.webapp b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile.webapp index f4b5d4d8..23f42385 100644 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile.webapp +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile.webapp @@ -1,4 +1,8 @@ FROM cerc/nextjs-base:local + +ARG CERC_NEXT_VERSION=latest +ARG CERC_BUILD_TOOL + WORKDIR /app COPY . . RUN rm -rf node_modules build .next* diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh index 793333c3..1e30b634 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh @@ -8,6 +8,15 @@ WORK_DIR="${1:-./}" SRC_DIR="${2:-.next}" TRG_DIR="${3:-.next-r}" +CERC_BUILD_TOOL="${CERC_BUILD_TOOL}" +if [ -z "$CERC_BUILD_TOOL" ]; then + if [ -f "yarn.lock" ]; then + CERC_BUILD_TOOL=npm + else + CERC_BUILD_TOOL=yarn + fi +fi + cd "${WORK_DIR}" || exit 1 rm -rf "$TRG_DIR" diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh index c2115f6a..83977268 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh @@ -4,8 +4,17 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +CERC_NEXT_VERSION="${CERC_NEXT_VERSION:-^14.0.2}" +CERC_BUILD_TOOL="${CERC_BUILD_TOOL}" +if [ -z "$CERC_BUILD_TOOL" ]; then + if [ -f "yarn.lock" ] && [ ! -f "package-lock.json" ]; then + CERC_BUILD_TOOL=yarn + else + CERC_BUILD_TOOL=npm + fi +fi +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) WORK_DIR="${1:-/app}" cd "${WORK_DIR}" || exit 1 @@ -20,6 +29,7 @@ if [ $? -ne 0 ]; then fi js-beautify next.config.dist > next.config.js +echo "" >> next.config.js WEBPACK_REQ_LINE=$(grep -n "require([\'\"]webpack[\'\"])" next.config.js | cut -d':' -f1) if [ -z "$WEBPACK_REQ_LINE" ]; then @@ -58,7 +68,7 @@ if [ -n "$WEBPACK_CONF_LINE" ]; then cat > next.config.js.3 < next.config.js.2 cat > next.config.js.3 <&2 @@ -88,7 +98,14 @@ fi cat package.dist | jq '.scripts.cerc_compile = "next experimental-compile"' | jq '.scripts.cerc_generate = "next experimental-generate"' > package.json -npm install || exit 1 -npm run cerc_compile || exit 1 +CUR_NEXT_VERSION="`jq -r '.dependencies.next' package.json`" + +if [ "$CERC_NEXT_VERSION" != "keep" ] && [ "$CUR_NEXT_VERSION" != "$CERC_NEXT_VERSION" ]; then + echo "Changing 'next' version from $CUR_NEXT_VERSION to $CERC_NEXT_VERSION (set with --build-arg CERC_NEXT_VERSION)" + cat package.json | jq ".dependencies.next = \"$CERC_NEXT_VERSION\"" | sponge package.json +fi + +$CERC_BUILD_TOOL install || exit 1 +$CERC_BUILD_TOOL run cerc_compile || exit 1 exit 0 diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh index 61664c68..04ab0ad3 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh @@ -5,6 +5,15 @@ fi SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +CERC_BUILD_TOOL="${CERC_BUILD_TOOL}" +if [ -z "$CERC_BUILD_TOOL" ]; then + if [ -f "yarn.lock" ] && [ ! -f "package-lock.json" ]; then + CERC_BUILD_TOOL=yarn + else + CERC_BUILD_TOOL=npm + fi +fi + CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/app}" cd "$CERC_WEBAPP_FILES_DIR" @@ -31,4 +40,4 @@ if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then fi fi -npm start . -p ${CERC_LISTEN_PORT:-3000} +$CERC_BUILD_TOOL start . -p ${CERC_LISTEN_PORT:-3000} diff --git a/tests/webapp-test/run-webapp-test.sh b/tests/webapp-test/run-webapp-test.sh index 71b4da16..d32e0eba 100755 --- a/tests/webapp-test/run-webapp-test.sh +++ b/tests/webapp-test/run-webapp-test.sh @@ -24,7 +24,7 @@ git clone https://git.vdb.to/cerc-io/test-progressive-web-app.git $CERC_REPO_BAS # Test webapp command execution $TEST_TARGET_SO build-webapp --source-repo $CERC_REPO_BASE_DIR/test-progressive-web-app -UUID=`uuidgen` +CHECK="SPECIAL_01234567890_TEST_STRING" set +e @@ -34,7 +34,7 @@ wget -O test.before -m http://localhost:3000 docker remove -f $CONTAINER_ID -CONTAINER_ID=$(docker run -p 3000:3000 -e CERC_WEBAPP_DEBUG=$UUID -d cerc/test-progressive-web-app:local) +CONTAINER_ID=$(docker run -p 3000:3000 -e CERC_WEBAPP_DEBUG=$CHECK -d cerc/test-progressive-web-app:local) sleep 3 wget -O test.after -m http://localhost:3000 @@ -43,7 +43,7 @@ docker remove -f $CONTAINER_ID echo "###########################################################################" echo "" -grep "$UUID" test.before > /dev/null +grep "$CHECK" test.before > /dev/null if [ $? -ne 1 ]; then echo "BEFORE: FAILED" exit 1 @@ -51,7 +51,7 @@ else echo "BEFORE: PASSED" fi -grep "$UUID" test.after > /dev/null +grep "$CHECK" test.after > /dev/null if [ $? -ne 0 ]; then echo "AFTER: FAILED" exit 1 From 95e881ba193194ab3abdcf57b410e3a4975574f2 Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Mon, 13 Nov 2023 10:58:55 +0530 Subject: [PATCH 05/75] Add a sushiswap-v3 watcher stack (#638) * Add a sushiswap-v3 watcher stack * Add services for watcher db and server * Add service for watcher job-runner * Use 0.0.0.0 for watcher server config --------- Co-authored-by: Nabarun --- .../docker-compose-watcher-sushiswap-v3.yml | 77 +++++++++++++++ .../watcher-sushiswap-v3/start-job-runner.sh | 20 ++++ .../watcher-sushiswap-v3/start-server.sh | 23 +++++ .../watcher-config-template.toml | 98 +++++++++++++++++++ .../cerc-watcher-sushiswap-v3/Dockerfile | 10 ++ .../cerc-watcher-sushiswap-v3/build.sh | 9 ++ .../data/stacks/sushiswap-v3/README.md | 62 ++++++++++++ .../data/stacks/sushiswap-v3/stack.yml | 9 ++ 8 files changed, 308 insertions(+) create mode 100644 stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml create mode 100755 stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh create mode 100755 stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh create mode 100644 stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml create mode 100644 stack_orchestrator/data/container-build/cerc-watcher-sushiswap-v3/Dockerfile create mode 100755 stack_orchestrator/data/container-build/cerc-watcher-sushiswap-v3/build.sh create mode 100644 stack_orchestrator/data/stacks/sushiswap-v3/README.md create mode 100644 stack_orchestrator/data/stacks/sushiswap-v3/stack.yml diff --git a/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml b/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml new file mode 100644 index 00000000..db367420 --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml @@ -0,0 +1,77 @@ +version: '3.2' + +services: + sushiswap-v3-watcher-db: + restart: unless-stopped + image: postgres:14-alpine + environment: + - POSTGRES_USER=vdbm + - POSTGRES_MULTIPLE_DATABASES=sushiswap-v3-watcher,sushiswap-v3-watcher-job-queue + - POSTGRES_EXTENSION=sushiswap-v3-watcher-job-queue:pgcrypto + - POSTGRES_PASSWORD=password + volumes: + - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh + - sushiswap_v3_watcher_db_data:/var/lib/postgresql/data + ports: + - "127.0.0.1:15432:5432" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "5432"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 10s + + sushiswap-v3-watcher-job-runner: + restart: unless-stopped + depends_on: + sushiswap-v3-watcher-db: + condition: service_healthy + image: cerc/watcher-sushiswap-v3:local + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT} + command: ["bash", "./start-job-runner.sh"] + volumes: + - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml + - ../config/watcher-sushiswap-v3/start-job-runner.sh:/app/start-job-runner.sh + ports: + - "127.0.0.1:9000:9000" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "9000"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + sushiswap-v3-watcher-server: + restart: unless-stopped + depends_on: + sushiswap-v3-watcher-db: + condition: service_healthy + sushiswap-v3-watcher-job-runner: + condition: service_healthy + image: cerc/watcher-sushiswap-v3:local + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT} + SUSHISWAP_START_BLOCK: ${SUSHISWAP_START_BLOCK:- 2867560} + command: ["bash", "./start-server.sh"] + volumes: + - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml + - ../config/watcher-sushiswap-v3/start-server.sh:/app/start-server.sh + ports: + - "127.0.0.1:3008:3008" + - "127.0.0.1:9001:9001" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "3008"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + +volumes: + sushiswap_v3_watcher_db_data: diff --git a/stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh b/stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh new file mode 100755 index 00000000..819b1096 --- /dev/null +++ b/stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi +set -u + +echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}" + +# Read in the config template TOML file and modify it +WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) +WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ + sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ") + +# Write the modified content to a new file +echo "$WATCHER_CONFIG" > environments/local.toml + +echo "Running job-runner..." +DEBUG=vulcanize:* exec node --enable-source-maps dist/job-runner.js diff --git a/stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh b/stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh new file mode 100755 index 00000000..1b14f2e3 --- /dev/null +++ b/stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi +set -u + +echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}" + +# Read in the config template TOML file and modify it +WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) +WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ + sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ") + +# Write the modified content to a new file +echo "$WATCHER_CONFIG" > environments/local.toml + +echo "Initializing watcher..." +yarn fill --start-block $SUSHISWAP_START_BLOCK --end-block $((SUSHISWAP_START_BLOCK + 1)) + +echo "Running server..." +DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js diff --git a/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml new file mode 100644 index 00000000..7c582b80 --- /dev/null +++ b/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml @@ -0,0 +1,98 @@ +[server] + host = "0.0.0.0" + port = 3008 + kind = "active" + + # Checkpointing state. + checkpointing = true + + # Checkpoint interval in number of blocks. + checkpointInterval = 2000 + + # Enable state creation + # CAUTION: Disable only if state creation is not desired or can be filled subsequently + enableState = false + + subgraphPath = "./subgraph-build" + + # Interval to restart wasm instance periodically + wasmRestartBlocksInterval = 20 + + # Interval in number of blocks at which to clear entities cache. + clearEntitiesCacheInterval = 1000 + + # Max block range for which to return events in eventsInRange GQL query. + # Use -1 for skipping check on block range. + maxEventsBlockRange = 1000 + + # Flag to specify whether RPC endpoint supports block hash as block tag parameter + rpcSupportsBlockHashParam = false + + # GQL cache settings + [server.gqlCache] + enabled = true + + # Max in-memory cache size (in bytes) (default 8 MB) + # maxCacheSize + + # GQL cache-control max-age settings (in seconds) + maxAge = 15 + timeTravelMaxAge = 86400 # 1 day + +[metrics] + host = "127.0.0.1" + port = 9000 + [metrics.gql] + port = 9001 + +[database] + type = "postgres" + host = "sushiswap-v3-watcher-db" + port = 5432 + database = "sushiswap-v3-watcher" + username = "vdbm" + password = "password" + synchronize = true + logging = false + +[upstream] + [upstream.ethServer] + rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT" + + # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client) + rpcClient = true + + # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint + isFEVM = true + + # Boolean flag to filter event logs by contracts + filterLogsByAddresses = true + # Boolean flag to filter event logs by topics + filterLogsByTopics = true + + [upstream.cache] + name = "requests" + enabled = false + deleteOnStart = false + +[jobQueue] + dbConnectionString = "postgres://vdbm:password@sushiswap-v3-watcher-db/sushiswap-v3-watcher-job-queue" + maxCompletionLagInSecs = 300 + jobDelayInMilliSecs = 100 + eventsInBatch = 50 + subgraphEventsOrder = true + blockDelayInMilliSecs = 2000 + prefetchBlocksInMem = false + prefetchBlockCount = 10 + + # Boolean to switch between modes of processing events when starting the server. + # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them. + # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head). + useBlockRanges = true + + # Block range in which logs are fetched during historical blocks processing + historicalLogsBlockRange = 2000 + + # Max block range of historical processing after which it waits for completion of events processing + # If set to -1 historical processing does not wait for events processing and completes till latest canonical block + historicalMaxFetchAhead = 10000 diff --git a/stack_orchestrator/data/container-build/cerc-watcher-sushiswap-v3/Dockerfile b/stack_orchestrator/data/container-build/cerc-watcher-sushiswap-v3/Dockerfile new file mode 100644 index 00000000..ac6241c4 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-watcher-sushiswap-v3/Dockerfile @@ -0,0 +1,10 @@ +FROM node:18.17.1-alpine3.18 + +RUN apk --update --no-cache add git python3 alpine-sdk bash curl jq + +WORKDIR /app + +COPY . . + +RUN echo "Installing dependencies and building sushiswap-v3-watcher-ts" && \ + yarn && yarn build diff --git a/stack_orchestrator/data/container-build/cerc-watcher-sushiswap-v3/build.sh b/stack_orchestrator/data/container-build/cerc-watcher-sushiswap-v3/build.sh new file mode 100755 index 00000000..4eb79eee --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-watcher-sushiswap-v3/build.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Build cerc/watcher-sushiswap-v3 + +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh + +# See: https://stackoverflow.com/a/246128/1701505 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/watcher-sushiswap-v3:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/sushiswap-v3-watcher-ts diff --git a/stack_orchestrator/data/stacks/sushiswap-v3/README.md b/stack_orchestrator/data/stacks/sushiswap-v3/README.md new file mode 100644 index 00000000..7116a6d9 --- /dev/null +++ b/stack_orchestrator/data/stacks/sushiswap-v3/README.md @@ -0,0 +1,62 @@ +# SushiSwap v3 Watcher + +## Setup + +Clone required repositories: + +```bash +laconic-so --stack sushiswap-v3 setup-repositories --git-ssh --pull +``` + +Build the container images: + +```bash +laconic-so --stack sushiswap-v3 build-containers +``` + +## Deploy + +### Configuration + +Create and update an env file to be used in the next step: + + ```bash + # External Filecoin (ETH RPC) endpoint to point the watcher + CERC_ETH_RPC_ENDPOINT= + ``` + +### Deploy the stack + +```bash +laconic-so --stack sushiswap-v3 deploy --cluster sushiswap_v3 --env-file up +``` + +* To list down and monitor the running containers: + + ```bash + laconic-so --stack sushiswap-v3 deploy --cluster sushiswap_v3 ps + + # With status + docker ps -a + + # Check logs for a container + docker logs -f + ``` + +## Clean up + +Stop all the services running in background: + +```bash +laconic-so --stack sushiswap-v3 deploy --cluster sushiswap_v3 down +``` + +Clear volumes created by this stack: + +```bash +# List all relevant volumes +docker volume ls -q --filter "name=sushiswap_v3" + +# Remove all the listed volumes +docker volume rm $(docker volume ls -q --filter "name=sushiswap_v3") +``` diff --git a/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml b/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml new file mode 100644 index 00000000..a851ce51 --- /dev/null +++ b/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml @@ -0,0 +1,9 @@ +version: "1.0" +name: sushiswap-v3 +description: "SushiSwap v3 watcher stack" +repos: + - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.0 +containers: + - cerc/watcher-sushiswap-v3 +pods: + - watcher-sushiswap-v3 From a04730e7acc6d2055887285bef16853f7b449aae Mon Sep 17 00:00:00 2001 From: Nabarun Gogoi Date: Mon, 13 Nov 2023 11:13:55 +0530 Subject: [PATCH 06/75] Add a merkl-sushiswap-v3 watcher stack (#641) * Add a merkl-sushiswap-v3 watcher stack * Remove unrequired image from list --- ...ker-compose-watcher-merkl-sushiswap-v3.yml | 77 +++++++++++++++ .../start-job-runner.sh | 20 ++++ .../start-server.sh | 23 +++++ .../watcher-config-template.toml | 98 +++++++++++++++++++ .../Dockerfile | 10 ++ .../cerc-watcher-merkl-sushiswap-v3/build.sh | 11 +++ .../data/container-image-list.txt | 1 + stack_orchestrator/data/pod-list.txt | 1 + stack_orchestrator/data/repository-list.txt | 1 + .../data/stacks/graph-node/README.md | 2 +- .../data/stacks/merkl-sushiswap-v3/README.md | 81 +++++++++++++++ .../data/stacks/merkl-sushiswap-v3/stack.yml | 9 ++ 12 files changed, 333 insertions(+), 1 deletion(-) create mode 100644 stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml create mode 100755 stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-job-runner.sh create mode 100755 stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-server.sh create mode 100644 stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml create mode 100644 stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/Dockerfile create mode 100755 stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/build.sh create mode 100644 stack_orchestrator/data/stacks/merkl-sushiswap-v3/README.md create mode 100644 stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml diff --git a/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml b/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml new file mode 100644 index 00000000..62e848e8 --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml @@ -0,0 +1,77 @@ +version: '3.2' + +services: + merkl-sushiswap-v3-watcher-db: + restart: unless-stopped + image: postgres:14-alpine + environment: + - POSTGRES_USER=vdbm + - POSTGRES_MULTIPLE_DATABASES=merkl-sushiswap-v3-watcher,merkl-sushiswap-v3-watcher-job-queue + - POSTGRES_EXTENSION=merkl-sushiswap-v3-watcher-job-queue:pgcrypto + - POSTGRES_PASSWORD=password + volumes: + - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh + - merkl_sushiswap_v3_watcher_db_data:/var/lib/postgresql/data + ports: + - "127.0.0.1:15432:5432" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "5432"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 10s + + merkl-sushiswap-v3-watcher-job-runner: + restart: unless-stopped + depends_on: + merkl-sushiswap-v3-watcher-db: + condition: service_healthy + image: cerc/watcher-merkl-sushiswap-v3:local + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT} + command: ["bash", "./start-job-runner.sh"] + volumes: + - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml + - ../config/watcher-merkl-sushiswap-v3/start-job-runner.sh:/app/start-job-runner.sh + ports: + - "127.0.0.1:9000:9000" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "9000"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + merkl-sushiswap-v3-watcher-server: + restart: unless-stopped + depends_on: + merkl-sushiswap-v3-watcher-db: + condition: service_healthy + merkl-sushiswap-v3-watcher-job-runner: + condition: service_healthy + image: cerc/watcher-merkl-sushiswap-v3:local + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT} + SUSHISWAP_START_BLOCK: ${SUSHISWAP_START_BLOCK:- 2867560} + command: ["bash", "./start-server.sh"] + volumes: + - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml + - ../config/watcher-merkl-sushiswap-v3/start-server.sh:/app/start-server.sh + ports: + - "127.0.0.1:3007:3008" + - "127.0.0.1:9001:9001" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "3008"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + +volumes: + merkl_sushiswap_v3_watcher_db_data: diff --git a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-job-runner.sh b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-job-runner.sh new file mode 100755 index 00000000..819b1096 --- /dev/null +++ b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-job-runner.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi +set -u + +echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}" + +# Read in the config template TOML file and modify it +WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) +WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ + sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ") + +# Write the modified content to a new file +echo "$WATCHER_CONFIG" > environments/local.toml + +echo "Running job-runner..." +DEBUG=vulcanize:* exec node --enable-source-maps dist/job-runner.js diff --git a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-server.sh b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-server.sh new file mode 100755 index 00000000..1b14f2e3 --- /dev/null +++ b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-server.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi +set -u + +echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}" + +# Read in the config template TOML file and modify it +WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) +WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ + sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ") + +# Write the modified content to a new file +echo "$WATCHER_CONFIG" > environments/local.toml + +echo "Initializing watcher..." +yarn fill --start-block $SUSHISWAP_START_BLOCK --end-block $((SUSHISWAP_START_BLOCK + 1)) + +echo "Running server..." +DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js diff --git a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml new file mode 100644 index 00000000..053e5544 --- /dev/null +++ b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml @@ -0,0 +1,98 @@ +[server] + host = "0.0.0.0" + port = 3008 + kind = "active" + + # Checkpointing state. + checkpointing = true + + # Checkpoint interval in number of blocks. + checkpointInterval = 2000 + + # Enable state creation + # CAUTION: Disable only if state creation is not desired or can be filled subsequently + enableState = false + + subgraphPath = "./subgraph-build" + + # Interval to restart wasm instance periodically + wasmRestartBlocksInterval = 20 + + # Interval in number of blocks at which to clear entities cache. + clearEntitiesCacheInterval = 1000 + + # Max block range for which to return events in eventsInRange GQL query. + # Use -1 for skipping check on block range. + maxEventsBlockRange = 1000 + + # Flag to specify whether RPC endpoint supports block hash as block tag parameter + rpcSupportsBlockHashParam = false + + # GQL cache settings + [server.gqlCache] + enabled = true + + # Max in-memory cache size (in bytes) (default 8 MB) + # maxCacheSize + + # GQL cache-control max-age settings (in seconds) + maxAge = 15 + timeTravelMaxAge = 86400 # 1 day + +[metrics] + host = "127.0.0.1" + port = 9000 + [metrics.gql] + port = 9001 + +[database] + type = "postgres" + host = "merkl-sushiswap-v3-watcher-db" + port = 5432 + database = "merkl-sushiswap-v3-watcher" + username = "vdbm" + password = "password" + synchronize = true + logging = false + +[upstream] + [upstream.ethServer] + rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT" + + # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client) + rpcClient = true + + # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint + isFEVM = true + + # Boolean flag to filter event logs by contracts + filterLogsByAddresses = true + # Boolean flag to filter event logs by topics + filterLogsByTopics = false + + [upstream.cache] + name = "requests" + enabled = false + deleteOnStart = false + +[jobQueue] + dbConnectionString = "postgres://vdbm:password@merkl-sushiswap-v3-watcher-db/merkl-sushiswap-v3-watcher-job-queue" + maxCompletionLagInSecs = 300 + jobDelayInMilliSecs = 100 + eventsInBatch = 50 + subgraphEventsOrder = true + blockDelayInMilliSecs = 2000 + prefetchBlocksInMem = false + prefetchBlockCount = 10 + + # Boolean to switch between modes of processing events when starting the server. + # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them. + # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head). + useBlockRanges = true + + # Block range in which logs are fetched during historical blocks processing + historicalLogsBlockRange = 2000 + + # Max block range of historical processing after which it waits for completion of events processing + # If set to -1 historical processing does not wait for events processing and completes till latest canonical block + historicalMaxFetchAhead = 10000 diff --git a/stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/Dockerfile b/stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/Dockerfile new file mode 100644 index 00000000..e09738ac --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/Dockerfile @@ -0,0 +1,10 @@ +FROM node:18.17.1-alpine3.18 + +RUN apk --update --no-cache add git python3 alpine-sdk bash curl jq + +WORKDIR /app + +COPY . . + +RUN echo "Installing dependencies and building merkl-sushiswap-v3-watcher-ts" && \ + yarn && yarn build diff --git a/stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/build.sh b/stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/build.sh new file mode 100755 index 00000000..b53ee621 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/build.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Build cerc/watcher-merkl-sushiswap-v3 + +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh + +# See: https://stackoverflow.com/a/246128/1701505 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/my-new-stack:local -f ${CERC_REPO_BASE_DIR}/my-new-stack/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/my-new-stack + +docker build -t cerc/watcher-merkl-sushiswap-v3:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/merkl-sushiswap-v3-watcher-ts diff --git a/stack_orchestrator/data/container-image-list.txt b/stack_orchestrator/data/container-image-list.txt index 256f0a6f..36df0201 100644 --- a/stack_orchestrator/data/container-image-list.txt +++ b/stack_orchestrator/data/container-image-list.txt @@ -57,3 +57,4 @@ cerc/nitro-contracts cerc/mobymask-snap cerc/ponder cerc/nitro-rpc-client +cerc/watcher-merkl-sushiswap-v3 \ No newline at end of file diff --git a/stack_orchestrator/data/pod-list.txt b/stack_orchestrator/data/pod-list.txt index 4ba1bac0..7b91600d 100644 --- a/stack_orchestrator/data/pod-list.txt +++ b/stack_orchestrator/data/pod-list.txt @@ -43,3 +43,4 @@ nitro-contracts mobymask-snap ponder ipld-eth-server-payments +merkl-sushiswap-v3 diff --git a/stack_orchestrator/data/repository-list.txt b/stack_orchestrator/data/repository-list.txt index ceaa910c..f6696464 100644 --- a/stack_orchestrator/data/repository-list.txt +++ b/stack_orchestrator/data/repository-list.txt @@ -47,3 +47,4 @@ github.com/cerc-io/go-nitro github.com/cerc-io/ts-nitro github.com/cerc-io/mobymask-snap github.com/cerc-io/ponder +github.com/cerc-io/merkl-sushiswap-v3-watcher-ts diff --git a/stack_orchestrator/data/stacks/graph-node/README.md b/stack_orchestrator/data/stacks/graph-node/README.md index 0527efc0..df3ae1eb 100644 --- a/stack_orchestrator/data/stacks/graph-node/README.md +++ b/stack_orchestrator/data/stacks/graph-node/README.md @@ -59,7 +59,7 @@ ports: Create deployment: ```bash -laconic-so deploy create --spec-file graph-node-spec.yml --deployment-dir graph-node-deployment +laconic-so --stack graph-node deploy create --spec-file graph-node-spec.yml --deployment-dir graph-node-deployment ``` ## Start the stack diff --git a/stack_orchestrator/data/stacks/merkl-sushiswap-v3/README.md b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/README.md new file mode 100644 index 00000000..4284c2ad --- /dev/null +++ b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/README.md @@ -0,0 +1,81 @@ +# Merkl SushiSwap v3 Watcher + +## Setup + +Clone required repositories: + +```bash +laconic-so --stack merkl-sushiswap-v3 setup-repositories --git-ssh --pull +``` + +Build the container images: + +```bash +laconic-so --stack merkl-sushiswap-v3 build-containers +``` + +## Deploy + +### Configuration + +Create and update an env file to be used in the next step: + + ```bash + # External Filecoin (ETH RPC) endpoint to point the watcher + CERC_ETH_RPC_ENDPOINT= + ``` + +### Deploy the stack + +```bash +laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 --env-file up +``` + +* To list down and monitor the running containers: + + ```bash + laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 ps + + # With status + docker ps -a + + # Check logs for a container + docker logs -f + ``` + +* Open the GQL playground at http://localhost:3007/graphql + + ```graphql + { + _meta { + block { + number + timestamp + } + hasIndexingErrors + } + + factories { + id + poolCount + } + } + ``` + +## Clean up + +Stop all the services running in background: + +```bash +laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 down +``` + +Clear volumes created by this stack: + +```bash +# List all relevant volumes +docker volume ls -q --filter "name=merkl_sushiswap_v3" + +# Remove all the listed volumes +docker volume rm $(docker volume ls -q --filter "name=merkl_sushiswap_v3") +``` diff --git a/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml new file mode 100644 index 00000000..e3b5abf1 --- /dev/null +++ b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml @@ -0,0 +1,9 @@ +version: "1.0" +name: merkl-sushiswap-v3 +description: "SushiSwap v3 watcher stack" +repos: + - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.0 +containers: + - cerc/watcher-merkl-sushiswap-v3 +pods: + - watcher-merkl-sushiswap-v3 From 0aca087558d1da8a9a14ccd6a49f407956119b2e Mon Sep 17 00:00:00 2001 From: Nabarun Gogoi Date: Mon, 13 Nov 2023 17:36:37 +0530 Subject: [PATCH 07/75] Upgrade release versions for merkl and sushiswap watchers (#642) * Upgrade merkl-sushiswap-v3-watcher-ts release * Increase blockDelayInMilliSecs for merkl-sushiswap-v3 watcher * Upgrade sushiswap-v3-watcher-ts release * Add sushiswap-v3 watcher to stack list * Avoid mapping ports that are not required to be exposed --- .../compose/docker-compose-watcher-merkl-sushiswap-v3.yml | 6 +++--- .../data/compose/docker-compose-watcher-sushiswap-v3.yml | 6 +++--- .../watcher-merkl-sushiswap-v3/watcher-config-template.toml | 3 ++- .../watcher-sushiswap-v3/watcher-config-template.toml | 3 ++- stack_orchestrator/data/container-image-list.txt | 3 ++- stack_orchestrator/data/pod-list.txt | 1 + stack_orchestrator/data/repository-list.txt | 1 + stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml | 2 +- stack_orchestrator/data/stacks/sushiswap-v3/stack.yml | 2 +- 9 files changed, 16 insertions(+), 11 deletions(-) diff --git a/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml b/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml index 62e848e8..d08c6214 100644 --- a/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml +++ b/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml @@ -13,7 +13,7 @@ services: - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh - merkl_sushiswap_v3_watcher_db_data:/var/lib/postgresql/data ports: - - "127.0.0.1:15432:5432" + - "5432" healthcheck: test: ["CMD", "nc", "-v", "localhost", "5432"] interval: 20s @@ -35,7 +35,7 @@ services: - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml - ../config/watcher-merkl-sushiswap-v3/start-job-runner.sh:/app/start-job-runner.sh ports: - - "127.0.0.1:9000:9000" + - "9000" healthcheck: test: ["CMD", "nc", "-v", "localhost", "9000"] interval: 20s @@ -63,7 +63,7 @@ services: - ../config/watcher-merkl-sushiswap-v3/start-server.sh:/app/start-server.sh ports: - "127.0.0.1:3007:3008" - - "127.0.0.1:9001:9001" + - "9001" healthcheck: test: ["CMD", "nc", "-v", "localhost", "3008"] interval: 20s diff --git a/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml b/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml index db367420..219688db 100644 --- a/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml +++ b/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml @@ -13,7 +13,7 @@ services: - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh - sushiswap_v3_watcher_db_data:/var/lib/postgresql/data ports: - - "127.0.0.1:15432:5432" + - "5432" healthcheck: test: ["CMD", "nc", "-v", "localhost", "5432"] interval: 20s @@ -35,7 +35,7 @@ services: - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml - ../config/watcher-sushiswap-v3/start-job-runner.sh:/app/start-job-runner.sh ports: - - "127.0.0.1:9000:9000" + - "9000" healthcheck: test: ["CMD", "nc", "-v", "localhost", "9000"] interval: 20s @@ -63,7 +63,7 @@ services: - ../config/watcher-sushiswap-v3/start-server.sh:/app/start-server.sh ports: - "127.0.0.1:3008:3008" - - "127.0.0.1:9001:9001" + - "9001" healthcheck: test: ["CMD", "nc", "-v", "localhost", "3008"] interval: 20s diff --git a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml index 053e5544..894a4660 100644 --- a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml +++ b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml @@ -81,7 +81,8 @@ jobDelayInMilliSecs = 100 eventsInBatch = 50 subgraphEventsOrder = true - blockDelayInMilliSecs = 2000 + # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime + blockDelayInMilliSecs = 30000 prefetchBlocksInMem = false prefetchBlockCount = 10 diff --git a/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml index 7c582b80..07880a8d 100644 --- a/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml +++ b/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml @@ -81,7 +81,8 @@ jobDelayInMilliSecs = 100 eventsInBatch = 50 subgraphEventsOrder = true - blockDelayInMilliSecs = 2000 + # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime + blockDelayInMilliSecs = 30000 prefetchBlocksInMem = false prefetchBlockCount = 10 diff --git a/stack_orchestrator/data/container-image-list.txt b/stack_orchestrator/data/container-image-list.txt index 36df0201..41dd8b21 100644 --- a/stack_orchestrator/data/container-image-list.txt +++ b/stack_orchestrator/data/container-image-list.txt @@ -57,4 +57,5 @@ cerc/nitro-contracts cerc/mobymask-snap cerc/ponder cerc/nitro-rpc-client -cerc/watcher-merkl-sushiswap-v3 \ No newline at end of file +cerc/watcher-merkl-sushiswap-v3 +cerc/watcher-sushiswap-v3 diff --git a/stack_orchestrator/data/pod-list.txt b/stack_orchestrator/data/pod-list.txt index 7b91600d..9ad000c7 100644 --- a/stack_orchestrator/data/pod-list.txt +++ b/stack_orchestrator/data/pod-list.txt @@ -44,3 +44,4 @@ mobymask-snap ponder ipld-eth-server-payments merkl-sushiswap-v3 +sushiswap-v3 diff --git a/stack_orchestrator/data/repository-list.txt b/stack_orchestrator/data/repository-list.txt index f6696464..192a831e 100644 --- a/stack_orchestrator/data/repository-list.txt +++ b/stack_orchestrator/data/repository-list.txt @@ -48,3 +48,4 @@ github.com/cerc-io/ts-nitro github.com/cerc-io/mobymask-snap github.com/cerc-io/ponder github.com/cerc-io/merkl-sushiswap-v3-watcher-ts +github.com/cerc-io/sushiswap-v3-watcher-ts diff --git a/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml index e3b5abf1..8b010a24 100644 --- a/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml +++ b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml @@ -2,7 +2,7 @@ version: "1.0" name: merkl-sushiswap-v3 description: "SushiSwap v3 watcher stack" repos: - - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.0 + - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.1 containers: - cerc/watcher-merkl-sushiswap-v3 pods: diff --git a/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml b/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml index a851ce51..66065b5d 100644 --- a/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml +++ b/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml @@ -2,7 +2,7 @@ version: "1.0" name: sushiswap-v3 description: "SushiSwap v3 watcher stack" repos: - - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.0 + - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.1 containers: - cerc/watcher-sushiswap-v3 pods: From f1f618c57a5658d93935602b6a9ef7f9bb1b48f8 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Mon, 13 Nov 2023 11:56:04 -0600 Subject: [PATCH 08/75] Don't change the next.js version by default. (#640) --- .../data/container-build/cerc-nextjs-base/Dockerfile.webapp | 2 +- .../container-build/cerc-nextjs-base/scripts/build-app.sh | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile.webapp b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile.webapp index 23f42385..51664deb 100644 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile.webapp +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile.webapp @@ -1,6 +1,6 @@ FROM cerc/nextjs-base:local -ARG CERC_NEXT_VERSION=latest +ARG CERC_NEXT_VERSION=keep ARG CERC_BUILD_TOOL WORKDIR /app diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh index 83977268..e62bc0d0 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh @@ -4,7 +4,7 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi -CERC_NEXT_VERSION="${CERC_NEXT_VERSION:-^14.0.2}" +CERC_NEXT_VERSION="${CERC_NEXT_VERSION:-keep}" CERC_BUILD_TOOL="${CERC_BUILD_TOOL}" if [ -z "$CERC_BUILD_TOOL" ]; then if [ -f "yarn.lock" ] && [ ! -f "package-lock.json" ]; then @@ -101,8 +101,10 @@ cat package.dist | jq '.scripts.cerc_compile = "next experimental-compile"' | jq CUR_NEXT_VERSION="`jq -r '.dependencies.next' package.json`" if [ "$CERC_NEXT_VERSION" != "keep" ] && [ "$CUR_NEXT_VERSION" != "$CERC_NEXT_VERSION" ]; then - echo "Changing 'next' version from $CUR_NEXT_VERSION to $CERC_NEXT_VERSION (set with --build-arg CERC_NEXT_VERSION)" + echo "Changing 'next' version specifier from '$CUR_NEXT_VERSION' to '$CERC_NEXT_VERSION' (set with --build-arg CERC_NEXT_VERSION)" cat package.json | jq ".dependencies.next = \"$CERC_NEXT_VERSION\"" | sponge package.json +else + echo "'next' version specifier '$CUR_NEXT_VERSION' (override with --build-arg CERC_NEXT_VERSION)" fi $CERC_BUILD_TOOL install || exit 1 From f088cbb3b0c64f864303b9f390007e976c0880cf Mon Sep 17 00:00:00 2001 From: iskay Date: Tue, 14 Nov 2023 14:38:49 +0000 Subject: [PATCH 09/75] fix linter errors --- .../stacks/fixturenet-optimism/deploy/commands.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/stack_orchestrator/data/stacks/fixturenet-optimism/deploy/commands.py b/stack_orchestrator/data/stacks/fixturenet-optimism/deploy/commands.py index 76668135..fa757cf5 100644 --- a/stack_orchestrator/data/stacks/fixturenet-optimism/deploy/commands.py +++ b/stack_orchestrator/data/stacks/fixturenet-optimism/deploy/commands.py @@ -12,26 +12,28 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . + from stack_orchestrator.deploy.deployment_context import DeploymentContext from ruamel.yaml import YAML + def create(context: DeploymentContext, extra_args): # Slightly modify the base fixturenet-eth compose file to replace the startup script for fixturenet-eth-geth-1 # We need to start geth with the flag to allow non eip-155 compliant transactions in order to publish the # deterministic-deployment-proxy contract, which itself is a prereq for Optimism contract deployment fixturenet_eth_compose_file = context.deployment_dir.joinpath('compose', 'docker-compose-fixturenet-eth.yml') - + with open(fixturenet_eth_compose_file, 'r') as yaml_file: - yaml=YAML() + yaml = YAML() yaml_data = yaml.load(yaml_file) - + new_script = '../config/fixturenet-optimism/run-geth.sh:/opt/testnet/run.sh' if new_script not in yaml_data['services']['fixturenet-eth-geth-1']['volumes']: yaml_data['services']['fixturenet-eth-geth-1']['volumes'].append(new_script) - + with open(fixturenet_eth_compose_file, 'w') as yaml_file: - yaml=YAML() + yaml = YAML() yaml.dump(yaml_data, yaml_file) return None From 9687d8446823cd63acff3b11bb72fd3b07ed2556 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 14 Nov 2023 16:07:26 -0600 Subject: [PATCH 10/75] 646: Add error message for webapp startup hang (#647) This fixes three issues: 1. #644 (build output) 2. #646 (error on startup) 3. automatic env quote handling (related to 2) For the build output we now have: ``` ################################################################# Built host container for /home/telackey/tmp/iglootools-home with tag: cerc/iglootools-home:local To test locally run: docker run -p 3000:3000 cerc/iglootools-home:local ``` For the startup error, it was hung waiting for the "success" message from the next generate output (itself a workaround for a nextjs bug fixed by this PR we submitted: https://github.com/vercel/next.js/pull/58276). I added a timeout which will cause it to wait up to a maximum _n_ seconds before issuing: ``` ERROR: 'npm run cerc_generate' exceeded CERC_MAX_GENERATE_TIME. ``` On the quoting itself, I plan on adding a new run-webapp command, but I realized I had a decent spot to do effect the quote replacement on-the-fly after all when I am already escaping the values for insertion/replacement into JS. The "dequoting" can be disabled with `CERC_RETAIN_ENV_QUOTES=true`. --- .../cerc-nextjs-base/Dockerfile | 2 +- .../container-build/cerc-nextjs-base/build.sh | 16 +++++++++ .../scripts/apply-runtime-env.sh | 3 ++ .../scripts/start-serving-app.sh | 35 +++++++++++++++---- 4 files changed, 49 insertions(+), 7 deletions(-) diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile index 69e38932..c2416b67 100644 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile @@ -39,4 +39,4 @@ EXPOSE 3000 COPY /scripts /scripts # Default command sleeps forever so docker doesn't kill it -CMD ["/scripts/start-serving-app.sh"] +ENTRYPOINT ["/scripts/start-serving-app.sh"] diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/build.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/build.sh index 3cf5f7f4..342dd3cd 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/build.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/build.sh @@ -11,3 +11,19 @@ CERC_CONTAINER_BUILD_DOCKERFILE=${CERC_CONTAINER_BUILD_DOCKERFILE:-$SCRIPT_DIR/D CERC_CONTAINER_BUILD_TAG=${CERC_CONTAINER_BUILD_TAG:-cerc/nextjs-base:local} docker build -t $CERC_CONTAINER_BUILD_TAG ${build_command_args} -f $CERC_CONTAINER_BUILD_DOCKERFILE $CERC_CONTAINER_BUILD_WORK_DIR + +if [ $? -eq 0 ] && [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/nextjs-base:local" ]; then + cat < /dev/null && pwd ) +CERC_MAX_GENERATE_TIME=${CERC_MAX_GENERATE_TIME:-60} +tpid="" + +ctrl_c() { + kill $tpid $(ps -ef | grep node | grep next | awk '{print $2}') 2>/dev/null +} + +trap ctrl_c INT CERC_BUILD_TOOL="${CERC_BUILD_TOOL}" if [ -z "$CERC_BUILD_TOOL" ]; then @@ -25,18 +34,32 @@ if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then jq -e '.scripts.cerc_generate' package.json >/dev/null if [ $? -eq 0 ]; then npm run cerc_generate > gen.out 2>&1 & - tail -n0 -f gen.out | sed '/rendered as static HTML/ q' + tail -f gen.out & + tpid=$! + count=0 - while [ $count -lt 10 ]; do + generate_done="false" + while [ $count -lt $CERC_MAX_GENERATE_TIME ]; do sleep 1 - ps -ef | grep 'node' | grep 'next' | grep 'generate' >/dev/null - if [ $? -ne 0 ]; then - break + grep 'rendered as static HTML' gen.out > /dev/null + if [ $? -eq 0 ]; then + generate_done="true" + ps -ef | grep 'node' | grep 'next' | grep 'generate' >/dev/null + if [ $? -ne 0 ]; then + break + fi else count=$((count + 1)) fi done - kill $(ps -ef |grep node | grep next | grep generate | awk '{print $2}') 2>/dev/null + + if [ $generate_done != "true" ]; then + echo "ERROR: 'npm run cerc_generate' not successful within CERC_MAX_GENERATE_TIME" 1>&2 + exit 1 + fi + + kill $tpid $(ps -ef | grep node | grep next | grep generate | awk '{print $2}') 2>/dev/null + tpid="" fi fi From 4ae4d3b61d8a7713e51d9d8116efdfd2e93cc427 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 14 Nov 2023 17:30:01 -0600 Subject: [PATCH 11/75] Print docker container logs in webapp test. (#649) --- .../cerc-nextjs-base/scripts/start-serving-app.sh | 9 ++------- tests/webapp-test/run-webapp-test.sh | 12 ++++++++---- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh index aec65f29..bf35bcdb 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh @@ -39,17 +39,12 @@ if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then count=0 generate_done="false" - while [ $count -lt $CERC_MAX_GENERATE_TIME ]; do + while [ $count -lt $CERC_MAX_GENERATE_TIME ] && [ "$generate_done" == "false" ]; do sleep 1 + count=$((count + 1)) grep 'rendered as static HTML' gen.out > /dev/null if [ $? -eq 0 ]; then generate_done="true" - ps -ef | grep 'node' | grep 'next' | grep 'generate' >/dev/null - if [ $? -ne 0 ]; then - break - fi - else - count=$((count + 1)) fi done diff --git a/tests/webapp-test/run-webapp-test.sh b/tests/webapp-test/run-webapp-test.sh index d32e0eba..5db382f8 100755 --- a/tests/webapp-test/run-webapp-test.sh +++ b/tests/webapp-test/run-webapp-test.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash set -e + if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi + # Dump environment variables for debugging echo "Environment variables:" env @@ -28,16 +30,18 @@ CHECK="SPECIAL_01234567890_TEST_STRING" set +e -CONTAINER_ID=$(docker run -p 3000:3000 -d cerc/test-progressive-web-app:local) +CONTAINER_ID=$(docker run -p 3000:3000 -d -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG cerc/test-progressive-web-app:local) sleep 3 -wget -O test.before -m http://localhost:3000 +wget -t 7 -O test.before -m http://localhost:3000 +docker logs $CONTAINER_ID docker remove -f $CONTAINER_ID -CONTAINER_ID=$(docker run -p 3000:3000 -e CERC_WEBAPP_DEBUG=$CHECK -d cerc/test-progressive-web-app:local) +CONTAINER_ID=$(docker run -p 3000:3000 -e CERC_WEBAPP_DEBUG=$CHECK -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG -d cerc/test-progressive-web-app:local) sleep 3 -wget -O test.after -m http://localhost:3000 +wget -t 7 -O test.after -m http://localhost:3000 +docker logs $CONTAINER_ID docker remove -f $CONTAINER_ID echo "###########################################################################" From 638fa016498a17a0ea24f3b4401459f464d208b5 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 14 Nov 2023 20:59:48 -0700 Subject: [PATCH 12/75] Support external stack file (#650) --- stack_orchestrator/build/build_containers.py | 32 +++++++++++++++---- stack_orchestrator/build/build_webapp.py | 4 +-- stack_orchestrator/constants.py | 16 ++++++++++ .../repos/setup_repositories.py | 20 ++++++++---- stack_orchestrator/util.py | 11 +++++++ 5 files changed, 68 insertions(+), 15 deletions(-) create mode 100644 stack_orchestrator/constants.py diff --git a/stack_orchestrator/build/build_containers.py b/stack_orchestrator/build/build_containers.py index 5b2748cc..e987c504 100644 --- a/stack_orchestrator/build/build_containers.py +++ b/stack_orchestrator/build/build_containers.py @@ -27,7 +27,7 @@ import subprocess import click import importlib.resources from pathlib import Path -from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config +from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external from stack_orchestrator.base import get_npm_registry_url # TODO: find a place for this @@ -58,7 +58,8 @@ def make_container_build_env(dev_root_path: str, return container_build_env -def process_container(container, +def process_container(stack: str, + container, container_build_dir: str, container_build_env: dict, dev_root_path: str, @@ -69,12 +70,29 @@ def process_container(container, ): if not quiet: print(f"Building: {container}") - build_dir = os.path.join(container_build_dir, container.replace("/", "-")) - build_script_filename = os.path.join(build_dir, "build.sh") + + default_container_tag = f"{container}:local" + container_build_env.update({"CERC_DEFAULT_CONTAINER_IMAGE_TAG": default_container_tag}) + + # Check if this is in an external stack + if stack_is_external(stack): + container_parent_dir = Path(stack).joinpath("container-build") + temp_build_dir = container_parent_dir.joinpath(container.replace("/", "-")) + temp_build_script_filename = temp_build_dir.joinpath("build.sh") + # Now check if the container exists in the external stack. + if not temp_build_script_filename.exists(): + # If not, revert to building an internal container + container_parent_dir = container_build_dir + else: + container_parent_dir = container_build_dir + + build_dir = container_parent_dir.joinpath(container.replace("/", "-")) + build_script_filename = build_dir.joinpath("build.sh") + if verbose: print(f"Build script filename: {build_script_filename}") if os.path.exists(build_script_filename): - build_command = build_script_filename + build_command = build_script_filename.as_posix() else: if verbose: print(f"No script file found: {build_script_filename}, using default build script") @@ -84,7 +102,7 @@ def process_container(container, repo_full_path = os.path.join(dev_root_path, repo_dir) repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir build_command = os.path.join(container_build_dir, - "default-build.sh") + f" {container}:local {repo_dir_or_build_dir}" + "default-build.sh") + f" {default_container_tag} {repo_dir_or_build_dir}" if not dry_run: if verbose: print(f"Executing: {build_command} with environment: {container_build_env}") @@ -158,7 +176,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args): for container in containers_in_scope: if include_exclude_check(container, include, exclude): - process_container(container, container_build_dir, container_build_env, + process_container(stack, container, container_build_dir, container_build_env, dev_root_path, quiet, verbose, dry_run, continue_on_error) else: if verbose: diff --git a/stack_orchestrator/build/build_webapp.py b/stack_orchestrator/build/build_webapp.py index 7a656ae7..ace334c4 100644 --- a/stack_orchestrator/build/build_webapp.py +++ b/stack_orchestrator/build/build_webapp.py @@ -60,7 +60,7 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args): container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug, force_rebuild, extra_build_args) - build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet, + build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet, verbose, dry_run, continue_on_error) @@ -73,5 +73,5 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args): webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1] container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local" - build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet, + build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet, verbose, dry_run, continue_on_error) diff --git a/stack_orchestrator/constants.py b/stack_orchestrator/constants.py new file mode 100644 index 00000000..1e24794c --- /dev/null +++ b/stack_orchestrator/constants.py @@ -0,0 +1,16 @@ +# Copyright © 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +stack_file_name = "stack.yml" diff --git a/stack_orchestrator/repos/setup_repositories.py b/stack_orchestrator/repos/setup_repositories.py index feca7897..3612aed0 100644 --- a/stack_orchestrator/repos/setup_repositories.py +++ b/stack_orchestrator/repos/setup_repositories.py @@ -25,7 +25,8 @@ import click import importlib.resources from pathlib import Path import yaml -from stack_orchestrator.util import include_exclude_check +from stack_orchestrator.constants import stack_file_name +from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit class GitProgress(git.RemoteProgress): @@ -238,13 +239,20 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches repos_in_scope = [] if stack: - # In order to be compatible with Python 3.8 we need to use this hack to get the path: - # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure - stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, "stack.yml") + if stack_is_external(stack): + stack_file_path = Path(stack).joinpath(stack_file_name) + else: + # In order to be compatible with Python 3.8 we need to use this hack to get the path: + # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure + stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, stack_file_name) + if not stack_file_path.exists(): + error_exit(f"stack {stack} does not exist") with stack_file_path: stack_config = yaml.safe_load(open(stack_file_path, "r")) - # TODO: syntax check the input here - repos_in_scope = stack_config['repos'] + if "repos" not in stack_config: + error_exit(f"stack {stack} does not define any repositories") + else: + repos_in_scope = stack_config["repos"] else: repos_in_scope = all_repos diff --git a/stack_orchestrator/util.py b/stack_orchestrator/util.py index d3b733a2..97d48963 100644 --- a/stack_orchestrator/util.py +++ b/stack_orchestrator/util.py @@ -150,6 +150,12 @@ def get_parsed_deployment_spec(spec_file): sys.exit(1) +def stack_is_external(stack: str): + # Bit of a hack: if the supplied stack string represents + # a path that exists then we assume it must be external + return Path(stack).exists() if stack is not None else False + + def get_yaml(): # See: https://stackoverflow.com/a/45701840/1701505 yaml = ruamel.yaml.YAML() @@ -167,3 +173,8 @@ def global_options(ctx): # TODO: hack def global_options2(ctx): return ctx.parent.obj + + +def error_exit(s): + print(f"ERROR: {s}") + sys.exit(1) From 2059d67dcabaf34f27fbd45ca6d325b28754a397 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 15 Nov 2023 11:54:27 -0600 Subject: [PATCH 13/75] Add run-webapp command. (#651) --- .../container-build/cerc-nextjs-base/build.sh | 2 +- .../deploy/compose/deploy_docker.py | 5 +- stack_orchestrator/deploy/deployer.py | 2 +- stack_orchestrator/deploy/k8s/deploy_k8s.py | 2 +- stack_orchestrator/deploy/run_webapp.py | 59 +++++++++++++++++++ stack_orchestrator/main.py | 2 + 6 files changed, 67 insertions(+), 5 deletions(-) create mode 100644 stack_orchestrator/deploy/run_webapp.py diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/build.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/build.sh index 342dd3cd..cca8d64b 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/build.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/build.sh @@ -23,7 +23,7 @@ Built host container for $CERC_CONTAINER_BUILD_WORK_DIR with tag: To test locally run: - docker run -p 3000:3000 --env-file /path/to/environment.env $CERC_CONTAINER_BUILD_TAG + laconic-so run-webapp --image $CERC_CONTAINER_BUILD_TAG --env-file /path/to/environment.env EOF fi diff --git a/stack_orchestrator/deploy/compose/deploy_docker.py b/stack_orchestrator/deploy/compose/deploy_docker.py index 79ab1482..b37f3cf0 100644 --- a/stack_orchestrator/deploy/compose/deploy_docker.py +++ b/stack_orchestrator/deploy/compose/deploy_docker.py @@ -61,9 +61,10 @@ class DockerDeployer(Deployer): except DockerException as e: raise DeployerException(e) - def run(self, image, command, user, volumes, entrypoint=None): + def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, detach=False): try: - return self.docker.run(image=image, command=command, user=user, volumes=volumes, entrypoint=entrypoint) + return self.docker.run(image=image, command=command, user=user, volumes=volumes, + entrypoint=entrypoint, envs=env, detach=detach, publish_all=True) except DockerException as e: raise DeployerException(e) diff --git a/stack_orchestrator/deploy/deployer.py b/stack_orchestrator/deploy/deployer.py index 68b0088a..79379c3d 100644 --- a/stack_orchestrator/deploy/deployer.py +++ b/stack_orchestrator/deploy/deployer.py @@ -44,7 +44,7 @@ class Deployer(ABC): pass @abstractmethod - def run(self, image, command, user, volumes, entrypoint): + def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, detach=False): pass diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index bc256b6b..627d6e0b 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -120,7 +120,7 @@ class K8sDeployer(Deployer): log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test") return log_stream_from_string(log_data) - def run(self, image, command, user, volumes, entrypoint=None): + def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, detach=False): # We need to figure out how to do this -- check why we're being called first pass diff --git a/stack_orchestrator/deploy/run_webapp.py b/stack_orchestrator/deploy/run_webapp.py new file mode 100644 index 00000000..8b1073b1 --- /dev/null +++ b/stack_orchestrator/deploy/run_webapp.py @@ -0,0 +1,59 @@ +# Copyright © 2022, 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# Builds webapp containers + +# env vars: +# CERC_REPO_BASE_DIR defaults to ~/cerc + +# TODO: display the available list of containers; allow re-build of either all or specific containers + +import hashlib +import click + +from dotenv import dotenv_values +from stack_orchestrator.deploy.deployer_factory import getDeployer + + +@click.command() +@click.option("--image", help="image to deploy", required=True) +@click.option("--deploy-to", default="compose", help="deployment type ([Docker] 'compose' or 'k8s')") +@click.option("--env-file", help="environment file for webapp") +@click.pass_context +def command(ctx, image, deploy_to, env_file): + '''build the specified webapp container''' + + env = {} + if env_file: + env = dotenv_values(env_file) + + unique_cluster_descriptor = f"{image},{env}" + hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest() + cluster = f"laconic-webapp-{hash}" + + deployer = getDeployer(deploy_to, + deployment_dir=None, + compose_files=None, + compose_project_name=cluster, + compose_env_file=None) + + container = deployer.run(image, command=[], user=None, volumes=[], entrypoint=None, env=env, detach=True) + + # Make configurable? + webappPort = "3000/tcp" + # TODO: This assumes a Docker container object... + if webappPort in container.network_settings.ports: + mapping = container.network_settings.ports[webappPort][0] + print(f"""Image: {image}\nID: {container.id}\nURL: http://localhost:{mapping['HostPort']}""") diff --git a/stack_orchestrator/main.py b/stack_orchestrator/main.py index 0b0585e0..8ee8ae61 100644 --- a/stack_orchestrator/main.py +++ b/stack_orchestrator/main.py @@ -20,6 +20,7 @@ from stack_orchestrator.repos import setup_repositories from stack_orchestrator.build import build_containers from stack_orchestrator.build import build_npms from stack_orchestrator.build import build_webapp +from stack_orchestrator.deploy import run_webapp from stack_orchestrator.deploy import deploy from stack_orchestrator import version from stack_orchestrator.deploy import deployment @@ -50,6 +51,7 @@ cli.add_command(setup_repositories.command, "setup-repositories") cli.add_command(build_containers.command, "build-containers") cli.add_command(build_npms.command, "build-npms") cli.add_command(build_webapp.command, "build-webapp") +cli.add_command(run_webapp.command, "run-webapp") cli.add_command(deploy.command, "deploy") # deploy is an alias for deploy-system cli.add_command(deploy.command, "deploy-system") cli.add_command(deployment.command, "deployment") From d37f80553d1a7048f30d5312ed4057e79c9b0b69 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 15 Nov 2023 12:28:07 -0600 Subject: [PATCH 14/75] Add webapp doc (#652) --- docs/webapp.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 docs/webapp.md diff --git a/docs/webapp.md b/docs/webapp.md new file mode 100644 index 00000000..19122d46 --- /dev/null +++ b/docs/webapp.md @@ -0,0 +1,42 @@ +### Building and Running Webapps + +It is possible to build and run webapps using the `build-webapp` and `run-webapp` subcommand. + +To make it easier to build once, and deploy to with varying configuration, compilation and static +page generation are separated in the `build-webapp` and `run-webapp` steps, and the use of the +environment variables via `process.env` is detected at compile-time and placeholder substituted +which will be filled in at runtime. + +This offers much more flexibilty in configuration and deployment than standard build methods. + +## Build + +``` +$ cd ~/cerc +$ git clone git@git.vdb.to:cerc-io/test-progressive-web-app.git +$ laconic-so build-webapp --source-repo ~/cerc/test-progressive-web-app +... +Successfully tagged cerc/test-progressive-web-app:local + + +################################################################# + +Built host container for ~/cerc/test-progressive-web-app with tag: + + cerc/test-progressive-web-app:local + +To test locally run: + + laconic-so run-webapp --image cerc/test-progressive-web-app:local --env-file /path/to/environment.env + +``` + +## Run + +``` +$ laconic-so run-webapp --image cerc/test-progressive-web-app:local --env-file ~/tmp/env.igloo + +Image: cerc/test-progressive-web-app:local +ID: 4c6e893bf436b3e91a2b92ce37e30e499685131705700bd92a90d2eb14eefd05 +URL: http://localhost:32768 +``` From a13f841f34eff6e4866c9e36a2c5a86215768b05 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 15 Nov 2023 12:37:30 -0600 Subject: [PATCH 15/75] Update webapp.md --- docs/webapp.md | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/docs/webapp.md b/docs/webapp.md index 19122d46..ed321cb4 100644 --- a/docs/webapp.md +++ b/docs/webapp.md @@ -1,25 +1,21 @@ ### Building and Running Webapps -It is possible to build and run webapps using the `build-webapp` and `run-webapp` subcommand. +It is possible to build and run Next.js webapps using the `build-webapp` and `run-webapp` subcommands. -To make it easier to build once, and deploy to with varying configuration, compilation and static -page generation are separated in the `build-webapp` and `run-webapp` steps, and the use of the -environment variables via `process.env` is detected at compile-time and placeholder substituted -which will be filled in at runtime. +To make it easier to build once and deploy into different environments and with different configuration, +compilation and static page generation are separated in the `build-webapp` and `run-webapp` steps. -This offers much more flexibilty in configuration and deployment than standard build methods. +This offers much more flexibilty than standard Next.js build methods, since any environment variables accessed +via `process.env`, whether for pages or for API, will have values drawn from their runtime deployment environment, +not their build environment. -## Build +## Building ``` $ cd ~/cerc $ git clone git@git.vdb.to:cerc-io/test-progressive-web-app.git $ laconic-so build-webapp --source-repo ~/cerc/test-progressive-web-app ... -Successfully tagged cerc/test-progressive-web-app:local - - -################################################################# Built host container for ~/cerc/test-progressive-web-app with tag: @@ -31,7 +27,7 @@ To test locally run: ``` -## Run +## Running ``` $ laconic-so run-webapp --image cerc/test-progressive-web-app:local --env-file ~/tmp/env.igloo From 99005657148dc2cd1e3a1a23cd0dcc403c49ccd2 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 15 Nov 2023 12:48:58 -0600 Subject: [PATCH 16/75] Update webapp.md --- docs/webapp.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/webapp.md b/docs/webapp.md index ed321cb4..ec1ee252 100644 --- a/docs/webapp.md +++ b/docs/webapp.md @@ -11,6 +11,11 @@ not their build environment. ## Building +Building usually requires no additional configuration. By default, the Next.js version specified in `package.json` +is used, and either `yarn` or `npm` will be used automatically depending on which lock files are present. These +can be overidden with the build arguments `CERC_NEXT_VERSION` and `CERC_BUILD_TOOL` respectively. For example: `--extra-build-args "--build-arg CERC_NEXT_VERSION=13.4.12"` + +**Example**: ``` $ cd ~/cerc $ git clone git@git.vdb.to:cerc-io/test-progressive-web-app.git @@ -29,10 +34,21 @@ To test locally run: ## Running +With `run-webapp` a new container will be launched with runtime configuration provided by `--env-file` (if specified) and published on an available port. Multiple instances can be launched with different configuration. + +**Example**: ``` -$ laconic-so run-webapp --image cerc/test-progressive-web-app:local --env-file ~/tmp/env.igloo +# Production env +$ laconic-so run-webapp --image cerc/test-progressive-web-app:local --env-file /path/to/environment/staging.env Image: cerc/test-progressive-web-app:local ID: 4c6e893bf436b3e91a2b92ce37e30e499685131705700bd92a90d2eb14eefd05 URL: http://localhost:32768 + +# Dev env +$ laconic-so run-webapp --image cerc/test-progressive-web-app:local --env-file /path/to/environment/dev.env + +Image: cerc/test-progressive-web-app:local +ID: 9ab96494f563aafb6c057d88df58f9eca81b90f8721a4e068493a289a976051c +URL: http://localhost:32769 ``` From 1e9d24a8ce6c6beca572fc5d93258499296db043 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 15 Nov 2023 12:52:34 -0600 Subject: [PATCH 17/75] Update webapp.md --- docs/webapp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/webapp.md b/docs/webapp.md index ec1ee252..3b1d0609 100644 --- a/docs/webapp.md +++ b/docs/webapp.md @@ -39,7 +39,7 @@ With `run-webapp` a new container will be launched with runtime configuration pr **Example**: ``` # Production env -$ laconic-so run-webapp --image cerc/test-progressive-web-app:local --env-file /path/to/environment/staging.env +$ laconic-so run-webapp --image cerc/test-progressive-web-app:local --env-file /path/to/environment/production.env Image: cerc/test-progressive-web-app:local ID: 4c6e893bf436b3e91a2b92ce37e30e499685131705700bd92a90d2eb14eefd05 From 70529c43e779f2ed81d750d61c95192bf9d7e20e Mon Sep 17 00:00:00 2001 From: Nabarun Gogoi Date: Thu, 16 Nov 2023 16:27:41 +0530 Subject: [PATCH 18/75] Upgrade merkl and sushiswap watcher versions (#654) --- stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml | 2 +- stack_orchestrator/data/stacks/sushiswap-v3/stack.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml index 8b010a24..8f5cb7ee 100644 --- a/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml +++ b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml @@ -2,7 +2,7 @@ version: "1.0" name: merkl-sushiswap-v3 description: "SushiSwap v3 watcher stack" repos: - - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.1 + - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.2 containers: - cerc/watcher-merkl-sushiswap-v3 pods: diff --git a/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml b/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml index 66065b5d..05350996 100644 --- a/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml +++ b/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml @@ -2,7 +2,7 @@ version: "1.0" name: sushiswap-v3 description: "SushiSwap v3 watcher stack" repos: - - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.1 + - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.2 containers: - cerc/watcher-sushiswap-v3 pods: From 5c80887215481c17ee41a68a3559f66f4fa2e925 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Thu, 16 Nov 2023 12:58:03 -0600 Subject: [PATCH 19/75] Fix missing tty parameter. (#653) --- stack_orchestrator/deploy/deploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack_orchestrator/deploy/deploy.py b/stack_orchestrator/deploy/deploy.py index 1c467067..f931a0d0 100644 --- a/stack_orchestrator/deploy/deploy.py +++ b/stack_orchestrator/deploy/deploy.py @@ -155,7 +155,7 @@ def exec_operation(ctx, extra_args): if global_context.verbose: print(f"Running compose exec {service_name} {command_to_exec}") try: - ctx.obj.deployer.execute(service_name, command_to_exec, envs=container_exec_env) + ctx.obj.deployer.execute(service_name, command_to_exec, envs=container_exec_env, tty=True) except DeployerException: print("container command returned error exit status") From c9c6a0eee3816c74e01b8cda1748daf0bd7654a5 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Mon, 20 Nov 2023 09:12:57 -0700 Subject: [PATCH 20/75] Changes for remote k8s (#655) --- stack_orchestrator/constants.py | 6 ++ .../deploy/compose/deploy_docker.py | 7 ++- stack_orchestrator/deploy/deploy.py | 6 +- stack_orchestrator/deploy/deployer_factory.py | 13 ++-- stack_orchestrator/deploy/deployment.py | 7 ++- .../deploy/deployment_create.py | 62 ++++++++++++------- stack_orchestrator/deploy/k8s/deploy_k8s.py | 54 ++++++++++------ tests/k8s-deploy/run-deploy-test.sh | 2 +- 8 files changed, 101 insertions(+), 56 deletions(-) diff --git a/stack_orchestrator/constants.py b/stack_orchestrator/constants.py index 1e24794c..f15e8870 100644 --- a/stack_orchestrator/constants.py +++ b/stack_orchestrator/constants.py @@ -14,3 +14,9 @@ # along with this program. If not, see . stack_file_name = "stack.yml" +compose_deploy_type = "compose" +k8s_kind_deploy_type = "k8s-kind" +k8s_deploy_type = "k8s" +kube_config_key = "kube-config" +kind_config_filename = "kind-config.yml" +kube_config_filename = "kubeconfig.yml" diff --git a/stack_orchestrator/deploy/compose/deploy_docker.py b/stack_orchestrator/deploy/compose/deploy_docker.py index b37f3cf0..fc249ebc 100644 --- a/stack_orchestrator/deploy/compose/deploy_docker.py +++ b/stack_orchestrator/deploy/compose/deploy_docker.py @@ -20,10 +20,12 @@ from stack_orchestrator.deploy.deployer import Deployer, DeployerException, Depl class DockerDeployer(Deployer): name: str = "compose" + type: str - def __init__(self, deployment_dir, compose_files, compose_project_name, compose_env_file) -> None: + def __init__(self, type, deployment_dir, compose_files, compose_project_name, compose_env_file) -> None: self.docker = DockerClient(compose_files=compose_files, compose_project_name=compose_project_name, compose_env_file=compose_env_file) + self.type = type def up(self, detach, services): try: @@ -70,9 +72,8 @@ class DockerDeployer(Deployer): class DockerDeployerConfigGenerator(DeployerConfigGenerator): - config_file_name: str = "kind-config.yml" - def __init__(self) -> None: + def __init__(self, type: str) -> None: super().__init__() # Nothing needed at present for the docker deployer diff --git a/stack_orchestrator/deploy/deploy.py b/stack_orchestrator/deploy/deploy.py index f931a0d0..c01a7e08 100644 --- a/stack_orchestrator/deploy/deploy.py +++ b/stack_orchestrator/deploy/deploy.py @@ -39,7 +39,7 @@ from stack_orchestrator.deploy.deployment_create import setup as deployment_setu @click.option("--exclude", help="don\'t start these components") @click.option("--env-file", help="env file to be used") @click.option("--cluster", help="specify a non-default cluster name") -@click.option("--deploy-to", help="cluster system to deploy to (compose or k8s)") +@click.option("--deploy-to", help="cluster system to deploy to (compose or k8s or k8s-kind)") @click.pass_context def command(ctx, include, exclude, env_file, cluster, deploy_to): '''deploy a stack''' @@ -62,11 +62,11 @@ def command(ctx, include, exclude, env_file, cluster, deploy_to): def create_deploy_context( - global_context, deployment_context: DeploymentContext, stack, include, exclude, cluster, env_file, deployer): + global_context, deployment_context: DeploymentContext, stack, include, exclude, cluster, env_file, deploy_to): cluster_context = _make_cluster_context(global_context, stack, include, exclude, cluster, env_file) deployment_dir = deployment_context.deployment_dir if deployment_context else None # See: https://gabrieldemarmiesse.github.io/python-on-whales/sub-commands/compose/ - deployer = getDeployer(deployer, deployment_dir, compose_files=cluster_context.compose_files, + deployer = getDeployer(deploy_to, deployment_dir, compose_files=cluster_context.compose_files, compose_project_name=cluster_context.cluster, compose_env_file=cluster_context.env_file) return DeployCommandContext(stack, cluster_context, deployer) diff --git a/stack_orchestrator/deploy/deployer_factory.py b/stack_orchestrator/deploy/deployer_factory.py index 5d515418..de2808c5 100644 --- a/stack_orchestrator/deploy/deployer_factory.py +++ b/stack_orchestrator/deploy/deployer_factory.py @@ -13,23 +13,24 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from stack_orchestrator import constants from stack_orchestrator.deploy.k8s.deploy_k8s import K8sDeployer, K8sDeployerConfigGenerator from stack_orchestrator.deploy.compose.deploy_docker import DockerDeployer, DockerDeployerConfigGenerator def getDeployerConfigGenerator(type: str): if type == "compose" or type is None: - return DockerDeployerConfigGenerator() - elif type == "k8s": - return K8sDeployerConfigGenerator() + return DockerDeployerConfigGenerator(type) + elif type == constants.k8s_deploy_type or type == constants.k8s_kind_deploy_type: + return K8sDeployerConfigGenerator(type) else: print(f"ERROR: deploy-to {type} is not valid") def getDeployer(type: str, deployment_dir, compose_files, compose_project_name, compose_env_file): if type == "compose" or type is None: - return DockerDeployer(deployment_dir, compose_files, compose_project_name, compose_env_file) - elif type == "k8s": - return K8sDeployer(deployment_dir, compose_files, compose_project_name, compose_env_file) + return DockerDeployer(type, deployment_dir, compose_files, compose_project_name, compose_env_file) + elif type == type == constants.k8s_deploy_type or type == constants.k8s_kind_deploy_type: + return K8sDeployer(type, deployment_dir, compose_files, compose_project_name, compose_env_file) else: print(f"ERROR: deploy-to {type} is not valid") diff --git a/stack_orchestrator/deploy/deployment.py b/stack_orchestrator/deploy/deployment.py index e22d7dcc..586a2b2a 100644 --- a/stack_orchestrator/deploy/deployment.py +++ b/stack_orchestrator/deploy/deployment.py @@ -16,6 +16,7 @@ import click from pathlib import Path import sys +from stack_orchestrator import constants from stack_orchestrator.deploy.deploy import up_operation, down_operation, ps_operation, port_operation from stack_orchestrator.deploy.deploy import exec_operation, logs_operation, create_deploy_context from stack_orchestrator.deploy.deployment_context import DeploymentContext @@ -50,8 +51,12 @@ def make_deploy_context(ctx): stack_file_path = context.get_stack_file() env_file = context.get_env_file() cluster_name = context.get_cluster_name() + if "deploy-to" in context.spec.obj: + deployment_type = context.spec.obj["deploy-to"] + else: + deployment_type = constants.compose_deploy_type return create_deploy_context(ctx.parent.parent.obj, context, stack_file_path, None, None, cluster_name, env_file, - context.spec.obj["deploy-to"]) + deployment_type) @command.command() diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py index c00c0dc6..c9ba3a10 100644 --- a/stack_orchestrator/deploy/deployment_create.py +++ b/stack_orchestrator/deploy/deployment_create.py @@ -21,16 +21,17 @@ from typing import List import random from shutil import copy, copyfile, copytree import sys +from stack_orchestrator import constants from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config, global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts, - get_pod_script_paths, get_plugin_code_paths) + get_pod_script_paths, get_plugin_code_paths, error_exit) from stack_orchestrator.deploy.deploy_types import LaconicStackSetupCommand from stack_orchestrator.deploy.deployer_factory import getDeployerConfigGenerator from stack_orchestrator.deploy.deployment_context import DeploymentContext def _make_default_deployment_dir(): - return "deployment-001" + return Path("deployment-001") def _get_ports(stack): @@ -248,17 +249,21 @@ def _parse_config_variables(variable_values: str): @click.command() @click.option("--config", help="Provide config variables for the deployment") +@click.option("--kube-config", help="Provide a config file for a k8s deployment") @click.option("--output", required=True, help="Write yaml spec file here") @click.option("--map-ports-to-host", required=False, help="Map ports to the host as one of: any-variable-random (default), " "localhost-same, any-same, localhost-fixed-random, any-fixed-random") @click.pass_context -def init(ctx, config, output, map_ports_to_host): +def init(ctx, config, kube_config, output, map_ports_to_host): yaml = get_yaml() stack = global_options(ctx).stack debug = global_options(ctx).debug + deployer_type = ctx.obj.deployer.type default_spec_file_content = call_stack_deploy_init(ctx.obj) - spec_file_content = {"stack": stack, "deploy-to": ctx.obj.deployer.name} + spec_file_content = {"stack": stack, "deploy-to": deployer_type} + if deployer_type == "k8s": + spec_file_content.update({constants.kube_config_key: kube_config}) if default_spec_file_content: spec_file_content.update(default_spec_file_content) config_variables = _parse_config_variables(config) @@ -296,6 +301,12 @@ def _write_config_file(spec_file: Path, config_env_file: Path): output_file.write(f"{variable_name}={variable_value}\n") +def _write_kube_config_file(external_path: Path, internal_path: Path): + if not external_path.exists(): + error_exit(f"Kube config file {external_path} does not exist") + copyfile(external_path, internal_path) + + def _copy_files_to_directory(file_paths: List[Path], directory: Path): for path in file_paths: # Using copy to preserve the execute bit @@ -310,29 +321,34 @@ def _copy_files_to_directory(file_paths: List[Path], directory: Path): @click.option("--initial-peers", help="Initial set of persistent peers") @click.pass_context def create(ctx, spec_file, deployment_dir, network_dir, initial_peers): - # This function fails with a useful error message if the file doens't exist parsed_spec = get_parsed_deployment_spec(spec_file) stack_name = parsed_spec["stack"] + deployment_type = parsed_spec["deploy-to"] stack_file = get_stack_file_path(stack_name) parsed_stack = get_parsed_stack_config(stack_name) if global_options(ctx).debug: print(f"parsed spec: {parsed_spec}") if deployment_dir is None: - deployment_dir = _make_default_deployment_dir() - if os.path.exists(deployment_dir): - print(f"Error: {deployment_dir} already exists") - sys.exit(1) - os.mkdir(deployment_dir) + deployment_dir_path = _make_default_deployment_dir() + else: + deployment_dir_path = Path(deployment_dir) + if deployment_dir_path.exists(): + error_exit(f"{deployment_dir_path} already exists") + os.mkdir(deployment_dir_path) # Copy spec file and the stack file into the deployment dir - copyfile(spec_file, os.path.join(deployment_dir, "spec.yml")) - copyfile(stack_file, os.path.join(deployment_dir, os.path.basename(stack_file))) + copyfile(spec_file, deployment_dir_path.joinpath("spec.yml")) + copyfile(stack_file, deployment_dir_path.joinpath(os.path.basename(stack_file))) # Copy any config varibles from the spec file into an env file suitable for compose - _write_config_file(spec_file, os.path.join(deployment_dir, "config.env")) + _write_config_file(spec_file, deployment_dir_path.joinpath("config.env")) + # Copy any k8s config file into the deployment dir + if deployment_type == "k8s": + _write_kube_config_file(Path(parsed_spec[constants.kube_config_key]), + deployment_dir_path.joinpath(constants.kube_config_filename)) # Copy the pod files into the deployment dir, fixing up content pods = get_pod_list(parsed_stack) - destination_compose_dir = os.path.join(deployment_dir, "compose") + destination_compose_dir = deployment_dir_path.joinpath("compose") os.mkdir(destination_compose_dir) - destination_pods_dir = os.path.join(deployment_dir, "pods") + destination_pods_dir = deployment_dir_path.joinpath("pods") os.mkdir(destination_pods_dir) data_dir = Path(__file__).absolute().parent.parent.joinpath("data") yaml = get_yaml() @@ -340,12 +356,12 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers): pod_file_path = get_pod_file_path(parsed_stack, pod) parsed_pod_file = yaml.load(open(pod_file_path, "r")) extra_config_dirs = _find_extra_config_dirs(parsed_pod_file, pod) - destination_pod_dir = os.path.join(destination_pods_dir, pod) + destination_pod_dir = destination_pods_dir.joinpath(pod) os.mkdir(destination_pod_dir) if global_options(ctx).debug: print(f"extra config dirs: {extra_config_dirs}") _fixup_pod_file(parsed_pod_file, parsed_spec, destination_compose_dir) - with open(os.path.join(destination_compose_dir, "docker-compose-%s.yml" % pod), "w") as output_file: + with open(destination_compose_dir.joinpath("docker-compose-%s.yml" % pod), "w") as output_file: yaml.dump(parsed_pod_file, output_file) # Copy the config files for the pod, if any config_dirs = {pod} @@ -353,13 +369,13 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers): for config_dir in config_dirs: source_config_dir = data_dir.joinpath("config", config_dir) if os.path.exists(source_config_dir): - destination_config_dir = os.path.join(deployment_dir, "config", config_dir) + destination_config_dir = deployment_dir_path.joinpath("config", config_dir) # If the same config dir appears in multiple pods, it may already have been copied if not os.path.exists(destination_config_dir): copytree(source_config_dir, destination_config_dir) # Copy the script files for the pod, if any if pod_has_scripts(parsed_stack, pod): - destination_script_dir = os.path.join(destination_pod_dir, "scripts") + destination_script_dir = destination_pod_dir.joinpath("scripts") os.mkdir(destination_script_dir) script_paths = get_pod_script_paths(parsed_stack, pod) _copy_files_to_directory(script_paths, destination_script_dir) @@ -369,11 +385,11 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers): deployment_command_context = ctx.obj deployment_command_context.stack = stack_name deployment_context = DeploymentContext() - deployment_context.init(Path(deployment_dir)) + deployment_context.init(deployment_dir_path) # Call the deployer to generate any deployer-specific files (e.g. for kind) - deployer_config_generator = getDeployerConfigGenerator(parsed_spec["deploy-to"]) - # TODO: make deployment_dir a Path above - deployer_config_generator.generate(Path(deployment_dir)) + deployer_config_generator = getDeployerConfigGenerator(deployment_type) + # TODO: make deployment_dir_path a Path above + deployer_config_generator.generate(deployment_dir_path) call_stack_deploy_create(deployment_context, [network_dir, initial_peers]) diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index 627d6e0b..44eb9f0e 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -16,6 +16,7 @@ from pathlib import Path from kubernetes import client, config +from stack_orchestrator import constants from stack_orchestrator.deploy.deployer import Deployer, DeployerConfigGenerator from stack_orchestrator.deploy.k8s.helpers import create_cluster, destroy_cluster, load_images_into_kind from stack_orchestrator.deploy.k8s.helpers import pods_in_deployment, log_stream_from_string, generate_kind_config @@ -25,6 +26,7 @@ from stack_orchestrator.opts import opts class K8sDeployer(Deployer): name: str = "k8s" + type: str core_api: client.CoreV1Api apps_api: client.AppsV1Api k8s_namespace: str = "default" @@ -32,28 +34,35 @@ class K8sDeployer(Deployer): cluster_info : ClusterInfo deployment_dir: Path - def __init__(self, deployment_dir, compose_files, compose_project_name, compose_env_file) -> None: + def __init__(self, type, deployment_dir, compose_files, compose_project_name, compose_env_file) -> None: if (opts.o.debug): print(f"Deployment dir: {deployment_dir}") print(f"Compose files: {compose_files}") print(f"Project name: {compose_project_name}") print(f"Env file: {compose_env_file}") + print(f"Type: {type}") + self.type = type self.deployment_dir = deployment_dir self.kind_cluster_name = compose_project_name self.cluster_info = ClusterInfo() self.cluster_info.int(compose_files, compose_env_file) def connect_api(self): - config.load_kube_config(context=f"kind-{self.kind_cluster_name}") + if self.is_kind(): + config.load_kube_config(context=f"kind-{self.kind_cluster_name}") + else: + # Get the config file and pass to load_kube_config() + config.load_kube_config(config_file=self.deployment_dir.joinpath(constants.kube_config_filename).as_posix()) self.core_api = client.CoreV1Api() self.apps_api = client.AppsV1Api() def up(self, detach, services): - # Create the kind cluster - create_cluster(self.kind_cluster_name, self.deployment_dir.joinpath("kind-config.yml")) + if self.is_kind(): + # Create the kind cluster + create_cluster(self.kind_cluster_name, self.deployment_dir.joinpath(constants.kind_config_filename)) + # Ensure the referenced containers are copied into kind + load_images_into_kind(self.kind_cluster_name, self.cluster_info.image_set) self.connect_api() - # Ensure the referenced containers are copied into kind - load_images_into_kind(self.kind_cluster_name, self.cluster_info.image_set) # Create the host-path-mounted PVs for this deployment pvs = self.cluster_info.get_pvs() @@ -89,8 +98,9 @@ class K8sDeployer(Deployer): def down(self, timeout, volumes): # Delete the k8s objects - # Destroy the kind cluster - destroy_cluster(self.kind_cluster_name) + if self.is_kind(): + # Destroy the kind cluster + destroy_cluster(self.kind_cluster_name) def ps(self): self.connect_api() @@ -124,20 +134,26 @@ class K8sDeployer(Deployer): # We need to figure out how to do this -- check why we're being called first pass + def is_kind(self): + return self.type == "k8s-kind" + class K8sDeployerConfigGenerator(DeployerConfigGenerator): - config_file_name: str = "kind-config.yml" + type: str - def __init__(self) -> None: + def __init__(self, type: str) -> None: + self.type = type super().__init__() def generate(self, deployment_dir: Path): - # Check the file isn't already there - # Get the config file contents - content = generate_kind_config(deployment_dir) - if opts.o.debug: - print(f"kind config is: {content}") - config_file = deployment_dir.joinpath(self.config_file_name) - # Write the file - with open(config_file, "w") as output_file: - output_file.write(content) + # No need to do this for the remote k8s case + if self.type == "k8s-kind": + # Check the file isn't already there + # Get the config file contents + content = generate_kind_config(deployment_dir) + if opts.o.debug: + print(f"kind config is: {content}") + config_file = deployment_dir.joinpath(constants.kind_config_filename) + # Write the file + with open(config_file, "w") as output_file: + output_file.write(content) diff --git a/tests/k8s-deploy/run-deploy-test.sh b/tests/k8s-deploy/run-deploy-test.sh index 91c7890c..b7ee9dd0 100755 --- a/tests/k8s-deploy/run-deploy-test.sh +++ b/tests/k8s-deploy/run-deploy-test.sh @@ -21,7 +21,7 @@ mkdir -p $CERC_REPO_BASE_DIR # Test basic stack-orchestrator deploy test_deployment_dir=$CERC_REPO_BASE_DIR/test-deployment-dir test_deployment_spec=$CERC_REPO_BASE_DIR/test-deployment-spec.yml -$TEST_TARGET_SO --stack test deploy --deploy-to k8s init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED +$TEST_TARGET_SO --stack test deploy --deploy-to k8s-kind init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED # Check the file now exists if [ ! -f "$test_deployment_spec" ]; then echo "deploy init test: spec file not present" From f6624cb33a5a3b0d9a5d1fb013a9da0a18f386b1 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Mon, 20 Nov 2023 20:23:55 -0700 Subject: [PATCH 21/75] Add image push command (#656) --- stack_orchestrator/constants.py | 2 + .../deploy/compose/deploy_docker.py | 3 +- stack_orchestrator/deploy/deploy.py | 13 +- stack_orchestrator/deploy/deploy_util.py | 30 ++++- stack_orchestrator/deploy/deployer_factory.py | 6 +- stack_orchestrator/deploy/deployment.py | 16 ++- .../deploy/deployment_create.py | 14 ++- stack_orchestrator/deploy/images.py | 62 +++++++++ stack_orchestrator/deploy/k8s/cluster_info.py | 24 ++-- stack_orchestrator/deploy/k8s/deploy_k8s.py | 39 +++++- stack_orchestrator/deploy/k8s/helpers.py | 15 +-- stack_orchestrator/deploy/run_webapp.py | 118 +++++++++--------- 12 files changed, 238 insertions(+), 104 deletions(-) create mode 100644 stack_orchestrator/deploy/images.py diff --git a/stack_orchestrator/constants.py b/stack_orchestrator/constants.py index f15e8870..aedc4f3c 100644 --- a/stack_orchestrator/constants.py +++ b/stack_orchestrator/constants.py @@ -18,5 +18,7 @@ compose_deploy_type = "compose" k8s_kind_deploy_type = "k8s-kind" k8s_deploy_type = "k8s" kube_config_key = "kube-config" +deploy_to_key = "deploy-to" +image_resigtry_key = "image-registry" kind_config_filename = "kind-config.yml" kube_config_filename = "kubeconfig.yml" diff --git a/stack_orchestrator/deploy/compose/deploy_docker.py b/stack_orchestrator/deploy/compose/deploy_docker.py index fc249ebc..4b4e7426 100644 --- a/stack_orchestrator/deploy/compose/deploy_docker.py +++ b/stack_orchestrator/deploy/compose/deploy_docker.py @@ -16,13 +16,14 @@ from pathlib import Path from python_on_whales import DockerClient, DockerException from stack_orchestrator.deploy.deployer import Deployer, DeployerException, DeployerConfigGenerator +from stack_orchestrator.deploy.deployment_context import DeploymentContext class DockerDeployer(Deployer): name: str = "compose" type: str - def __init__(self, type, deployment_dir, compose_files, compose_project_name, compose_env_file) -> None: + def __init__(self, type, deployment_context: DeploymentContext, compose_files, compose_project_name, compose_env_file) -> None: self.docker = DockerClient(compose_files=compose_files, compose_project_name=compose_project_name, compose_env_file=compose_env_file) self.type = type diff --git a/stack_orchestrator/deploy/deploy.py b/stack_orchestrator/deploy/deploy.py index c01a7e08..32c13a61 100644 --- a/stack_orchestrator/deploy/deploy.py +++ b/stack_orchestrator/deploy/deploy.py @@ -62,11 +62,16 @@ def command(ctx, include, exclude, env_file, cluster, deploy_to): def create_deploy_context( - global_context, deployment_context: DeploymentContext, stack, include, exclude, cluster, env_file, deploy_to): + global_context, + deployment_context: DeploymentContext, + stack, + include, + exclude, + cluster, + env_file, + deploy_to) -> DeployCommandContext: cluster_context = _make_cluster_context(global_context, stack, include, exclude, cluster, env_file) - deployment_dir = deployment_context.deployment_dir if deployment_context else None - # See: https://gabrieldemarmiesse.github.io/python-on-whales/sub-commands/compose/ - deployer = getDeployer(deploy_to, deployment_dir, compose_files=cluster_context.compose_files, + deployer = getDeployer(deploy_to, deployment_context, compose_files=cluster_context.compose_files, compose_project_name=cluster_context.cluster, compose_env_file=cluster_context.env_file) return DeployCommandContext(stack, cluster_context, deployer) diff --git a/stack_orchestrator/deploy/deploy_util.py b/stack_orchestrator/deploy/deploy_util.py index 9829490d..8b812d3a 100644 --- a/stack_orchestrator/deploy/deploy_util.py +++ b/stack_orchestrator/deploy/deploy_util.py @@ -14,9 +14,10 @@ # along with this program. If not, see . import os -from typing import List +from typing import List, Any from stack_orchestrator.deploy.deploy_types import DeployCommandContext, VolumeMapping from stack_orchestrator.util import get_parsed_stack_config, get_yaml, get_compose_file_dir, get_pod_list +from stack_orchestrator.opts import opts def _container_image_from_service(stack: str, service: str): @@ -37,6 +38,33 @@ def _container_image_from_service(stack: str, service: str): return image_name +def parsed_pod_files_map_from_file_names(pod_files): + parsed_pod_yaml_map : Any = {} + for pod_file in pod_files: + with open(pod_file, "r") as pod_file_descriptor: + parsed_pod_file = get_yaml().load(pod_file_descriptor) + parsed_pod_yaml_map[pod_file] = parsed_pod_file + if opts.o.debug: + print(f"parsed_pod_yaml_map: {parsed_pod_yaml_map}") + return parsed_pod_yaml_map + + +def images_for_deployment(pod_files: List[str]): + image_set = set() + parsed_pod_yaml_map = parsed_pod_files_map_from_file_names(pod_files) + # Find the set of images in the pods + for pod_name in parsed_pod_yaml_map: + pod = parsed_pod_yaml_map[pod_name] + services = pod["services"] + for service_name in services: + service_info = services[service_name] + image = service_info["image"] + image_set.add(image) + if opts.o.debug: + print(f"image_set: {image_set}") + return image_set + + def _volumes_to_docker(mounts: List[VolumeMapping]): # Example from doc: [("/", "/host"), ("/etc/hosts", "/etc/hosts", "rw")] result = [] diff --git a/stack_orchestrator/deploy/deployer_factory.py b/stack_orchestrator/deploy/deployer_factory.py index de2808c5..959c1b7a 100644 --- a/stack_orchestrator/deploy/deployer_factory.py +++ b/stack_orchestrator/deploy/deployer_factory.py @@ -27,10 +27,10 @@ def getDeployerConfigGenerator(type: str): print(f"ERROR: deploy-to {type} is not valid") -def getDeployer(type: str, deployment_dir, compose_files, compose_project_name, compose_env_file): +def getDeployer(type: str, deployment_context, compose_files, compose_project_name, compose_env_file): if type == "compose" or type is None: - return DockerDeployer(type, deployment_dir, compose_files, compose_project_name, compose_env_file) + return DockerDeployer(type, deployment_context, compose_files, compose_project_name, compose_env_file) elif type == type == constants.k8s_deploy_type or type == constants.k8s_kind_deploy_type: - return K8sDeployer(type, deployment_dir, compose_files, compose_project_name, compose_env_file) + return K8sDeployer(type, deployment_context, compose_files, compose_project_name, compose_env_file) else: print(f"ERROR: deploy-to {type} is not valid") diff --git a/stack_orchestrator/deploy/deployment.py b/stack_orchestrator/deploy/deployment.py index 586a2b2a..8d74a62d 100644 --- a/stack_orchestrator/deploy/deployment.py +++ b/stack_orchestrator/deploy/deployment.py @@ -17,8 +17,10 @@ import click from pathlib import Path import sys from stack_orchestrator import constants +from stack_orchestrator.deploy.images import push_images_operation from stack_orchestrator.deploy.deploy import up_operation, down_operation, ps_operation, port_operation from stack_orchestrator.deploy.deploy import exec_operation, logs_operation, create_deploy_context +from stack_orchestrator.deploy.deploy_types import DeployCommandContext from stack_orchestrator.deploy.deployment_context import DeploymentContext @@ -46,13 +48,13 @@ def command(ctx, dir): ctx.obj = deployment_context -def make_deploy_context(ctx): +def make_deploy_context(ctx) -> DeployCommandContext: context: DeploymentContext = ctx.obj stack_file_path = context.get_stack_file() env_file = context.get_env_file() cluster_name = context.get_cluster_name() - if "deploy-to" in context.spec.obj: - deployment_type = context.spec.obj["deploy-to"] + if constants.deploy_to_key in context.spec.obj: + deployment_type = context.spec.obj[constants.deploy_to_key] else: deployment_type = constants.compose_deploy_type return create_deploy_context(ctx.parent.parent.obj, context, stack_file_path, None, None, cluster_name, env_file, @@ -109,6 +111,14 @@ def ps(ctx): ps_operation(ctx) +@command.command() +@click.pass_context +def push_images(ctx): + deploy_command_context: DeployCommandContext = make_deploy_context(ctx) + deployment_context: DeploymentContext = ctx.obj + push_images_operation(deploy_command_context, deployment_context) + + @command.command() @click.argument('extra_args', nargs=-1) # help: command: port @click.pass_context diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py index c9ba3a10..e999c1df 100644 --- a/stack_orchestrator/deploy/deployment_create.py +++ b/stack_orchestrator/deploy/deployment_create.py @@ -250,20 +250,28 @@ def _parse_config_variables(variable_values: str): @click.command() @click.option("--config", help="Provide config variables for the deployment") @click.option("--kube-config", help="Provide a config file for a k8s deployment") +@click.option("--image-registry", help="Provide a container image registry url for this k8s cluster") @click.option("--output", required=True, help="Write yaml spec file here") @click.option("--map-ports-to-host", required=False, help="Map ports to the host as one of: any-variable-random (default), " "localhost-same, any-same, localhost-fixed-random, any-fixed-random") @click.pass_context -def init(ctx, config, kube_config, output, map_ports_to_host): +def init(ctx, config, kube_config, image_registry, output, map_ports_to_host): yaml = get_yaml() stack = global_options(ctx).stack debug = global_options(ctx).debug deployer_type = ctx.obj.deployer.type default_spec_file_content = call_stack_deploy_init(ctx.obj) - spec_file_content = {"stack": stack, "deploy-to": deployer_type} + spec_file_content = {"stack": stack, constants.deploy_to_key: deployer_type} if deployer_type == "k8s": spec_file_content.update({constants.kube_config_key: kube_config}) + spec_file_content.update({constants.image_resigtry_key: image_registry}) + else: + # Check for --kube-config supplied for non-relevant deployer types + if kube_config is not None: + error_exit(f"--kube-config is not allowed with a {deployer_type} deployment") + if image_registry is not None: + error_exit(f"--image-registry is not allowed with a {deployer_type} deployment") if default_spec_file_content: spec_file_content.update(default_spec_file_content) config_variables = _parse_config_variables(config) @@ -323,7 +331,7 @@ def _copy_files_to_directory(file_paths: List[Path], directory: Path): def create(ctx, spec_file, deployment_dir, network_dir, initial_peers): parsed_spec = get_parsed_deployment_spec(spec_file) stack_name = parsed_spec["stack"] - deployment_type = parsed_spec["deploy-to"] + deployment_type = parsed_spec[constants.deploy_to_key] stack_file = get_stack_file_path(stack_name) parsed_stack = get_parsed_stack_config(stack_name) if global_options(ctx).debug: diff --git a/stack_orchestrator/deploy/images.py b/stack_orchestrator/deploy/images.py new file mode 100644 index 00000000..ddbb33f7 --- /dev/null +++ b/stack_orchestrator/deploy/images.py @@ -0,0 +1,62 @@ +# Copyright © 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from typing import Set + +from python_on_whales import DockerClient + +from stack_orchestrator import constants +from stack_orchestrator.opts import opts +from stack_orchestrator.deploy.deployment_context import DeploymentContext +from stack_orchestrator.deploy.deploy_types import DeployCommandContext +from stack_orchestrator.deploy.deploy_util import images_for_deployment + + +def _image_needs_pushed(image: str): + # TODO: this needs to be more intelligent + return image.endswith(":local") + + +def remote_tag_for_image(image: str, remote_repo_url: str): + # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy + (org, image_name_with_version) = image.split("/") + (image_name, image_version) = image_name_with_version.split(":") + if image_version == "local": + return f"{remote_repo_url}/{image_name}:deploy" + else: + return image + + +# TODO: needs lots of error handling +def push_images_operation(command_context: DeployCommandContext, deployment_context: DeploymentContext): + # Get the list of images for the stack + cluster_context = command_context.cluster_context + images: Set[str] = images_for_deployment(cluster_context.compose_files) + # Tag the images for the remote repo + remote_repo_url = deployment_context.spec.obj[constants.image_resigtry_key] + docker = DockerClient() + for image in images: + if _image_needs_pushed(image): + remote_tag = remote_tag_for_image(image, remote_repo_url) + if opts.o.verbose: + print(f"Tagging {image} to {remote_tag}") + docker.image.tag(image, remote_tag) + # Run docker push commands to upload + for image in images: + if _image_needs_pushed(image): + remote_tag = remote_tag_for_image(image, remote_repo_url) + if opts.o.verbose: + print(f"Pushing image {remote_tag}") + docker.image.push(remote_tag) diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 9275db2b..ff052bf9 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -18,34 +18,30 @@ from typing import Any, List, Set from stack_orchestrator.opts import opts from stack_orchestrator.deploy.k8s.helpers import named_volumes_from_pod_files, volume_mounts_for_service, volumes_for_pod_files -from stack_orchestrator.deploy.k8s.helpers import parsed_pod_files_map_from_file_names, get_node_pv_mount_path +from stack_orchestrator.deploy.k8s.helpers import get_node_pv_mount_path from stack_orchestrator.deploy.k8s.helpers import env_var_map_from_file, envs_from_environment_variables_map +from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment from stack_orchestrator.deploy.deploy_types import DeployEnvVars +from stack_orchestrator.deploy.images import remote_tag_for_image class ClusterInfo: - parsed_pod_yaml_map: Any = {} + parsed_pod_yaml_map: Any image_set: Set[str] = set() app_name: str = "test-app" deployment_name: str = "test-deployment" environment_variables: DeployEnvVars + remote_image_repo: str def __init__(self) -> None: pass - def int(self, pod_files: List[str], compose_env_file): + def int(self, pod_files: List[str], compose_env_file, remote_image_repo): self.parsed_pod_yaml_map = parsed_pod_files_map_from_file_names(pod_files) # Find the set of images in the pods - for pod_name in self.parsed_pod_yaml_map: - pod = self.parsed_pod_yaml_map[pod_name] - services = pod["services"] - for service_name in services: - service_info = services[service_name] - image = service_info["image"] - self.image_set.add(image) - if opts.o.debug: - print(f"image_set: {self.image_set}") + self.image_set = images_for_deployment(pod_files) self.environment_variables = DeployEnvVars(env_var_map_from_file(compose_env_file)) + self.remote_image_repo = remote_image_repo if (opts.o.debug): print(f"Env vars: {self.environment_variables.map}") @@ -99,10 +95,12 @@ class ClusterInfo: container_name = service_name service_info = services[service_name] image = service_info["image"] + # Re-write the image tag for remote deployment + image_to_use = remote_tag_for_image(image, self.remote_image_repo) if self.remote_image_repo is not None else image volume_mounts = volume_mounts_for_service(self.parsed_pod_yaml_map, service_name) container = client.V1Container( name=container_name, - image=image, + image=image_to_use, env=envs_from_environment_variables_map(self.environment_variables.map), ports=[client.V1ContainerPort(container_port=80)], volume_mounts=volume_mounts, diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index 44eb9f0e..3d0ef3ff 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -22,6 +22,7 @@ from stack_orchestrator.deploy.k8s.helpers import create_cluster, destroy_cluste from stack_orchestrator.deploy.k8s.helpers import pods_in_deployment, log_stream_from_string, generate_kind_config from stack_orchestrator.deploy.k8s.cluster_info import ClusterInfo from stack_orchestrator.opts import opts +from stack_orchestrator.deploy.deployment_context import DeploymentContext class K8sDeployer(Deployer): @@ -33,19 +34,21 @@ class K8sDeployer(Deployer): kind_cluster_name: str cluster_info : ClusterInfo deployment_dir: Path + deployment_context: DeploymentContext - def __init__(self, type, deployment_dir, compose_files, compose_project_name, compose_env_file) -> None: + def __init__(self, type, deployment_context: DeploymentContext, compose_files, compose_project_name, compose_env_file) -> None: if (opts.o.debug): - print(f"Deployment dir: {deployment_dir}") + print(f"Deployment dir: {deployment_context.deployment_dir}") print(f"Compose files: {compose_files}") print(f"Project name: {compose_project_name}") print(f"Env file: {compose_env_file}") print(f"Type: {type}") self.type = type - self.deployment_dir = deployment_dir + self.deployment_dir = deployment_context.deployment_dir + self.deployment_context = deployment_context self.kind_cluster_name = compose_project_name self.cluster_info = ClusterInfo() - self.cluster_info.int(compose_files, compose_env_file) + self.cluster_info.int(compose_files, compose_env_file, deployment_context.spec.obj[constants.image_resigtry_key]) def connect_api(self): if self.is_kind(): @@ -97,7 +100,35 @@ class K8sDeployer(Deployer): {deployment_resp.metadata.generation} {deployment_resp.spec.template.spec.containers[0].image}") def down(self, timeout, volumes): + self.connect_api() # Delete the k8s objects + # Create the host-path-mounted PVs for this deployment + pvs = self.cluster_info.get_pvs() + for pv in pvs: + if opts.o.debug: + print(f"Deleting this pv: {pv}") + pv_resp = self.core_api.delete_persistent_volume(name=pv.metadata.name) + if opts.o.debug: + print("PV deleted:") + print(f"{pv_resp}") + + # Figure out the PVCs for this deployment + pvcs = self.cluster_info.get_pvcs() + for pvc in pvcs: + if opts.o.debug: + print(f"Deleting this pvc: {pvc}") + pvc_resp = self.core_api.delete_namespaced_persistent_volume_claim(name=pvc.metadata.name, namespace=self.k8s_namespace) + if opts.o.debug: + print("PVCs deleted:") + print(f"{pvc_resp}") + # Process compose files into a Deployment + deployment = self.cluster_info.get_deployment() + # Create the k8s objects + if opts.o.debug: + print(f"Deleting this deployment: {deployment}") + self.apps_api.delete_namespaced_deployment( + name=deployment.metadata.name, namespace=self.k8s_namespace + ) if self.is_kind(): # Destroy the kind cluster destroy_cluster(self.kind_cluster_name) diff --git a/stack_orchestrator/deploy/k8s/helpers.py b/stack_orchestrator/deploy/k8s/helpers.py index db1ef075..82a33792 100644 --- a/stack_orchestrator/deploy/k8s/helpers.py +++ b/stack_orchestrator/deploy/k8s/helpers.py @@ -18,10 +18,10 @@ from dotenv import dotenv_values import os from pathlib import Path import subprocess -from typing import Any, Set, Mapping, List +from typing import Set, Mapping, List from stack_orchestrator.opts import opts -from stack_orchestrator.util import get_yaml +from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names def _run_command(command: str): @@ -133,17 +133,6 @@ def _make_absolute_host_path(data_mount_path: Path, deployment_dir: Path) -> Pat return Path.cwd().joinpath(deployment_dir.joinpath("compose").joinpath(data_mount_path)).resolve() -def parsed_pod_files_map_from_file_names(pod_files): - parsed_pod_yaml_map : Any = {} - for pod_file in pod_files: - with open(pod_file, "r") as pod_file_descriptor: - parsed_pod_file = get_yaml().load(pod_file_descriptor) - parsed_pod_yaml_map[pod_file] = parsed_pod_file - if opts.o.debug: - print(f"parsed_pod_yaml_map: {parsed_pod_yaml_map}") - return parsed_pod_yaml_map - - def _generate_kind_mounts(parsed_pod_files, deployment_dir): volume_definitions = [] volume_host_path_map = _get_host_paths_for_volumes(parsed_pod_files) diff --git a/stack_orchestrator/deploy/run_webapp.py b/stack_orchestrator/deploy/run_webapp.py index 8b1073b1..aa22acdf 100644 --- a/stack_orchestrator/deploy/run_webapp.py +++ b/stack_orchestrator/deploy/run_webapp.py @@ -1,59 +1,59 @@ -# Copyright © 2022, 2023 Vulcanize - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. - -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -# Builds webapp containers - -# env vars: -# CERC_REPO_BASE_DIR defaults to ~/cerc - -# TODO: display the available list of containers; allow re-build of either all or specific containers - -import hashlib -import click - -from dotenv import dotenv_values -from stack_orchestrator.deploy.deployer_factory import getDeployer - - -@click.command() -@click.option("--image", help="image to deploy", required=True) -@click.option("--deploy-to", default="compose", help="deployment type ([Docker] 'compose' or 'k8s')") -@click.option("--env-file", help="environment file for webapp") -@click.pass_context -def command(ctx, image, deploy_to, env_file): - '''build the specified webapp container''' - - env = {} - if env_file: - env = dotenv_values(env_file) - - unique_cluster_descriptor = f"{image},{env}" - hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest() - cluster = f"laconic-webapp-{hash}" - - deployer = getDeployer(deploy_to, - deployment_dir=None, - compose_files=None, - compose_project_name=cluster, - compose_env_file=None) - - container = deployer.run(image, command=[], user=None, volumes=[], entrypoint=None, env=env, detach=True) - - # Make configurable? - webappPort = "3000/tcp" - # TODO: This assumes a Docker container object... - if webappPort in container.network_settings.ports: - mapping = container.network_settings.ports[webappPort][0] - print(f"""Image: {image}\nID: {container.id}\nURL: http://localhost:{mapping['HostPort']}""") +# Copyright © 2022, 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# Builds webapp containers + +# env vars: +# CERC_REPO_BASE_DIR defaults to ~/cerc + +# TODO: display the available list of containers; allow re-build of either all or specific containers + +import hashlib +import click + +from dotenv import dotenv_values +from stack_orchestrator.deploy.deployer_factory import getDeployer + + +@click.command() +@click.option("--image", help="image to deploy", required=True) +@click.option("--deploy-to", default="compose", help="deployment type ([Docker] 'compose' or 'k8s')") +@click.option("--env-file", help="environment file for webapp") +@click.pass_context +def command(ctx, image, deploy_to, env_file): + '''build the specified webapp container''' + + env = {} + if env_file: + env = dotenv_values(env_file) + + unique_cluster_descriptor = f"{image},{env}" + hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest() + cluster = f"laconic-webapp-{hash}" + + deployer = getDeployer(deploy_to, + deployment_context=None, + compose_files=None, + compose_project_name=cluster, + compose_env_file=None) + + container = deployer.run(image, command=[], user=None, volumes=[], entrypoint=None, env=env, detach=True) + + # Make configurable? + webappPort = "3000/tcp" + # TODO: This assumes a Docker container object... + if webappPort in container.network_settings.ports: + mapping = container.network_settings.ports[webappPort][0] + print(f"""Image: {image}\nID: {container.id}\nURL: http://localhost:{mapping['HostPort']}""") From 0b87c12c134dfc18b5e76447842b43d956a46ace Mon Sep 17 00:00:00 2001 From: Nabarun Gogoi Date: Tue, 21 Nov 2023 19:07:09 +0530 Subject: [PATCH 22/75] Upgrade merkl and sushiswap watcher to `v0.1.4` (#657) * Upgrade merkl and sushi watcher versions * Set gqlPath to base URL and remove filling start block * Upgrade watcher versions to 0.1.4 --- .../data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml | 1 - .../data/compose/docker-compose-watcher-sushiswap-v3.yml | 1 - .../data/config/watcher-merkl-sushiswap-v3/start-server.sh | 3 --- .../watcher-merkl-sushiswap-v3/watcher-config-template.toml | 1 + .../data/config/watcher-sushiswap-v3/start-server.sh | 3 --- .../config/watcher-sushiswap-v3/watcher-config-template.toml | 1 + stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml | 2 +- stack_orchestrator/data/stacks/sushiswap-v3/stack.yml | 2 +- 8 files changed, 4 insertions(+), 10 deletions(-) diff --git a/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml b/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml index d08c6214..0a83af89 100644 --- a/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml +++ b/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml @@ -56,7 +56,6 @@ services: environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT} - SUSHISWAP_START_BLOCK: ${SUSHISWAP_START_BLOCK:- 2867560} command: ["bash", "./start-server.sh"] volumes: - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml diff --git a/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml b/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml index 219688db..f7b75ca5 100644 --- a/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml +++ b/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml @@ -56,7 +56,6 @@ services: environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT} - SUSHISWAP_START_BLOCK: ${SUSHISWAP_START_BLOCK:- 2867560} command: ["bash", "./start-server.sh"] volumes: - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml diff --git a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-server.sh b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-server.sh index 1b14f2e3..e2bbdaad 100755 --- a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-server.sh +++ b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-server.sh @@ -16,8 +16,5 @@ WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ # Write the modified content to a new file echo "$WATCHER_CONFIG" > environments/local.toml -echo "Initializing watcher..." -yarn fill --start-block $SUSHISWAP_START_BLOCK --end-block $((SUSHISWAP_START_BLOCK + 1)) - echo "Running server..." DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js diff --git a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml index 894a4660..f5355a4b 100644 --- a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml +++ b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml @@ -2,6 +2,7 @@ host = "0.0.0.0" port = 3008 kind = "active" + gqlPath = '/' # Checkpointing state. checkpointing = true diff --git a/stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh b/stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh index 1b14f2e3..e2bbdaad 100755 --- a/stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh +++ b/stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh @@ -16,8 +16,5 @@ WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ # Write the modified content to a new file echo "$WATCHER_CONFIG" > environments/local.toml -echo "Initializing watcher..." -yarn fill --start-block $SUSHISWAP_START_BLOCK --end-block $((SUSHISWAP_START_BLOCK + 1)) - echo "Running server..." DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js diff --git a/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml index 07880a8d..7cfabedd 100644 --- a/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml +++ b/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml @@ -2,6 +2,7 @@ host = "0.0.0.0" port = 3008 kind = "active" + gqlPath = "/" # Checkpointing state. checkpointing = true diff --git a/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml index 8f5cb7ee..3f9dd43e 100644 --- a/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml +++ b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml @@ -2,7 +2,7 @@ version: "1.0" name: merkl-sushiswap-v3 description: "SushiSwap v3 watcher stack" repos: - - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.2 + - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.4 containers: - cerc/watcher-merkl-sushiswap-v3 pods: diff --git a/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml b/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml index 05350996..49c604bf 100644 --- a/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml +++ b/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml @@ -2,7 +2,7 @@ version: "1.0" name: sushiswap-v3 description: "SushiSwap v3 watcher stack" repos: - - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.2 + - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.4 containers: - cerc/watcher-sushiswap-v3 pods: From 01029cf7aa0cedac6b184964c19950263172a041 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 21 Nov 2023 08:35:31 -0700 Subject: [PATCH 23/75] Fix for code path that doesn't create a DeploymentContext (#658) --- stack_orchestrator/deploy/k8s/deploy_k8s.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index 3d0ef3ff..483f64c6 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -37,18 +37,21 @@ class K8sDeployer(Deployer): deployment_context: DeploymentContext def __init__(self, type, deployment_context: DeploymentContext, compose_files, compose_project_name, compose_env_file) -> None: + self.type = type + # TODO: workaround pending refactoring above to cope with being created with a null deployment_context + if deployment_context is None: + return + self.deployment_dir = deployment_context.deployment_dir + self.deployment_context = deployment_context + self.kind_cluster_name = compose_project_name + self.cluster_info = ClusterInfo() + self.cluster_info.int(compose_files, compose_env_file, deployment_context.spec.obj[constants.image_resigtry_key]) if (opts.o.debug): print(f"Deployment dir: {deployment_context.deployment_dir}") print(f"Compose files: {compose_files}") print(f"Project name: {compose_project_name}") print(f"Env file: {compose_env_file}") print(f"Type: {type}") - self.type = type - self.deployment_dir = deployment_context.deployment_dir - self.deployment_context = deployment_context - self.kind_cluster_name = compose_project_name - self.cluster_info = ClusterInfo() - self.cluster_info.int(compose_files, compose_env_file, deployment_context.spec.obj[constants.image_resigtry_key]) def connect_api(self): if self.is_kind(): From 87bedde5cbfe00a039499026b2c4a53d8daff471 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 21 Nov 2023 16:04:36 -0700 Subject: [PATCH 24/75] Support for k8s ingress and tls (#659) --- stack_orchestrator/constants.py | 2 + .../deploy/deployment_create.py | 6 +- stack_orchestrator/deploy/k8s/cluster_info.py | 83 +++++++++++++++-- stack_orchestrator/deploy/k8s/deploy_k8s.py | 92 ++++++++++++++++--- stack_orchestrator/deploy/spec.py | 12 +++ 5 files changed, 172 insertions(+), 23 deletions(-) diff --git a/stack_orchestrator/constants.py b/stack_orchestrator/constants.py index aedc4f3c..1cff6055 100644 --- a/stack_orchestrator/constants.py +++ b/stack_orchestrator/constants.py @@ -19,6 +19,8 @@ k8s_kind_deploy_type = "k8s-kind" k8s_deploy_type = "k8s" kube_config_key = "kube-config" deploy_to_key = "deploy-to" +network_key = "network" +http_proxy_key = "http-proxy" image_resigtry_key = "image-registry" kind_config_filename = "kind-config.yml" kube_config_filename = "kubeconfig.yml" diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py index e999c1df..64647ab2 100644 --- a/stack_orchestrator/deploy/deployment_create.py +++ b/stack_orchestrator/deploy/deployment_create.py @@ -103,8 +103,8 @@ def _fixup_pod_file(pod, spec, compose_dir): } pod["volumes"][volume] = new_volume_spec # Fix up ports - if "ports" in spec: - spec_ports = spec["ports"] + if "network" in spec and "ports" in spec["network"]: + spec_ports = spec["network"]["ports"] for container_name, container_ports in spec_ports.items(): if container_name in pod["services"]: pod["services"][container_name]["ports"] = container_ports @@ -285,7 +285,7 @@ def init(ctx, config, kube_config, image_registry, output, map_ports_to_host): print(f"Creating spec file for stack: {stack} with content: {spec_file_content}") ports = _get_mapped_ports(stack, map_ports_to_host) - spec_file_content["ports"] = ports + spec_file_content.update({"network": {"ports": ports}}) named_volumes = _get_named_volumes(stack) if named_volumes: diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index ff052bf9..a7426804 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -22,6 +22,7 @@ from stack_orchestrator.deploy.k8s.helpers import get_node_pv_mount_path from stack_orchestrator.deploy.k8s.helpers import env_var_map_from_file, envs_from_environment_variables_map from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment from stack_orchestrator.deploy.deploy_types import DeployEnvVars +from stack_orchestrator.deploy.spec import Spec from stack_orchestrator.deploy.images import remote_tag_for_image @@ -29,22 +30,91 @@ class ClusterInfo: parsed_pod_yaml_map: Any image_set: Set[str] = set() app_name: str = "test-app" - deployment_name: str = "test-deployment" environment_variables: DeployEnvVars - remote_image_repo: str + spec: Spec def __init__(self) -> None: pass - def int(self, pod_files: List[str], compose_env_file, remote_image_repo): + def int(self, pod_files: List[str], compose_env_file, spec: Spec): self.parsed_pod_yaml_map = parsed_pod_files_map_from_file_names(pod_files) # Find the set of images in the pods self.image_set = images_for_deployment(pod_files) self.environment_variables = DeployEnvVars(env_var_map_from_file(compose_env_file)) - self.remote_image_repo = remote_image_repo + self.spec = spec if (opts.o.debug): print(f"Env vars: {self.environment_variables.map}") + def get_ingress(self): + # No ingress for a deployment that has no http-proxy defined, for now + http_proxy_info_list = self.spec.get_http_proxy() + ingress = None + if http_proxy_info_list: + # TODO: handle multiple definitions + http_proxy_info = http_proxy_info_list[0] + if opts.o.debug: + print(f"http-proxy: {http_proxy_info}") + # TODO: good enough parsing for webapp deployment for now + host_name = http_proxy_info["host-name"] + rules = [] + tls = [client.V1IngressTLS( + hosts=[host_name], + secret_name=f"{self.app_name}-tls" + )] + paths = [] + for route in http_proxy_info["routes"]: + path = route["path"] + proxy_to = route["proxy-to"] + if opts.o.debug: + print(f"proxy config: {path} -> {proxy_to}") + paths.append(client.V1HTTPIngressPath( + path_type="Prefix", + path=path, + backend=client.V1IngressBackend( + service=client.V1IngressServiceBackend( + # TODO: this looks wrong + name=f"{self.app_name}-service", + # TODO: pull port number from the service + port=client.V1ServiceBackendPort(number=80) + ) + ) + )) + rules.append(client.V1IngressRule( + host=host_name, + http=client.V1HTTPIngressRuleValue( + paths=paths + ) + )) + spec = client.V1IngressSpec( + tls=tls, + rules=rules + ) + ingress = client.V1Ingress( + metadata=client.V1ObjectMeta( + name=f"{self.app_name}-ingress", + annotations={ + "kubernetes.io/ingress.class": "nginx", + "cert-manager.io/cluster-issuer": "letsencrypt-prod" + } + ), + spec=spec + ) + return ingress + + def get_service(self): + service = client.V1Service( + metadata=client.V1ObjectMeta(name=f"{self.app_name}-service"), + spec=client.V1ServiceSpec( + type="ClusterIP", + ports=[client.V1ServicePort( + port=80, + target_port=80 + )], + selector={"app": self.app_name} + ) + ) + return service + def get_pvcs(self): result = [] volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map) @@ -96,7 +166,8 @@ class ClusterInfo: service_info = services[service_name] image = service_info["image"] # Re-write the image tag for remote deployment - image_to_use = remote_tag_for_image(image, self.remote_image_repo) if self.remote_image_repo is not None else image + image_to_use = remote_tag_for_image( + image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image volume_mounts = volume_mounts_for_service(self.parsed_pod_yaml_map, service_name) container = client.V1Container( name=container_name, @@ -123,7 +194,7 @@ class ClusterInfo: deployment = client.V1Deployment( api_version="apps/v1", kind="Deployment", - metadata=client.V1ObjectMeta(name=self.deployment_name), + metadata=client.V1ObjectMeta(name=f"{self.app_name}-deployment"), spec=spec, ) return deployment diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index 483f64c6..8e790d10 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -23,6 +23,15 @@ from stack_orchestrator.deploy.k8s.helpers import pods_in_deployment, log_stream from stack_orchestrator.deploy.k8s.cluster_info import ClusterInfo from stack_orchestrator.opts import opts from stack_orchestrator.deploy.deployment_context import DeploymentContext +from stack_orchestrator.util import error_exit + + +def _check_delete_exception(e: client.exceptions.ApiException): + if e.status == 404: + if opts.o.debug: + print("Failed to delete object, continuing") + else: + error_exit(f"k8s api error: {e}") class K8sDeployer(Deployer): @@ -30,6 +39,7 @@ class K8sDeployer(Deployer): type: str core_api: client.CoreV1Api apps_api: client.AppsV1Api + networking_api: client.NetworkingV1Api k8s_namespace: str = "default" kind_cluster_name: str cluster_info : ClusterInfo @@ -45,7 +55,7 @@ class K8sDeployer(Deployer): self.deployment_context = deployment_context self.kind_cluster_name = compose_project_name self.cluster_info = ClusterInfo() - self.cluster_info.int(compose_files, compose_env_file, deployment_context.spec.obj[constants.image_resigtry_key]) + self.cluster_info.int(compose_files, compose_env_file, deployment_context.spec) if (opts.o.debug): print(f"Deployment dir: {deployment_context.deployment_dir}") print(f"Compose files: {compose_files}") @@ -60,9 +70,11 @@ class K8sDeployer(Deployer): # Get the config file and pass to load_kube_config() config.load_kube_config(config_file=self.deployment_dir.joinpath(constants.kube_config_filename).as_posix()) self.core_api = client.CoreV1Api() + self.networking_api = client.NetworkingV1Api() self.apps_api = client.AppsV1Api() def up(self, detach, services): + if self.is_kind(): # Create the kind cluster create_cluster(self.kind_cluster_name, self.deployment_dir.joinpath(constants.kind_config_filename)) @@ -102,6 +114,26 @@ class K8sDeployer(Deployer): print(f"{deployment_resp.metadata.namespace} {deployment_resp.metadata.name} \ {deployment_resp.metadata.generation} {deployment_resp.spec.template.spec.containers[0].image}") + service: client.V1Service = self.cluster_info.get_service() + service_resp = self.core_api.create_namespaced_service( + namespace=self.k8s_namespace, + body=service + ) + if opts.o.debug: + print("Service created:") + print(f"{service_resp}") + + # TODO: disable ingress for kind + ingress: client.V1Ingress = self.cluster_info.get_ingress() + + ingress_resp = self.networking_api.create_namespaced_ingress( + namespace=self.k8s_namespace, + body=ingress + ) + if opts.o.debug: + print("Ingress created:") + print(f"{ingress_resp}") + def down(self, timeout, volumes): self.connect_api() # Delete the k8s objects @@ -110,28 +142,60 @@ class K8sDeployer(Deployer): for pv in pvs: if opts.o.debug: print(f"Deleting this pv: {pv}") - pv_resp = self.core_api.delete_persistent_volume(name=pv.metadata.name) - if opts.o.debug: - print("PV deleted:") - print(f"{pv_resp}") + try: + pv_resp = self.core_api.delete_persistent_volume(name=pv.metadata.name) + if opts.o.debug: + print("PV deleted:") + print(f"{pv_resp}") + except client.exceptions.ApiException as e: + _check_delete_exception(e) # Figure out the PVCs for this deployment pvcs = self.cluster_info.get_pvcs() for pvc in pvcs: if opts.o.debug: print(f"Deleting this pvc: {pvc}") - pvc_resp = self.core_api.delete_namespaced_persistent_volume_claim(name=pvc.metadata.name, namespace=self.k8s_namespace) - if opts.o.debug: - print("PVCs deleted:") - print(f"{pvc_resp}") - # Process compose files into a Deployment + try: + pvc_resp = self.core_api.delete_namespaced_persistent_volume_claim( + name=pvc.metadata.name, namespace=self.k8s_namespace + ) + if opts.o.debug: + print("PVCs deleted:") + print(f"{pvc_resp}") + except client.exceptions.ApiException as e: + _check_delete_exception(e) deployment = self.cluster_info.get_deployment() - # Create the k8s objects if opts.o.debug: print(f"Deleting this deployment: {deployment}") - self.apps_api.delete_namespaced_deployment( - name=deployment.metadata.name, namespace=self.k8s_namespace - ) + try: + self.apps_api.delete_namespaced_deployment( + name=deployment.metadata.name, namespace=self.k8s_namespace + ) + except client.exceptions.ApiException as e: + _check_delete_exception(e) + + service: client.V1Service = self.cluster_info.get_service() + if opts.o.debug: + print(f"Deleting service: {service}") + try: + self.core_api.delete_namespaced_service( + namespace=self.k8s_namespace, + name=service.metadata.name + ) + except client.exceptions.ApiException as e: + _check_delete_exception(e) + + # TODO: disable ingress for kind + ingress: client.V1Ingress = self.cluster_info.get_ingress() + if opts.o.debug: + print(f"Deleting this ingress: {ingress}") + try: + self.networking_api.delete_namespaced_ingress( + name=ingress.metadata.name, namespace=self.k8s_namespace + ) + except client.exceptions.ApiException as e: + _check_delete_exception(e) + if self.is_kind(): # Destroy the kind cluster destroy_cluster(self.kind_cluster_name) diff --git a/stack_orchestrator/deploy/spec.py b/stack_orchestrator/deploy/spec.py index 9ee893b9..c4f791bf 100644 --- a/stack_orchestrator/deploy/spec.py +++ b/stack_orchestrator/deploy/spec.py @@ -16,6 +16,7 @@ from pathlib import Path import typing from stack_orchestrator.util import get_yaml +from stack_orchestrator import constants class Spec: @@ -28,3 +29,14 @@ class Spec: def init_from_file(self, file_path: Path): with file_path: self.obj = get_yaml().load(open(file_path, "r")) + + def get_image_registry(self): + return (self.obj[constants.image_resigtry_key] + if self.obj and constants.image_resigtry_key in self.obj + else None) + + def get_http_proxy(self): + return (self.obj[constants.network_key][constants.http_proxy_key] + if self.obj and constants.network_key in self.obj + and constants.http_proxy_key in self.obj[constants.network_key] + else None) From 1a37255c187040672c83b7f145d21b7517d2706c Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 22 Nov 2023 11:31:30 -0600 Subject: [PATCH 25/75] Tweak laconicd config to allow setting endpoint port and to make the fixturenet restartable. (#660) * Endpoint includes port * Make it restartable * Don't try to remove the mounted directory * Make copy of init.sh --- ...ker-compose-fixturenet-laconic-console.yml | 2 +- .../docker-compose-fixturenet-laconicd.yml | 2 +- .../fixturenet-laconicd/create-fixturenet.sh | 187 +++++++++--------- .../cerc-laconic-console-host/config.yml | 4 +- 4 files changed, 101 insertions(+), 94 deletions(-) diff --git a/stack_orchestrator/data/compose/docker-compose-fixturenet-laconic-console.yml b/stack_orchestrator/data/compose/docker-compose-fixturenet-laconic-console.yml index da2fd95f..a186e761 100644 --- a/stack_orchestrator/data/compose/docker-compose-fixturenet-laconic-console.yml +++ b/stack_orchestrator/data/compose/docker-compose-fixturenet-laconic-console.yml @@ -4,6 +4,6 @@ services: image: cerc/laconic-console-host:local environment: - CERC_WEBAPP_FILES_DIR=${CERC_WEBAPP_FILES_DIR:-/usr/local/share/.config/yarn/global/node_modules/@cerc-io/console-app/dist/production} - - LACONIC_HOSTED_ENDPOINT=${LACONIC_HOSTED_ENDPOINT:-http://localhost} + - LACONIC_HOSTED_ENDPOINT=${LACONIC_HOSTED_ENDPOINT:-http://localhost:9473} ports: - "80" diff --git a/stack_orchestrator/data/compose/docker-compose-fixturenet-laconicd.yml b/stack_orchestrator/data/compose/docker-compose-fixturenet-laconicd.yml index 641229d4..7b48f60d 100644 --- a/stack_orchestrator/data/compose/docker-compose-fixturenet-laconicd.yml +++ b/stack_orchestrator/data/compose/docker-compose-fixturenet-laconicd.yml @@ -5,7 +5,7 @@ services: command: ["sh", "/docker-entrypoint-scripts.d/create-fixturenet.sh"] volumes: # The cosmos-sdk node's database directory: - - laconicd-data:/root/.laconicd/data + - laconicd-data:/root/.laconicd # TODO: look at folding these scripts into the container - ../config/fixturenet-laconicd/create-fixturenet.sh:/docker-entrypoint-scripts.d/create-fixturenet.sh - ../config/fixturenet-laconicd/export-mykey.sh:/docker-entrypoint-scripts.d/export-mykey.sh diff --git a/stack_orchestrator/data/config/fixturenet-laconicd/create-fixturenet.sh b/stack_orchestrator/data/config/fixturenet-laconicd/create-fixturenet.sh index 9c30bff8..d444fcad 100644 --- a/stack_orchestrator/data/config/fixturenet-laconicd/create-fixturenet.sh +++ b/stack_orchestrator/data/config/fixturenet-laconicd/create-fixturenet.sh @@ -14,104 +14,111 @@ LOGLEVEL="info" TRACE="--trace" # TRACE="" -# validate dependencies are installed -command -v jq > /dev/null 2>&1 || { echo >&2 "jq not installed. More info: https://stedolan.github.io/jq/download/"; exit 1; } +if [ "$1" == "clean" ] || [ ! -d "$HOME/.laconicd/data/blockstore.db" ]; then + # validate dependencies are installed + command -v jq > /dev/null 2>&1 || { echo >&2 "jq not installed. More info: https://stedolan.github.io/jq/download/"; exit 1; } -# remove existing daemon and client -rm -rf ~/.laconic* + # remove existing daemon and client + rm -rf $HOME/.laconicd/* + rm -rf $HOME/.laconic/* -make install - -laconicd config keyring-backend $KEYRING -laconicd config chain-id $CHAINID - -# if $KEY exists it should be deleted -laconicd keys add $KEY --keyring-backend $KEYRING --algo $KEYALGO - -# Set moniker and chain-id for Ethermint (Moniker can be anything, chain-id must be an integer) -laconicd init $MONIKER --chain-id $CHAINID - -# Change parameter token denominations to aphoton -cat $HOME/.laconicd/config/genesis.json | jq '.app_state["staking"]["params"]["bond_denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json -cat $HOME/.laconicd/config/genesis.json | jq '.app_state["crisis"]["constant_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json -cat $HOME/.laconicd/config/genesis.json | jq '.app_state["gov"]["deposit_params"]["min_deposit"][0]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json -cat $HOME/.laconicd/config/genesis.json | jq '.app_state["mint"]["params"]["mint_denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json -# Custom modules -cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["record_rent"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json -cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json -cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_commit_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json -cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_reveal_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json -cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_minimum_bid"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json - -if [[ "$TEST_REGISTRY_EXPIRY" == "true" ]]; then - echo "Setting timers for expiry tests." - - cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["record_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json - cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_grace_period"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json - cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json -fi - -if [[ "$TEST_AUCTION_ENABLED" == "true" ]]; then - echo "Enabling auction and setting timers." - - cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_enabled"]=true' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json - cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json - cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_grace_period"]="300s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json - cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_commits_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json - cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_reveals_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json -fi - -# increase block time (?) -cat $HOME/.laconicd/config/genesis.json | jq '.consensus_params["block"]["time_iota_ms"]="1000"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json - -# Set gas limit in genesis -cat $HOME/.laconicd/config/genesis.json | jq '.consensus_params["block"]["max_gas"]="10000000"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json - -# disable produce empty block -if [[ "$OSTYPE" == "darwin"* ]]; then - sed -i '' 's/create_empty_blocks = true/create_empty_blocks = false/g' $HOME/.laconicd/config/config.toml - else - sed -i 's/create_empty_blocks = true/create_empty_blocks = false/g' $HOME/.laconicd/config/config.toml -fi - -if [[ $1 == "pending" ]]; then - if [[ "$OSTYPE" == "darwin"* ]]; then - sed -i '' 's/create_empty_blocks_interval = "0s"/create_empty_blocks_interval = "30s"/g' $HOME/.laconicd/config/config.toml - sed -i '' 's/timeout_propose = "3s"/timeout_propose = "30s"/g' $HOME/.laconicd/config/config.toml - sed -i '' 's/timeout_propose_delta = "500ms"/timeout_propose_delta = "5s"/g' $HOME/.laconicd/config/config.toml - sed -i '' 's/timeout_prevote = "1s"/timeout_prevote = "10s"/g' $HOME/.laconicd/config/config.toml - sed -i '' 's/timeout_prevote_delta = "500ms"/timeout_prevote_delta = "5s"/g' $HOME/.laconicd/config/config.toml - sed -i '' 's/timeout_precommit = "1s"/timeout_precommit = "10s"/g' $HOME/.laconicd/config/config.toml - sed -i '' 's/timeout_precommit_delta = "500ms"/timeout_precommit_delta = "5s"/g' $HOME/.laconicd/config/config.toml - sed -i '' 's/timeout_commit = "5s"/timeout_commit = "150s"/g' $HOME/.laconicd/config/config.toml - sed -i '' 's/timeout_broadcast_tx_commit = "10s"/timeout_broadcast_tx_commit = "150s"/g' $HOME/.laconicd/config/config.toml - else - sed -i 's/create_empty_blocks_interval = "0s"/create_empty_blocks_interval = "30s"/g' $HOME/.laconicd/config/config.toml - sed -i 's/timeout_propose = "3s"/timeout_propose = "30s"/g' $HOME/.laconicd/config/config.toml - sed -i 's/timeout_propose_delta = "500ms"/timeout_propose_delta = "5s"/g' $HOME/.laconicd/config/config.toml - sed -i 's/timeout_prevote = "1s"/timeout_prevote = "10s"/g' $HOME/.laconicd/config/config.toml - sed -i 's/timeout_prevote_delta = "500ms"/timeout_prevote_delta = "5s"/g' $HOME/.laconicd/config/config.toml - sed -i 's/timeout_precommit = "1s"/timeout_precommit = "10s"/g' $HOME/.laconicd/config/config.toml - sed -i 's/timeout_precommit_delta = "500ms"/timeout_precommit_delta = "5s"/g' $HOME/.laconicd/config/config.toml - sed -i 's/timeout_commit = "5s"/timeout_commit = "150s"/g' $HOME/.laconicd/config/config.toml - sed -i 's/timeout_broadcast_tx_commit = "10s"/timeout_broadcast_tx_commit = "150s"/g' $HOME/.laconicd/config/config.toml + if [ -n "`which make`" ]; then + make install fi -fi -# Allocate genesis accounts (cosmos formatted addresses) -laconicd add-genesis-account $KEY 100000000000000000000000000aphoton --keyring-backend $KEYRING + laconicd config keyring-backend $KEYRING + laconicd config chain-id $CHAINID -# Sign genesis transaction -laconicd gentx $KEY 1000000000000000000000aphoton --keyring-backend $KEYRING --chain-id $CHAINID + # if $KEY exists it should be deleted + laconicd keys add $KEY --keyring-backend $KEYRING --algo $KEYALGO -# Collect genesis tx -laconicd collect-gentxs + # Set moniker and chain-id for Ethermint (Moniker can be anything, chain-id must be an integer) + laconicd init $MONIKER --chain-id $CHAINID -# Run this to ensure everything worked and that the genesis file is setup correctly -laconicd validate-genesis + # Change parameter token denominations to aphoton + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["staking"]["params"]["bond_denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["crisis"]["constant_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["gov"]["deposit_params"]["min_deposit"][0]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["mint"]["params"]["mint_denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + # Custom modules + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["record_rent"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_commit_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_reveal_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_minimum_bid"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json -if [[ $1 == "pending" ]]; then - echo "pending mode is on, please wait for the first block committed." + if [[ "$TEST_REGISTRY_EXPIRY" == "true" ]]; then + echo "Setting timers for expiry tests." + + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["record_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_grace_period"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + fi + + if [[ "$TEST_AUCTION_ENABLED" == "true" ]]; then + echo "Enabling auction and setting timers." + + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_enabled"]=true' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_grace_period"]="300s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_commits_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_reveals_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + fi + + # increase block time (?) + cat $HOME/.laconicd/config/genesis.json | jq '.consensus_params["block"]["time_iota_ms"]="1000"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + + # Set gas limit in genesis + cat $HOME/.laconicd/config/genesis.json | jq '.consensus_params["block"]["max_gas"]="10000000"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + + # disable produce empty block + if [[ "$OSTYPE" == "darwin"* ]]; then + sed -i '' 's/create_empty_blocks = true/create_empty_blocks = false/g' $HOME/.laconicd/config/config.toml + else + sed -i 's/create_empty_blocks = true/create_empty_blocks = false/g' $HOME/.laconicd/config/config.toml + fi + + if [[ $1 == "pending" ]]; then + if [[ "$OSTYPE" == "darwin"* ]]; then + sed -i '' 's/create_empty_blocks_interval = "0s"/create_empty_blocks_interval = "30s"/g' $HOME/.laconicd/config/config.toml + sed -i '' 's/timeout_propose = "3s"/timeout_propose = "30s"/g' $HOME/.laconicd/config/config.toml + sed -i '' 's/timeout_propose_delta = "500ms"/timeout_propose_delta = "5s"/g' $HOME/.laconicd/config/config.toml + sed -i '' 's/timeout_prevote = "1s"/timeout_prevote = "10s"/g' $HOME/.laconicd/config/config.toml + sed -i '' 's/timeout_prevote_delta = "500ms"/timeout_prevote_delta = "5s"/g' $HOME/.laconicd/config/config.toml + sed -i '' 's/timeout_precommit = "1s"/timeout_precommit = "10s"/g' $HOME/.laconicd/config/config.toml + sed -i '' 's/timeout_precommit_delta = "500ms"/timeout_precommit_delta = "5s"/g' $HOME/.laconicd/config/config.toml + sed -i '' 's/timeout_commit = "5s"/timeout_commit = "150s"/g' $HOME/.laconicd/config/config.toml + sed -i '' 's/timeout_broadcast_tx_commit = "10s"/timeout_broadcast_tx_commit = "150s"/g' $HOME/.laconicd/config/config.toml + else + sed -i 's/create_empty_blocks_interval = "0s"/create_empty_blocks_interval = "30s"/g' $HOME/.laconicd/config/config.toml + sed -i 's/timeout_propose = "3s"/timeout_propose = "30s"/g' $HOME/.laconicd/config/config.toml + sed -i 's/timeout_propose_delta = "500ms"/timeout_propose_delta = "5s"/g' $HOME/.laconicd/config/config.toml + sed -i 's/timeout_prevote = "1s"/timeout_prevote = "10s"/g' $HOME/.laconicd/config/config.toml + sed -i 's/timeout_prevote_delta = "500ms"/timeout_prevote_delta = "5s"/g' $HOME/.laconicd/config/config.toml + sed -i 's/timeout_precommit = "1s"/timeout_precommit = "10s"/g' $HOME/.laconicd/config/config.toml + sed -i 's/timeout_precommit_delta = "500ms"/timeout_precommit_delta = "5s"/g' $HOME/.laconicd/config/config.toml + sed -i 's/timeout_commit = "5s"/timeout_commit = "150s"/g' $HOME/.laconicd/config/config.toml + sed -i 's/timeout_broadcast_tx_commit = "10s"/timeout_broadcast_tx_commit = "150s"/g' $HOME/.laconicd/config/config.toml + fi + fi + + # Allocate genesis accounts (cosmos formatted addresses) + laconicd add-genesis-account $KEY 100000000000000000000000000aphoton --keyring-backend $KEYRING + + # Sign genesis transaction + laconicd gentx $KEY 1000000000000000000000aphoton --keyring-backend $KEYRING --chain-id $CHAINID + + # Collect genesis tx + laconicd collect-gentxs + + # Run this to ensure everything worked and that the genesis file is setup correctly + laconicd validate-genesis + + if [[ $1 == "pending" ]]; then + echo "pending mode is on, please wait for the first block committed." + fi +else + echo "Using existing database at $HOME/.laconicd. To replace, run '`basename $0` clean'" fi # Start the node (remove the --pruning=nothing flag if historical queries are not needed) diff --git a/stack_orchestrator/data/container-build/cerc-laconic-console-host/config.yml b/stack_orchestrator/data/container-build/cerc-laconic-console-host/config.yml index d557ace5..6c310842 100644 --- a/stack_orchestrator/data/container-build/cerc-laconic-console-host/config.yml +++ b/stack_orchestrator/data/container-build/cerc-laconic-console-host/config.yml @@ -2,5 +2,5 @@ services: wns: - server: 'LACONIC_HOSTED_ENDPOINT:9473/api' - webui: 'LACONIC_HOSTED_ENDPOINT:9473/console' + server: 'LACONIC_HOSTED_ENDPOINT/api' + webui: 'LACONIC_HOSTED_ENDPOINT/console' From 3fefc67e7787fd8b4138221448d277e16f91cd66 Mon Sep 17 00:00:00 2001 From: Nabarun Gogoi Date: Thu, 23 Nov 2023 15:00:18 +0530 Subject: [PATCH 26/75] Run azimuth contract watcher in active mode (#661) * Update stack to run azimuth job runner * Run azimuth watcher in active mode * Update stack to run job-runners for all watchers * Update ports in job-runner health checks * Map metrics ports to host * Configure historical block processing batch size for Azimuth watcher * Use deployment command for azimuth stack --------- Co-authored-by: neeraj Co-authored-by: Prathamesh Musale --- .../docker-compose-watcher-azimuth.yml | 281 +++++++++++++++++- .../watcher-azimuth/start-job-runner.sh | 28 ++ .../config/watcher-azimuth/start-server.sh | 7 +- .../watcher-config-template.toml | 6 + .../config/watcher-azimuth/watcher-params.env | 5 - .../data/stacks/azimuth/README.md | 93 ++++-- .../data/stacks/azimuth/stack.yml | 2 +- 7 files changed, 367 insertions(+), 55 deletions(-) create mode 100755 stack_orchestrator/data/config/watcher-azimuth/start-job-runner.sh delete mode 100644 stack_orchestrator/data/config/watcher-azimuth/watcher-params.env diff --git a/stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml b/stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml index 327c77fc..b620202a 100644 --- a/stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml +++ b/stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml @@ -22,6 +22,38 @@ services: retries: 15 start_period: 10s + # Starts the azimuth-watcher job runner + azimuth-watcher-job-runner: + image: cerc/watcher-azimuth:local + restart: unless-stopped + depends_on: + watcher-db: + condition: service_healthy + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CERC_HISTORICAL_BLOCK_RANGE: 500 + CONTRACT_ADDRESS: 0x223c067F8CF28ae173EE5CafEa60cA44C335fecB + CONTRACT_NAME: Azimuth + STARTING_BLOCK: 6784880 + working_dir: /app/packages/azimuth-watcher + command: "./start-job-runner.sh" + volumes: + - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/azimuth-watcher/environments/watcher-config-template.toml + - ../config/watcher-azimuth/merge-toml.js:/app/packages/azimuth-watcher/merge-toml.js + - ../config/watcher-azimuth/start-job-runner.sh:/app/packages/azimuth-watcher/start-job-runner.sh + ports: + - "9000" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "9000"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + # Starts the azimuth-watcher server azimuth-watcher-server: image: cerc/watcher-azimuth:local @@ -29,8 +61,8 @@ services: depends_on: watcher-db: condition: service_healthy - env_file: - - ../config/watcher-azimuth/watcher-params.env + azimuth-watcher-job-runner: + condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} @@ -52,6 +84,37 @@ services: extra_hosts: - "host.docker.internal:host-gateway" + # Starts the censures-watcher job runner + censures-watcher-job-runner: + image: cerc/watcher-azimuth:local + restart: unless-stopped + depends_on: + watcher-db: + condition: service_healthy + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CONTRACT_ADDRESS: 0x325f68d32BdEe6Ed86E7235ff2480e2A433D6189 + CONTRACT_NAME: Censures + STARTING_BLOCK: 6784954 + working_dir: /app/packages/censures-watcher + command: "./start-job-runner.sh" + volumes: + - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/censures-watcher/environments/watcher-config-template.toml + - ../config/watcher-azimuth/merge-toml.js:/app/packages/censures-watcher/merge-toml.js + - ../config/watcher-azimuth/start-job-runner.sh:/app/packages/censures-watcher/start-job-runner.sh + ports: + - "9002" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "9002"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + # Starts the censures-watcher server censures-watcher-server: image: cerc/watcher-azimuth:local @@ -59,8 +122,8 @@ services: depends_on: watcher-db: condition: service_healthy - env_file: - - ../config/watcher-azimuth/watcher-params.env + censures-watcher-job-runner: + condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} @@ -82,6 +145,37 @@ services: extra_hosts: - "host.docker.internal:host-gateway" + # Starts the claims-watcher job runner + claims-watcher-job-runner: + image: cerc/watcher-azimuth:local + restart: unless-stopped + depends_on: + watcher-db: + condition: service_healthy + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CONTRACT_ADDRESS: 0xe7e7f69b34D7d9Bd8d61Fb22C33b22708947971A + CONTRACT_NAME: Claims + STARTING_BLOCK: 6784941 + working_dir: /app/packages/claims-watcher + command: "./start-job-runner.sh" + volumes: + - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/claims-watcher/environments/watcher-config-template.toml + - ../config/watcher-azimuth/merge-toml.js:/app/packages/claims-watcher/merge-toml.js + - ../config/watcher-azimuth/start-job-runner.sh:/app/packages/claims-watcher/start-job-runner.sh + ports: + - "9004" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "9004"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + # Starts the claims-watcher server claims-watcher-server: image: cerc/watcher-azimuth:local @@ -89,8 +183,8 @@ services: depends_on: watcher-db: condition: service_healthy - env_file: - - ../config/watcher-azimuth/watcher-params.env + claims-watcher-job-runner: + condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} @@ -112,6 +206,37 @@ services: extra_hosts: - "host.docker.internal:host-gateway" + # Starts the conditional-star-release-watcher job runner + conditional-star-release-watcher-job-runner: + image: cerc/watcher-azimuth:local + restart: unless-stopped + depends_on: + watcher-db: + condition: service_healthy + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CONTRACT_ADDRESS: 0x8C241098C3D3498Fe1261421633FD57986D74AeA + CONTRACT_NAME: ConditionalStarRelease + STARTING_BLOCK: 6828004 + working_dir: /app/packages/conditional-star-release-watcher + command: "./start-job-runner.sh" + volumes: + - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/conditional-star-release-watcher/environments/watcher-config-template.toml + - ../config/watcher-azimuth/merge-toml.js:/app/packages/conditional-star-release-watcher/merge-toml.js + - ../config/watcher-azimuth/start-job-runner.sh:/app/packages/conditional-star-release-watcher/start-job-runner.sh + ports: + - "9006" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "9006"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + # Starts the conditional-star-release-watcher server conditional-star-release-watcher-server: image: cerc/watcher-azimuth:local @@ -119,8 +244,8 @@ services: depends_on: watcher-db: condition: service_healthy - env_file: - - ../config/watcher-azimuth/watcher-params.env + conditional-star-release-watcher-job-runner: + condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} @@ -142,6 +267,37 @@ services: extra_hosts: - "host.docker.internal:host-gateway" + # Starts the delegated-sending-watcher job runner + delegated-sending-watcher-job-runner: + image: cerc/watcher-azimuth:local + restart: unless-stopped + depends_on: + watcher-db: + condition: service_healthy + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CONTRACT_ADDRESS: 0xf6b461fE1aD4bd2ce25B23Fe0aff2ac19B3dFA76 + CONTRACT_NAME: DelegatedSending + STARTING_BLOCK: 6784956 + working_dir: /app/packages/delegated-sending-watcher + command: "./start-job-runner.sh" + volumes: + - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/delegated-sending-watcher/environments/watcher-config-template.toml + - ../config/watcher-azimuth/merge-toml.js:/app/packages/delegated-sending-watcher/merge-toml.js + - ../config/watcher-azimuth/start-job-runner.sh:/app/packages/delegated-sending-watcher/start-job-runner.sh + ports: + - "9008" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "9008"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + # Starts the delegated-sending-watcher server delegated-sending-watcher-server: image: cerc/watcher-azimuth:local @@ -149,8 +305,8 @@ services: depends_on: watcher-db: condition: service_healthy - env_file: - - ../config/watcher-azimuth/watcher-params.env + delegated-sending-watcher-job-runner: + condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} @@ -172,6 +328,37 @@ services: extra_hosts: - "host.docker.internal:host-gateway" + # Starts the ecliptic-watcher job runner + ecliptic-watcher-job-runner: + image: cerc/watcher-azimuth:local + restart: unless-stopped + depends_on: + watcher-db: + condition: service_healthy + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CONTRACT_ADDRESS: 0x33EeCbf908478C10614626A9D304bfe18B78DD73 + CONTRACT_NAME: Ecliptic + STARTING_BLOCK: 13692129 + working_dir: /app/packages/ecliptic-watcher + command: "./start-job-runner.sh" + volumes: + - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/ecliptic-watcher/environments/watcher-config-template.toml + - ../config/watcher-azimuth/merge-toml.js:/app/packages/ecliptic-watcher/merge-toml.js + - ../config/watcher-azimuth/start-job-runner.sh:/app/packages/ecliptic-watcher/start-job-runner.sh + ports: + - "9010" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "9010"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + # Starts the ecliptic-watcher server ecliptic-watcher-server: image: cerc/watcher-azimuth:local @@ -179,8 +366,8 @@ services: depends_on: watcher-db: condition: service_healthy - env_file: - - ../config/watcher-azimuth/watcher-params.env + ecliptic-watcher-job-runner: + condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} @@ -202,6 +389,37 @@ services: extra_hosts: - "host.docker.internal:host-gateway" + # Starts the linear-star-release-watcher job runner + linear-star-release-watcher-job-runner: + image: cerc/watcher-azimuth:local + restart: unless-stopped + depends_on: + watcher-db: + condition: service_healthy + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CONTRACT_ADDRESS: 0x86cd9cd0992F04231751E3761De45cEceA5d1801 + CONTRACT_NAME: LinearStarRelease + STARTING_BLOCK: 6784943 + working_dir: /app/packages/linear-star-release-watcher + command: "./start-job-runner.sh" + volumes: + - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/linear-star-release-watcher/environments/watcher-config-template.toml + - ../config/watcher-azimuth/merge-toml.js:/app/packages/linear-star-release-watcher/merge-toml.js + - ../config/watcher-azimuth/start-job-runner.sh:/app/packages/linear-star-release-watcher/start-job-runner.sh + ports: + - "9012" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "9012"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + # Starts the linear-star-release-watcher server linear-star-release-watcher-server: image: cerc/watcher-azimuth:local @@ -209,8 +427,8 @@ services: depends_on: watcher-db: condition: service_healthy - env_file: - - ../config/watcher-azimuth/watcher-params.env + linear-star-release-watcher-job-runner: + condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} @@ -232,6 +450,37 @@ services: extra_hosts: - "host.docker.internal:host-gateway" + # Starts the polls-watcher job runner + polls-watcher-job-runner: + image: cerc/watcher-azimuth:local + restart: unless-stopped + depends_on: + watcher-db: + condition: service_healthy + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CONTRACT_ADDRESS: 0x7fEcaB617c868Bb5996d99D95200D2Fa708218e4 + CONTRACT_NAME: Polls + STARTING_BLOCK: 6784912 + working_dir: /app/packages/polls-watcher + command: "./start-job-runner.sh" + volumes: + - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/polls-watcher/environments/watcher-config-template.toml + - ../config/watcher-azimuth/merge-toml.js:/app/packages/polls-watcher/merge-toml.js + - ../config/watcher-azimuth/start-job-runner.sh:/app/packages/polls-watcher/start-job-runner.sh + ports: + - "9014" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "9014"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + # Starts the polls-watcher server polls-watcher-server: image: cerc/watcher-azimuth:local @@ -239,8 +488,8 @@ services: depends_on: watcher-db: condition: service_healthy - env_file: - - ../config/watcher-azimuth/watcher-params.env + polls-watcher-job-runner: + condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} diff --git a/stack_orchestrator/data/config/watcher-azimuth/start-job-runner.sh b/stack_orchestrator/data/config/watcher-azimuth/start-job-runner.sh new file mode 100755 index 00000000..4bcad74c --- /dev/null +++ b/stack_orchestrator/data/config/watcher-azimuth/start-job-runner.sh @@ -0,0 +1,28 @@ +#!/bin/sh +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}" +echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}" +echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}" + +# Replace env variables in template TOML file +# Read in the config template TOML file and modify it +WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) +WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ + sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \ + s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \ + s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ") + +# Write the modified content to a new file +echo "$WATCHER_CONFIG" > environments/watcher-config.toml + +# Merge SO watcher config with existing config file +node merge-toml.js + +yarn watch:contract --address $CONTRACT_ADDRESS --kind $CONTRACT_NAME --checkpoint true --starting-block $STARTING_BLOCK + +echo 'yarn job-runner' +yarn job-runner diff --git a/stack_orchestrator/data/config/watcher-azimuth/start-server.sh b/stack_orchestrator/data/config/watcher-azimuth/start-server.sh index c84c58d0..fa334653 100755 --- a/stack_orchestrator/data/config/watcher-azimuth/start-server.sh +++ b/stack_orchestrator/data/config/watcher-azimuth/start-server.sh @@ -4,18 +4,17 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi -CERC_IPLD_ETH_RPC="${CERC_IPLD_ETH_RPC:-${DEFAULT_CERC_IPLD_ETH_RPC}}" -CERC_IPLD_ETH_GQL="${CERC_IPLD_ETH_GQL:-${DEFAULT_CERC_IPLD_ETH_GQL}}" - echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}" echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}" +echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}" # Replace env variables in template TOML file # Read in the config template TOML file and modify it WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \ - s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}| ") + s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \ + s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ") # Write the modified content to a new file echo "$WATCHER_CONFIG" > environments/watcher-config.toml diff --git a/stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml index 1a4616fc..2eb814a3 100644 --- a/stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml +++ b/stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml @@ -2,6 +2,9 @@ host = "0.0.0.0" maxSimultaneousRequests = -1 +[metrics] + host = "0.0.0.0" + [database] host = "watcher-db" port = 5432 @@ -12,3 +15,6 @@ [upstream.ethServer] gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL" rpcProviderEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_RPC" + +[jobQueue] + historicalLogsBlockRange = REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE diff --git a/stack_orchestrator/data/config/watcher-azimuth/watcher-params.env b/stack_orchestrator/data/config/watcher-azimuth/watcher-params.env deleted file mode 100644 index 8fcdc2d6..00000000 --- a/stack_orchestrator/data/config/watcher-azimuth/watcher-params.env +++ /dev/null @@ -1,5 +0,0 @@ -# Defaults - -# ipld-eth-server endpoints -DEFAULT_CERC_IPLD_ETH_RPC= -DEFAULT_CERC_IPLD_ETH_GQL= diff --git a/stack_orchestrator/data/stacks/azimuth/README.md b/stack_orchestrator/data/stacks/azimuth/README.md index 67f42b75..f7d93f33 100644 --- a/stack_orchestrator/data/stacks/azimuth/README.md +++ b/stack_orchestrator/data/stacks/azimuth/README.md @@ -9,19 +9,11 @@ Prerequisite: `ipld-eth-server` RPC and GQL endpoints Clone required repositories: ```bash -laconic-so --stack azimuth setup-repositories +laconic-so --stack azimuth setup-repositories --pull ``` NOTE: If the repository already exists and checked out to a different version, `setup-repositories` command will throw an error. -For getting around this, the `azimuth-watcher-ts` repository can be removed and then run the command. - -Checkout to the required versions and branches in repos - -```bash -# azimuth-watcher-ts -cd ~/cerc/azimuth-watcher-ts -git checkout v0.1.0 -``` +For getting around this, the `azimuth-watcher-ts` repository can be removed and then run the command again. Build the container images: @@ -31,42 +23,85 @@ laconic-so --stack azimuth build-containers This should create the required docker images in the local image registry. -### Configuration +## Create a deployment -* Create and update an env file to be used in the next step: +First, create a spec file for the deployment, which will map the stack's ports and volumes to the host: +```bash +laconic-so --stack azimuth deploy init --output azimuth-spec.yml +``` + +### Ports + +Edit `network` in spec file to map container ports to same ports in host + +```yaml +... +network: + ports: + watcher-db: + - 0.0.0.0:15432:5432 + azimuth-watcher-server: + - 0.0.0.0:3001:3001 + censures-watcher-server: + - 0.0.0.0:3002:3002 + claims-watcher-server: + - 0.0.0.0:3003:3003 + conditional-star-release-watcher-server: + - 0.0.0.0:3004:3004 + delegated-sending-watcher-server: + - 0.0.0.0:3005:3005 + ecliptic-watcher-server: + - 0.0.0.0:3006:3006 + linear-star-release-watcher-server: + - 0.0.0.0:3007:3007 + polls-watcher-server: + - 0.0.0.0:3008:3008 + gateway-server: + - 0.0.0.0:4000:4000 +... +``` + +### Data volumes +Container data volumes are bind-mounted to specified paths in the host filesystem. +The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory. The default mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`. + +--- + +Once you've made any needed changes to the spec file, create a deployment from it: +```bash +laconic-so --stack azimuth deploy create --spec-file azimuth-spec.yml --deployment-dir azimuth-deployment +``` + +## Set env variables + +Inside the deployment directory, open the file `config.env` and add variable to update RPC endpoint : ```bash - # External ipld-eth-server endpoints + # External RPC endpoints CERC_IPLD_ETH_RPC= - CERC_IPLD_ETH_GQL= ``` -* NOTE: If `ipld-eth-server` is running on the host machine, use `host.docker.internal` as the hostname to access host ports +* NOTE: If RPC endpoint is on the host machine, use `host.docker.internal` as the hostname to access the host port, or use the `ip a` command to find the IP address of the `docker0` interface (this will usually be something like `172.17.0.1` or `172.18.0.1`) -### Deploy the stack +## Start the stack -* Deploy the containers: - - ```bash - laconic-so --stack azimuth deploy-system --env-file up - ``` +Start the deployment: +```bash +laconic-so deployment --dir azimuth-deployment start +``` * List and check the health status of all the containers using `docker ps` and wait for them to be `healthy` ## Clean up -Stop all the services running in background: +To stop all azimuth services running in the background, while preserving chain data: ```bash -laconic-so --stack azimuth deploy-system down +laconic-so deployment --dir azimuth-deployment stop ``` -Clear volumes created by this stack: +To stop all azimuth services and also delete data: ```bash -# List all relevant volumes -docker volume ls -q --filter "name=.*watcher_db_data" - -# Remove all the listed volumes -docker volume rm $(docker volume ls -q --filter "name=.*watcher_db_data") +laconic-so deployment --dir azimuth-deployment stop --delete-volumes ``` diff --git a/stack_orchestrator/data/stacks/azimuth/stack.yml b/stack_orchestrator/data/stacks/azimuth/stack.yml index 47e0d058..bb7f2a88 100644 --- a/stack_orchestrator/data/stacks/azimuth/stack.yml +++ b/stack_orchestrator/data/stacks/azimuth/stack.yml @@ -1,7 +1,7 @@ version: "1.0" name: azimuth repos: - - github.com/cerc-io/azimuth-watcher-ts@v0.1.1 + - github.com/cerc-io/azimuth-watcher-ts containers: - cerc/watcher-azimuth pods: From 9499941891c66acbc5fd2cf358b32cf8a97058b8 Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Thu, 23 Nov 2023 19:11:02 +0530 Subject: [PATCH 27/75] Increase max connections for Azimuth watcher dbs (#665) --- .../data/compose/docker-compose-watcher-azimuth.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml b/stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml index b620202a..48e77082 100644 --- a/stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml +++ b/stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml @@ -10,6 +10,7 @@ services: - POSTGRES_MULTIPLE_DATABASES=azimuth-watcher,azimuth-watcher-job-queue,censures-watcher,censures-watcher-job-queue,claims-watcher,claims-watcher-job-queue,conditional-star-release-watcher,conditional-star-release-watcher-job-queue,delegated-sending-watcher,delegated-sending-watcher-job-queue,ecliptic-watcher,ecliptic-watcher-job-queue,linear-star-release-watcher,linear-star-release-watcher-job-queue,polls-watcher,polls-watcher-job-queue - POSTGRES_EXTENSION=azimuth-watcher-job-queue:pgcrypto,censures-watcher-job-queue:pgcrypto,claims-watcher-job-queue:pgcrypto,conditional-star-release-watcher-job-queue:pgcrypto,delegated-sending-watcher-job-queue:pgcrypto,ecliptic-watcher-job-queue:pgcrypto,linear-star-release-watcher-job-queue:pgcrypto,polls-watcher-job-queue:pgcrypto, - POSTGRES_PASSWORD=password + command: ["postgres", "-c", "max_connections=200"] volumes: - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh - watcher_db_data:/var/lib/postgresql/data From 1b94db27c18a4a10a944de18d0733209f57e6225 Mon Sep 17 00:00:00 2001 From: Nabarun Gogoi Date: Fri, 24 Nov 2023 14:05:37 +0530 Subject: [PATCH 28/75] Upgrade azimuth watcher release version to `0.1.2` (#666) * Upgrade azimuth watcher release version * Fix version for azimuth watcher repo --- .../data/config/watcher-azimuth/watcher-config-template.toml | 1 + stack_orchestrator/data/stacks/azimuth/stack.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml index 2eb814a3..2a91fedf 100644 --- a/stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml +++ b/stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml @@ -18,3 +18,4 @@ [jobQueue] historicalLogsBlockRange = REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE + blockDelayInMilliSecs = 12000 diff --git a/stack_orchestrator/data/stacks/azimuth/stack.yml b/stack_orchestrator/data/stacks/azimuth/stack.yml index bb7f2a88..7adbe663 100644 --- a/stack_orchestrator/data/stacks/azimuth/stack.yml +++ b/stack_orchestrator/data/stacks/azimuth/stack.yml @@ -1,7 +1,7 @@ version: "1.0" name: azimuth repos: - - github.com/cerc-io/azimuth-watcher-ts + - github.com/cerc-io/azimuth-watcher-ts@v0.1.2 containers: - cerc/watcher-azimuth pods: From a68cd5d65ce6404fd1d1528d094b097b140ef400 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Mon, 27 Nov 2023 22:02:16 -0700 Subject: [PATCH 29/75] Webapp deploy (#662) --- .../docker-compose-webapp-template.yml | 8 ++ .../data/stacks/webapp-template/README.md | 1 + .../data/stacks/webapp-template/stack.yml | 7 ++ stack_orchestrator/deploy/deploy.py | 2 +- .../deploy/deployment_create.py | 37 ++++-- stack_orchestrator/deploy/k8s/cluster_info.py | 28 ++++- stack_orchestrator/deploy/k8s/deploy_k8s.py | 4 +- stack_orchestrator/deploy/webapp/__init__.py | 0 .../deploy/webapp/deploy_webapp.py | 113 ++++++++++++++++++ .../deploy/{ => webapp}/run_webapp.py | 8 +- stack_orchestrator/main.py | 3 +- 11 files changed, 191 insertions(+), 20 deletions(-) create mode 100644 stack_orchestrator/data/compose/docker-compose-webapp-template.yml create mode 100644 stack_orchestrator/data/stacks/webapp-template/README.md create mode 100644 stack_orchestrator/data/stacks/webapp-template/stack.yml create mode 100644 stack_orchestrator/deploy/webapp/__init__.py create mode 100644 stack_orchestrator/deploy/webapp/deploy_webapp.py rename stack_orchestrator/deploy/{ => webapp}/run_webapp.py (91%) diff --git a/stack_orchestrator/data/compose/docker-compose-webapp-template.yml b/stack_orchestrator/data/compose/docker-compose-webapp-template.yml new file mode 100644 index 00000000..b8697afa --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-webapp-template.yml @@ -0,0 +1,8 @@ +services: + webapp: + image: cerc/webapp-container:local + restart: always + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + ports: + - "3000" diff --git a/stack_orchestrator/data/stacks/webapp-template/README.md b/stack_orchestrator/data/stacks/webapp-template/README.md new file mode 100644 index 00000000..4441e475 --- /dev/null +++ b/stack_orchestrator/data/stacks/webapp-template/README.md @@ -0,0 +1 @@ +# Template stack for webapp deployments diff --git a/stack_orchestrator/data/stacks/webapp-template/stack.yml b/stack_orchestrator/data/stacks/webapp-template/stack.yml new file mode 100644 index 00000000..d574e764 --- /dev/null +++ b/stack_orchestrator/data/stacks/webapp-template/stack.yml @@ -0,0 +1,7 @@ +version: "1.0" +name: test +description: "Webapp deployment stack" +containers: + - cerc/webapp-template-container +pods: + - webapp-template diff --git a/stack_orchestrator/deploy/deploy.py b/stack_orchestrator/deploy/deploy.py index 32c13a61..df231e74 100644 --- a/stack_orchestrator/deploy/deploy.py +++ b/stack_orchestrator/deploy/deploy.py @@ -276,7 +276,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file): unique_cluster_descriptor = f"{path},{stack},{include},{exclude}" if ctx.debug: print(f"pre-hash descriptor: {unique_cluster_descriptor}") - hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest() + hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest()[:16] cluster = f"laconic-{hash}" if ctx.verbose: print(f"Using cluster name: {cluster}") diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py index 64647ab2..fd52dba8 100644 --- a/stack_orchestrator/deploy/deployment_create.py +++ b/stack_orchestrator/deploy/deployment_create.py @@ -22,6 +22,7 @@ import random from shutil import copy, copyfile, copytree import sys from stack_orchestrator import constants +from stack_orchestrator.opts import opts from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config, global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts, get_pod_script_paths, get_plugin_code_paths, error_exit) @@ -257,13 +258,29 @@ def _parse_config_variables(variable_values: str): "localhost-same, any-same, localhost-fixed-random, any-fixed-random") @click.pass_context def init(ctx, config, kube_config, image_registry, output, map_ports_to_host): - yaml = get_yaml() stack = global_options(ctx).stack - debug = global_options(ctx).debug deployer_type = ctx.obj.deployer.type - default_spec_file_content = call_stack_deploy_init(ctx.obj) + deploy_command_context = ctx.obj + return init_operation( + deploy_command_context, + stack, deployer_type, + config, kube_config, + image_registry, + output, + map_ports_to_host) + + +# The init command's implementation is in a separate function so that we can +# call it from other commands, bypassing the click decoration stuff +def init_operation(deploy_command_context, stack, deployer_type, config, kube_config, image_registry, output, map_ports_to_host): + yaml = get_yaml() + default_spec_file_content = call_stack_deploy_init(deploy_command_context) spec_file_content = {"stack": stack, constants.deploy_to_key: deployer_type} if deployer_type == "k8s": + if kube_config is None: + error_exit("--kube-config must be supplied with --deploy-to k8s") + if image_registry is None: + error_exit("--image-registry must be supplied with --deploy-to k8s") spec_file_content.update({constants.kube_config_key: kube_config}) spec_file_content.update({constants.image_resigtry_key: image_registry}) else: @@ -281,7 +298,7 @@ def init(ctx, config, kube_config, image_registry, output, map_ports_to_host): new_config = config_variables["config"] merged_config = {**new_config, **orig_config} spec_file_content.update({"config": merged_config}) - if debug: + if opts.o.debug: print(f"Creating spec file for stack: {stack} with content: {spec_file_content}") ports = _get_mapped_ports(stack, map_ports_to_host) @@ -329,12 +346,19 @@ def _copy_files_to_directory(file_paths: List[Path], directory: Path): @click.option("--initial-peers", help="Initial set of persistent peers") @click.pass_context def create(ctx, spec_file, deployment_dir, network_dir, initial_peers): + deployment_command_context = ctx.obj + return create_operation(deployment_command_context, spec_file, deployment_dir, network_dir, initial_peers) + + +# The init command's implementation is in a separate function so that we can +# call it from other commands, bypassing the click decoration stuff +def create_operation(deployment_command_context, spec_file, deployment_dir, network_dir, initial_peers): parsed_spec = get_parsed_deployment_spec(spec_file) stack_name = parsed_spec["stack"] deployment_type = parsed_spec[constants.deploy_to_key] stack_file = get_stack_file_path(stack_name) parsed_stack = get_parsed_stack_config(stack_name) - if global_options(ctx).debug: + if opts.o.debug: print(f"parsed spec: {parsed_spec}") if deployment_dir is None: deployment_dir_path = _make_default_deployment_dir() @@ -366,7 +390,7 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers): extra_config_dirs = _find_extra_config_dirs(parsed_pod_file, pod) destination_pod_dir = destination_pods_dir.joinpath(pod) os.mkdir(destination_pod_dir) - if global_options(ctx).debug: + if opts.o.debug: print(f"extra config dirs: {extra_config_dirs}") _fixup_pod_file(parsed_pod_file, parsed_spec, destination_compose_dir) with open(destination_compose_dir.joinpath("docker-compose-%s.yml" % pod), "w") as output_file: @@ -390,7 +414,6 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers): # Delegate to the stack's Python code # The deploy create command doesn't require a --stack argument so we need to insert the # stack member here. - deployment_command_context = ctx.obj deployment_command_context.stack = stack_name deployment_context = DeploymentContext() deployment_context.init(deployment_dir_path) diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index a7426804..6c19b20a 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -29,18 +29,19 @@ from stack_orchestrator.deploy.images import remote_tag_for_image class ClusterInfo: parsed_pod_yaml_map: Any image_set: Set[str] = set() - app_name: str = "test-app" + app_name: str environment_variables: DeployEnvVars spec: Spec def __init__(self) -> None: pass - def int(self, pod_files: List[str], compose_env_file, spec: Spec): + def int(self, pod_files: List[str], compose_env_file, deployment_name, spec: Spec): self.parsed_pod_yaml_map = parsed_pod_files_map_from_file_names(pod_files) # Find the set of images in the pods self.image_set = images_for_deployment(pod_files) self.environment_variables = DeployEnvVars(env_var_map_from_file(compose_env_file)) + self.app_name = deployment_name self.spec = spec if (opts.o.debug): print(f"Env vars: {self.environment_variables.map}") @@ -67,6 +68,8 @@ class ClusterInfo: proxy_to = route["proxy-to"] if opts.o.debug: print(f"proxy config: {path} -> {proxy_to}") + # proxy_to has the form : + proxy_to_port = int(proxy_to.split(":")[1]) paths.append(client.V1HTTPIngressPath( path_type="Prefix", path=path, @@ -75,7 +78,7 @@ class ClusterInfo: # TODO: this looks wrong name=f"{self.app_name}-service", # TODO: pull port number from the service - port=client.V1ServiceBackendPort(number=80) + port=client.V1ServiceBackendPort(number=proxy_to_port) ) ) )) @@ -101,14 +104,23 @@ class ClusterInfo: ) return ingress + # TODO: suppoprt multiple services def get_service(self): + for pod_name in self.parsed_pod_yaml_map: + pod = self.parsed_pod_yaml_map[pod_name] + services = pod["services"] + for service_name in services: + service_info = services[service_name] + port = int(service_info["ports"][0]) + if opts.o.debug: + print(f"service port: {port}") service = client.V1Service( metadata=client.V1ObjectMeta(name=f"{self.app_name}-service"), spec=client.V1ServiceSpec( type="ClusterIP", ports=[client.V1ServicePort( - port=80, - target_port=80 + port=port, + target_port=port )], selector={"app": self.app_name} ) @@ -165,6 +177,10 @@ class ClusterInfo: container_name = service_name service_info = services[service_name] image = service_info["image"] + port = int(service_info["ports"][0]) + if opts.o.debug: + print(f"image: {image}") + print(f"service port: {port}") # Re-write the image tag for remote deployment image_to_use = remote_tag_for_image( image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image @@ -173,7 +189,7 @@ class ClusterInfo: name=container_name, image=image_to_use, env=envs_from_environment_variables_map(self.environment_variables.map), - ports=[client.V1ContainerPort(container_port=80)], + ports=[client.V1ContainerPort(container_port=port)], volume_mounts=volume_mounts, resources=client.V1ResourceRequirements( requests={"cpu": "100m", "memory": "200Mi"}, diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index 8e790d10..5d41ae23 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -55,7 +55,7 @@ class K8sDeployer(Deployer): self.deployment_context = deployment_context self.kind_cluster_name = compose_project_name self.cluster_info = ClusterInfo() - self.cluster_info.int(compose_files, compose_env_file, deployment_context.spec) + self.cluster_info.int(compose_files, compose_env_file, compose_project_name, deployment_context.spec) if (opts.o.debug): print(f"Deployment dir: {deployment_context.deployment_dir}") print(f"Compose files: {compose_files}") @@ -126,6 +126,8 @@ class K8sDeployer(Deployer): # TODO: disable ingress for kind ingress: client.V1Ingress = self.cluster_info.get_ingress() + if opts.o.debug: + print(f"Sending this ingress: {ingress}") ingress_resp = self.networking_api.create_namespaced_ingress( namespace=self.k8s_namespace, body=ingress diff --git a/stack_orchestrator/deploy/webapp/__init__.py b/stack_orchestrator/deploy/webapp/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp.py b/stack_orchestrator/deploy/webapp/deploy_webapp.py new file mode 100644 index 00000000..a1e573fb --- /dev/null +++ b/stack_orchestrator/deploy/webapp/deploy_webapp.py @@ -0,0 +1,113 @@ +# Copyright ©2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import click +from pathlib import Path +from urllib.parse import urlparse + +from stack_orchestrator.util import error_exit, global_options2 +from stack_orchestrator.deploy.deployment_create import init_operation, create_operation +from stack_orchestrator.deploy.deploy import create_deploy_context +from stack_orchestrator.deploy.deploy_types import DeployCommandContext + + +def _fixup_container_tag(deployment_dir: str, image: str): + deployment_dir_path = Path(deployment_dir) + compose_file = deployment_dir_path.joinpath("compose", "docker-compose-webapp-template.yml") + # replace "cerc/webapp-container:local" in the file with our image tag + with open(compose_file) as rfile: + contents = rfile.read() + contents = contents.replace("cerc/webapp-container:local", image) + with open(compose_file, "w") as wfile: + wfile.write(contents) + + +def _fixup_url_spec(spec_file_name: str, url: str): + # url is like: https://example.com/path + parsed_url = urlparse(url) + http_proxy_spec = f''' + http-proxy: + - host-name: {parsed_url.hostname} + routes: + - path: '{parsed_url.path if parsed_url.path else "/"}' + proxy-to: webapp:3000 + ''' + spec_file_path = Path(spec_file_name) + with open(spec_file_path) as rfile: + contents = rfile.read() + contents = contents + http_proxy_spec + with open(spec_file_path, "w") as wfile: + wfile.write(contents) + + +@click.group() +@click.pass_context +def command(ctx): + '''manage a webapp deployment''' + + # Check that --stack wasn't supplied + if ctx.parent.obj.stack: + error_exit("--stack can't be supplied with the deploy-webapp command") + + +@command.command() +@click.option("--kube-config", help="Provide a config file for a k8s deployment") +@click.option("--image-registry", help="Provide a container image registry url for this k8s cluster") +@click.option("--deployment-dir", help="Create deployment files in this directory", required=True) +@click.option("--image", help="image to deploy", required=True) +@click.option("--url", help="url to serve", required=True) +@click.option("--env-file", help="environment file for webapp") +@click.pass_context +def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_file): + '''create a deployment for the specified webapp container''' + # Do the equivalent of: + # 1. laconic-so --stack webapp-template deploy --deploy-to k8s init --output webapp-spec.yml + # --config (eqivalent of the contents of my-config.env) + # 2. laconic-so --stack webapp-template deploy --deploy-to k8s create --deployment-dir test-deployment + # --spec-file webapp-spec.yml + # 3. Replace the container image tag with the specified image + deployment_dir_path = Path(deployment_dir) + # Check the deployment dir does not exist + if deployment_dir_path.exists(): + error_exit(f"Deployment dir {deployment_dir} already exists") + # Generate a temporary file name for the spec file + spec_file_name = "webapp-spec.yml" + # Specify the webapp template stack + stack = "webapp-template" + # TODO: support env file + deploy_command_context: DeployCommandContext = create_deploy_context( + global_options2(ctx), None, stack, None, None, None, env_file, "k8s" + ) + init_operation( + deploy_command_context, + stack, + "k8s", + None, + kube_config, + image_registry, + spec_file_name, + None + ) + # Add the TLS and DNS spec + _fixup_url_spec(spec_file_name, url) + create_operation( + deploy_command_context, + spec_file_name, + deployment_dir, + None, + None + ) + # Fix up the container tag inside the deployment compose file + _fixup_container_tag(deployment_dir, image) diff --git a/stack_orchestrator/deploy/run_webapp.py b/stack_orchestrator/deploy/webapp/run_webapp.py similarity index 91% rename from stack_orchestrator/deploy/run_webapp.py rename to stack_orchestrator/deploy/webapp/run_webapp.py index aa22acdf..e4e01171 100644 --- a/stack_orchestrator/deploy/run_webapp.py +++ b/stack_orchestrator/deploy/webapp/run_webapp.py @@ -22,17 +22,17 @@ import hashlib import click - from dotenv import dotenv_values + +from stack_orchestrator import constants from stack_orchestrator.deploy.deployer_factory import getDeployer @click.command() @click.option("--image", help="image to deploy", required=True) -@click.option("--deploy-to", default="compose", help="deployment type ([Docker] 'compose' or 'k8s')") @click.option("--env-file", help="environment file for webapp") @click.pass_context -def command(ctx, image, deploy_to, env_file): +def command(ctx, image, env_file): '''build the specified webapp container''' env = {} @@ -43,7 +43,7 @@ def command(ctx, image, deploy_to, env_file): hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest() cluster = f"laconic-webapp-{hash}" - deployer = getDeployer(deploy_to, + deployer = getDeployer(type=constants.compose_deploy_type, deployment_context=None, compose_files=None, compose_project_name=cluster, diff --git a/stack_orchestrator/main.py b/stack_orchestrator/main.py index 8ee8ae61..26a011b0 100644 --- a/stack_orchestrator/main.py +++ b/stack_orchestrator/main.py @@ -20,7 +20,7 @@ from stack_orchestrator.repos import setup_repositories from stack_orchestrator.build import build_containers from stack_orchestrator.build import build_npms from stack_orchestrator.build import build_webapp -from stack_orchestrator.deploy import run_webapp +from stack_orchestrator.deploy.webapp import run_webapp, deploy_webapp from stack_orchestrator.deploy import deploy from stack_orchestrator import version from stack_orchestrator.deploy import deployment @@ -52,6 +52,7 @@ cli.add_command(build_containers.command, "build-containers") cli.add_command(build_npms.command, "build-npms") cli.add_command(build_webapp.command, "build-webapp") cli.add_command(run_webapp.command, "run-webapp") +cli.add_command(deploy_webapp.command, "deploy-webapp") cli.add_command(deploy.command, "deploy") # deploy is an alias for deploy-system cli.add_command(deploy.command, "deploy-system") cli.add_command(deployment.command, "deployment") From 1a069a6816300ac26dfa7797508ecee0f38c320e Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 28 Nov 2023 19:56:12 -0700 Subject: [PATCH 30/75] Use a temp file for the spec file name (#668) --- stack_orchestrator/deploy/webapp/deploy_webapp.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp.py b/stack_orchestrator/deploy/webapp/deploy_webapp.py index a1e573fb..a2cd7253 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp.py @@ -14,8 +14,10 @@ # along with this program. If not, see . import click +import os from pathlib import Path from urllib.parse import urlparse +from tempfile import NamedTemporaryFile from stack_orchestrator.util import error_exit, global_options2 from stack_orchestrator.deploy.deployment_create import init_operation, create_operation @@ -83,7 +85,8 @@ def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_fil if deployment_dir_path.exists(): error_exit(f"Deployment dir {deployment_dir} already exists") # Generate a temporary file name for the spec file - spec_file_name = "webapp-spec.yml" + tf = NamedTemporaryFile(prefix="webapp-", suffix=".yml", delete=False) + spec_file_name = tf.name # Specify the webapp template stack stack = "webapp-template" # TODO: support env file @@ -111,3 +114,4 @@ def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_fil ) # Fix up the container tag inside the deployment compose file _fixup_container_tag(deployment_dir, image) + os.remove(spec_file_name) From 113c0bfbf163d1158c4ab67c38dd50cdd60667e9 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 28 Nov 2023 21:14:02 -0700 Subject: [PATCH 31/75] Propagate env file for webapp deployment (#669) --- stack_orchestrator/constants.py | 2 ++ .../deploy/deployment_create.py | 31 +++++++++++++------ stack_orchestrator/deploy/k8s/cluster_info.py | 3 +- stack_orchestrator/deploy/k8s/helpers.py | 5 --- .../deploy/webapp/deploy_webapp.py | 1 + stack_orchestrator/util.py | 6 ++++ 6 files changed, 33 insertions(+), 15 deletions(-) diff --git a/stack_orchestrator/constants.py b/stack_orchestrator/constants.py index 1cff6055..54cfe355 100644 --- a/stack_orchestrator/constants.py +++ b/stack_orchestrator/constants.py @@ -14,6 +14,8 @@ # along with this program. If not, see . stack_file_name = "stack.yml" +spec_file_name = "spec.yml" +config_file_name = "config.env" compose_deploy_type = "compose" k8s_kind_deploy_type = "k8s-kind" k8s_deploy_type = "k8s" diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py index fd52dba8..88ce0b2a 100644 --- a/stack_orchestrator/deploy/deployment_create.py +++ b/stack_orchestrator/deploy/deployment_create.py @@ -25,7 +25,7 @@ from stack_orchestrator import constants from stack_orchestrator.opts import opts from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config, global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts, - get_pod_script_paths, get_plugin_code_paths, error_exit) + get_pod_script_paths, get_plugin_code_paths, error_exit, env_var_map_from_file) from stack_orchestrator.deploy.deploy_types import LaconicStackSetupCommand from stack_orchestrator.deploy.deployer_factory import getDeployerConfigGenerator from stack_orchestrator.deploy.deployment_context import DeploymentContext @@ -244,12 +244,13 @@ def _parse_config_variables(variable_values: str): variable_name = variable_value_pair[0] variable_value = variable_value_pair[1] result_values[variable_name] = variable_value - result = {"config": result_values} + result = result_values return result @click.command() @click.option("--config", help="Provide config variables for the deployment") +@click.option("--config-file", help="Provide config variables in a file for the deployment") @click.option("--kube-config", help="Provide a config file for a k8s deployment") @click.option("--image-registry", help="Provide a container image registry url for this k8s cluster") @click.option("--output", required=True, help="Write yaml spec file here") @@ -257,14 +258,15 @@ def _parse_config_variables(variable_values: str): help="Map ports to the host as one of: any-variable-random (default), " "localhost-same, any-same, localhost-fixed-random, any-fixed-random") @click.pass_context -def init(ctx, config, kube_config, image_registry, output, map_ports_to_host): +def init(ctx, config, config_file, kube_config, image_registry, output, map_ports_to_host): stack = global_options(ctx).stack deployer_type = ctx.obj.deployer.type deploy_command_context = ctx.obj return init_operation( deploy_command_context, stack, deployer_type, - config, kube_config, + config, config_file, + kube_config, image_registry, output, map_ports_to_host) @@ -272,7 +274,8 @@ def init(ctx, config, kube_config, image_registry, output, map_ports_to_host): # The init command's implementation is in a separate function so that we can # call it from other commands, bypassing the click decoration stuff -def init_operation(deploy_command_context, stack, deployer_type, config, kube_config, image_registry, output, map_ports_to_host): +def init_operation(deploy_command_context, stack, deployer_type, config, + config_file, kube_config, image_registry, output, map_ports_to_host): yaml = get_yaml() default_spec_file_content = call_stack_deploy_init(deploy_command_context) spec_file_content = {"stack": stack, constants.deploy_to_key: deployer_type} @@ -292,12 +295,22 @@ def init_operation(deploy_command_context, stack, deployer_type, config, kube_co if default_spec_file_content: spec_file_content.update(default_spec_file_content) config_variables = _parse_config_variables(config) + # Implement merge, since update() overwrites if config_variables: - # Implement merge, since update() overwrites orig_config = spec_file_content.get("config", {}) - new_config = config_variables["config"] + new_config = config_variables merged_config = {**new_config, **orig_config} spec_file_content.update({"config": merged_config}) + if config_file: + config_file_path = Path(config_file) + if not config_file_path.exists(): + error_exit(f"config file: {config_file} does not exist") + config_file_variables = env_var_map_from_file(config_file_path) + if config_file_variables: + orig_config = spec_file_content.get("config", {}) + new_config = config_file_variables + merged_config = {**new_config, **orig_config} + spec_file_content.update({"config": merged_config}) if opts.o.debug: print(f"Creating spec file for stack: {stack} with content: {spec_file_content}") @@ -368,10 +381,10 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw error_exit(f"{deployment_dir_path} already exists") os.mkdir(deployment_dir_path) # Copy spec file and the stack file into the deployment dir - copyfile(spec_file, deployment_dir_path.joinpath("spec.yml")) + copyfile(spec_file, deployment_dir_path.joinpath(constants.spec_file_name)) copyfile(stack_file, deployment_dir_path.joinpath(os.path.basename(stack_file))) # Copy any config varibles from the spec file into an env file suitable for compose - _write_config_file(spec_file, deployment_dir_path.joinpath("config.env")) + _write_config_file(spec_file, deployment_dir_path.joinpath(constants.config_file_name)) # Copy any k8s config file into the deployment dir if deployment_type == "k8s": _write_kube_config_file(Path(parsed_spec[constants.kube_config_key]), diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 6c19b20a..0aa74189 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -17,9 +17,10 @@ from kubernetes import client from typing import Any, List, Set from stack_orchestrator.opts import opts +from stack_orchestrator.util import env_var_map_from_file from stack_orchestrator.deploy.k8s.helpers import named_volumes_from_pod_files, volume_mounts_for_service, volumes_for_pod_files from stack_orchestrator.deploy.k8s.helpers import get_node_pv_mount_path -from stack_orchestrator.deploy.k8s.helpers import env_var_map_from_file, envs_from_environment_variables_map +from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variables_map from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment from stack_orchestrator.deploy.deploy_types import DeployEnvVars from stack_orchestrator.deploy.spec import Spec diff --git a/stack_orchestrator/deploy/k8s/helpers.py b/stack_orchestrator/deploy/k8s/helpers.py index 82a33792..9f968dbf 100644 --- a/stack_orchestrator/deploy/k8s/helpers.py +++ b/stack_orchestrator/deploy/k8s/helpers.py @@ -14,7 +14,6 @@ # along with this program. If not, see . from kubernetes import client -from dotenv import dotenv_values import os from pathlib import Path import subprocess @@ -224,7 +223,3 @@ def generate_kind_config(deployment_dir: Path): f"{port_mappings_yml}\n" f"{mounts_yml}\n" ) - - -def env_var_map_from_file(file: Path) -> Mapping[str, str]: - return dotenv_values(file) diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp.py b/stack_orchestrator/deploy/webapp/deploy_webapp.py index a2cd7253..391162c9 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp.py @@ -98,6 +98,7 @@ def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_fil stack, "k8s", None, + env_file, kube_config, image_registry, spec_file_name, diff --git a/stack_orchestrator/util.py b/stack_orchestrator/util.py index 97d48963..0bd1a609 100644 --- a/stack_orchestrator/util.py +++ b/stack_orchestrator/util.py @@ -18,6 +18,8 @@ import os.path import sys import ruamel.yaml from pathlib import Path +from dotenv import dotenv_values +from typing import Mapping def include_exclude_check(s, include, exclude): @@ -178,3 +180,7 @@ def global_options2(ctx): def error_exit(s): print(f"ERROR: {s}") sys.exit(1) + + +def env_var_map_from_file(file: Path) -> Mapping[str, str]: + return dotenv_values(file) From 03a3645b3c5486668f7fbd03d60dc72ce4eac574 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 29 Nov 2023 11:32:28 -0600 Subject: [PATCH 32/75] Add --port option to run-webapp. (#667) * Add --port option to run-webapp * Fixed merge * lint --- stack_orchestrator/deploy/compose/deploy_docker.py | 4 ++-- stack_orchestrator/deploy/deployer.py | 2 +- stack_orchestrator/deploy/k8s/deploy_k8s.py | 2 +- stack_orchestrator/deploy/webapp/run_webapp.py | 14 ++++++++++---- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/stack_orchestrator/deploy/compose/deploy_docker.py b/stack_orchestrator/deploy/compose/deploy_docker.py index 4b4e7426..04f24df5 100644 --- a/stack_orchestrator/deploy/compose/deploy_docker.py +++ b/stack_orchestrator/deploy/compose/deploy_docker.py @@ -64,10 +64,10 @@ class DockerDeployer(Deployer): except DockerException as e: raise DeployerException(e) - def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, detach=False): + def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, ports=[], detach=False): try: return self.docker.run(image=image, command=command, user=user, volumes=volumes, - entrypoint=entrypoint, envs=env, detach=detach, publish_all=True) + entrypoint=entrypoint, envs=env, detach=detach, publish=ports, publish_all=len(ports) == 0) except DockerException as e: raise DeployerException(e) diff --git a/stack_orchestrator/deploy/deployer.py b/stack_orchestrator/deploy/deployer.py index 79379c3d..984945ed 100644 --- a/stack_orchestrator/deploy/deployer.py +++ b/stack_orchestrator/deploy/deployer.py @@ -44,7 +44,7 @@ class Deployer(ABC): pass @abstractmethod - def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, detach=False): + def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, ports=[], detach=False): pass diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index 5d41ae23..c84aa34a 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -230,7 +230,7 @@ class K8sDeployer(Deployer): log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test") return log_stream_from_string(log_data) - def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, detach=False): + def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, ports=[], detach=False): # We need to figure out how to do this -- check why we're being called first pass diff --git a/stack_orchestrator/deploy/webapp/run_webapp.py b/stack_orchestrator/deploy/webapp/run_webapp.py index e4e01171..4dbf234a 100644 --- a/stack_orchestrator/deploy/webapp/run_webapp.py +++ b/stack_orchestrator/deploy/webapp/run_webapp.py @@ -27,13 +27,16 @@ from dotenv import dotenv_values from stack_orchestrator import constants from stack_orchestrator.deploy.deployer_factory import getDeployer +WEBAPP_PORT = 3000 + @click.command() @click.option("--image", help="image to deploy", required=True) @click.option("--env-file", help="environment file for webapp") +@click.option("--port", help="port to use (default random)") @click.pass_context -def command(ctx, image, env_file): - '''build the specified webapp container''' +def command(ctx, image, env_file, port): + '''run the specified webapp container''' env = {} if env_file: @@ -49,10 +52,13 @@ def command(ctx, image, env_file): compose_project_name=cluster, compose_env_file=None) - container = deployer.run(image, command=[], user=None, volumes=[], entrypoint=None, env=env, detach=True) + ports = [] + if port: + ports = [(port, WEBAPP_PORT)] + container = deployer.run(image, command=[], user=None, volumes=[], entrypoint=None, env=env, ports=ports, detach=True) # Make configurable? - webappPort = "3000/tcp" + webappPort = f"{WEBAPP_PORT}/tcp" # TODO: This assumes a Docker container object... if webappPort in container.network_settings.ports: mapping = container.network_settings.ports[webappPort][0] From d7093277b4743b7eb0d533bf6c6d0327d696f762 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Wed, 29 Nov 2023 20:50:53 -0700 Subject: [PATCH 33/75] Use constants (#671) --- stack_orchestrator/deploy/deployment_context.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/stack_orchestrator/deploy/deployment_context.py b/stack_orchestrator/deploy/deployment_context.py index cd731394..cbee4151 100644 --- a/stack_orchestrator/deploy/deployment_context.py +++ b/stack_orchestrator/deploy/deployment_context.py @@ -16,6 +16,7 @@ from pathlib import Path +from stack_orchestrator import constants from stack_orchestrator.deploy.stack import Stack from stack_orchestrator.deploy.spec import Spec @@ -26,13 +27,13 @@ class DeploymentContext: stack: Stack def get_stack_file(self): - return self.deployment_dir.joinpath("stack.yml") + return self.deployment_dir.joinpath(constants.stack_file_name) def get_spec_file(self): - return self.deployment_dir.joinpath("spec.yml") + return self.deployment_dir.joinpath(constants.spec_file_name) def get_env_file(self): - return self.deployment_dir.joinpath("config.env") + return self.deployment_dir.joinpath(constants.config_file_name) # TODO: implement me def get_cluster_name(self): From c19559967d667c59554721bb85226d23ab781a12 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Wed, 29 Nov 2023 20:55:14 -0700 Subject: [PATCH 34/75] Add doc for deploy-webapp (#672) --- docs/webapp.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/webapp.md b/docs/webapp.md index 3b1d0609..fcf4ffcb 100644 --- a/docs/webapp.md +++ b/docs/webapp.md @@ -34,7 +34,7 @@ To test locally run: ## Running -With `run-webapp` a new container will be launched with runtime configuration provided by `--env-file` (if specified) and published on an available port. Multiple instances can be launched with different configuration. +With `run-webapp` a new container will be launched on the local machine, with runtime configuration provided by `--env-file` (if specified) and published on an available port. Multiple instances can be launched with different configuration. **Example**: ``` @@ -52,3 +52,13 @@ Image: cerc/test-progressive-web-app:local ID: 9ab96494f563aafb6c057d88df58f9eca81b90f8721a4e068493a289a976051c URL: http://localhost:32769 ``` + +## Deploying + +Use the subcommand `deploy-webapp create` to make a deployment directory that can be subsequently deployed to a Kubernetes cluster. +Example commands are shown below, assuming that the webapp container image `cerc/test-progressive-web-app:local` has already been built: +``` +$ laconic-so deploy-webapp create --kube-config ~/kubectl/k8s-kubeconfig.yaml --image-registry registry.digitalocean.com/laconic-registry --deployment-dir webapp-k8s-deployment --image cerc/test-progressive-web-app:local --url https://test-pwa-app.hosting.laconic.com/ --env-file test-webapp.env +$ laconic-so deployment --dir webapp-k8s-deployment push-images +$ laconic-so deployment --dir webapp-k8s-deployment start +``` From 2173e7ce6a1eab21ee11149d61fb6537c3cccc8c Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Thu, 30 Nov 2023 12:33:06 -0600 Subject: [PATCH 35/75] If the next version is unsupported, print a big warning and try higher version. (#674) --- .../cerc-nextjs-base/Dockerfile | 2 ++ .../cerc-nextjs-base/scripts/build-app.sh | 32 ++++++++++++++++--- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile index c2416b67..d3ff3f1b 100644 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile @@ -24,6 +24,8 @@ RUN \ && su ${USERNAME} -c "npm config -g set prefix ${NPM_GLOBAL}" \ # Install eslint && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \ + # Install semver + && su ${USERNAME} -c "umask 0002 && npm install -g semver" \ && npm cache clean --force > /dev/null 2>&1 # [Optional] Uncomment this section to install additional OS packages. diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh index e62bc0d0..ef6244cf 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh @@ -4,10 +4,12 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi +CERC_MIN_NEXTVER=13.4.2 + CERC_NEXT_VERSION="${CERC_NEXT_VERSION:-keep}" CERC_BUILD_TOOL="${CERC_BUILD_TOOL}" if [ -z "$CERC_BUILD_TOOL" ]; then - if [ -f "yarn.lock" ] && [ ! -f "package-lock.json" ]; then + if [ -f "yarn.lock" ]; then CERC_BUILD_TOOL=yarn else CERC_BUILD_TOOL=npm @@ -101,13 +103,35 @@ cat package.dist | jq '.scripts.cerc_compile = "next experimental-compile"' | jq CUR_NEXT_VERSION="`jq -r '.dependencies.next' package.json`" if [ "$CERC_NEXT_VERSION" != "keep" ] && [ "$CUR_NEXT_VERSION" != "$CERC_NEXT_VERSION" ]; then - echo "Changing 'next' version specifier from '$CUR_NEXT_VERSION' to '$CERC_NEXT_VERSION' (set with --build-arg CERC_NEXT_VERSION)" + echo "Changing 'next' version specifier from '$CUR_NEXT_VERSION' to '$CERC_NEXT_VERSION' (set with '--extra-build-args \"--build-arg CERC_NEXT_VERSION=$CERC_NEXT_VERSION\"')" cat package.json | jq ".dependencies.next = \"$CERC_NEXT_VERSION\"" | sponge package.json -else - echo "'next' version specifier '$CUR_NEXT_VERSION' (override with --build-arg CERC_NEXT_VERSION)" fi $CERC_BUILD_TOOL install || exit 1 + +CUR_NEXT_VERSION=`jq -r '.version' node_modules/next/package.json` + +semver -p -r ">=$CERC_MIN_NEXTVER" $CUR_NEXT_VERSION +if [ $? -ne 0 ]; then + cat <" + +############################################################################### + +EOF + cat package.json | jq ".dependencies.next = \"^$CERC_MIN_NEXTVER\"" | sponge package.json + $CERC_BUILD_TOOL install || exit 1 +fi + $CERC_BUILD_TOOL run cerc_compile || exit 1 exit 0 From c319e90ddddb215ff4049afbce3ec87742e155bb Mon Sep 17 00:00:00 2001 From: Nabarun Gogoi Date: Mon, 4 Dec 2023 18:39:19 +0530 Subject: [PATCH 36/75] Add a stack for running uniswap frontend on urbit (#670) * Create uniswap-frontend stack * Add stack for building uniswap frontend app * Add a container for Urbit fake ship * Update with deployment command * Add a service for uniswap app deployment to urbit * Use a script to start urbit ship to handle restarts * Rename stack name to uniswap-urbit-app * Rename build.sh to build-app.sh and check if build already exists * Rename stack directory name * Update uniswap build restart on failure * Perform uniswap app deployment in the urbit container * Add steps to create glob for the app * Tail /dev/null after deployment * Add steps to install the app to desk * Host glob files for uniswap * Update repo branch * Update readme with command to get urbit password * Update readme * Update readme to open urbit web UI * Expose the port on glob hosting container * Avoid exposing urbit http port * Add scripts for installing uniswap on remote urbit instance * Configure GQL proxy for uniswap app * Use laconic branch for app repo * Rename urbit pod for uniswap app deployment --------- Co-authored-by: Prathamesh Musale --- .../docker-compose-uniswap-interface.yml | 49 ++++++ .../compose/docker-compose-uniswap-urbit.yml | 26 ++++ .../config/uniswap-interface/build-app.sh | 18 +++ .../uniswap-interface/deploy-uniswap-app.sh | 142 ++++++++++++++++++ .../uniswap-interface/host-uniswap-glob.sh | 23 +++ .../uniswap-interface/install-uniswap-app.sh | 109 ++++++++++++++ .../remote-deploy-uniswap.sh | 18 +++ .../data/config/urbit/run-urbit-ship.sh | 18 +++ .../cerc-uniswap-interface/Dockerfile | 10 ++ .../cerc-uniswap-interface/build.sh | 8 + .../cerc-urbit-globs-host/Dockerfile | 7 + .../cerc-urbit-globs-host/build.sh | 9 ++ .../data/container-image-list.txt | 1 + stack_orchestrator/data/repository-list.txt | 1 + .../data/stacks/uniswap-urbit-app/README.md | 122 +++++++++++++++ .../data/stacks/uniswap-urbit-app/stack.yml | 10 ++ 16 files changed, 571 insertions(+) create mode 100644 stack_orchestrator/data/compose/docker-compose-uniswap-interface.yml create mode 100644 stack_orchestrator/data/compose/docker-compose-uniswap-urbit.yml create mode 100755 stack_orchestrator/data/config/uniswap-interface/build-app.sh create mode 100755 stack_orchestrator/data/config/uniswap-interface/deploy-uniswap-app.sh create mode 100755 stack_orchestrator/data/config/uniswap-interface/host-uniswap-glob.sh create mode 100755 stack_orchestrator/data/config/uniswap-interface/install-uniswap-app.sh create mode 100755 stack_orchestrator/data/config/uniswap-interface/remote-deploy-uniswap.sh create mode 100755 stack_orchestrator/data/config/urbit/run-urbit-ship.sh create mode 100644 stack_orchestrator/data/container-build/cerc-uniswap-interface/Dockerfile create mode 100755 stack_orchestrator/data/container-build/cerc-uniswap-interface/build.sh create mode 100644 stack_orchestrator/data/container-build/cerc-urbit-globs-host/Dockerfile create mode 100755 stack_orchestrator/data/container-build/cerc-urbit-globs-host/build.sh create mode 100644 stack_orchestrator/data/stacks/uniswap-urbit-app/README.md create mode 100644 stack_orchestrator/data/stacks/uniswap-urbit-app/stack.yml diff --git a/stack_orchestrator/data/compose/docker-compose-uniswap-interface.yml b/stack_orchestrator/data/compose/docker-compose-uniswap-interface.yml new file mode 100644 index 00000000..f6a5c53f --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-uniswap-interface.yml @@ -0,0 +1,49 @@ +version: "3.2" + +services: + uniswap-interface: + image: cerc/uniswap-interface:local + restart: on-failure + environment: + - REACT_APP_INFURA_KEY=${CERC_INFURA_KEY} + - REACT_APP_AWS_API_ENDPOINT=${CERC_UNISWAP_GQL} + command: ["./build-app.sh"] + volumes: + - app_builds:/app-builds + - ../config/uniswap-interface/build-app.sh:/app/build-app.sh + + uniswap-glob-host: + image: cerc/urbit-globs-host:local + restart: unless-stopped + depends_on: + uniswap-interface: + condition: service_completed_successfully + command: ["./host-uniswap-glob.sh"] + volumes: + - app_globs:/app-globs + - ../config/uniswap-interface/host-uniswap-glob.sh:/app/host-uniswap-glob.sh + ports: + - "3000" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "3000"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 10s + + uniswap-gql-proxy: + image: cerc/uniswap-interface:local + restart: on-failure + command: ["bash", "-c", "yarn proxy-gql"] + ports: + - "4000" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "4000"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 10s + +volumes: + app_builds: + app_globs: diff --git a/stack_orchestrator/data/compose/docker-compose-uniswap-urbit.yml b/stack_orchestrator/data/compose/docker-compose-uniswap-urbit.yml new file mode 100644 index 00000000..ae0b3709 --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-uniswap-urbit.yml @@ -0,0 +1,26 @@ +version: '3.7' + +services: + urbit-fake-ship: + restart: unless-stopped + image: tloncorp/vere + entrypoint: ["bash", "-c", "./run-urbit-ship.sh && ./deploy-uniswap-app.sh && tail -f /dev/null"] + volumes: + - urbit_data:/urbit + - app_builds:/app-builds + - app_globs:/app-globs + - ../config/urbit/run-urbit-ship.sh:/urbit/run-urbit-ship.sh + - ../config/uniswap-interface/deploy-uniswap-app.sh:/urbit/deploy-uniswap-app.sh + ports: + - "80" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "80"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 10s + +volumes: + urbit_data: + app_builds: + app_globs: diff --git a/stack_orchestrator/data/config/uniswap-interface/build-app.sh b/stack_orchestrator/data/config/uniswap-interface/build-app.sh new file mode 100755 index 00000000..d3b012e6 --- /dev/null +++ b/stack_orchestrator/data/config/uniswap-interface/build-app.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +# Check and exit if a deployment already exists (on restarts) +if [ -d /app-builds/uniswap/build ]; then + echo "Build already exists, remove volume to rebuild" + exit 0 +fi + +yarn build + +# Move build to app-builds so urbit can deploy it +mkdir /app-builds/uniswap +cp -r ./build /app-builds/uniswap/ diff --git a/stack_orchestrator/data/config/uniswap-interface/deploy-uniswap-app.sh b/stack_orchestrator/data/config/uniswap-interface/deploy-uniswap-app.sh new file mode 100755 index 00000000..6c147083 --- /dev/null +++ b/stack_orchestrator/data/config/uniswap-interface/deploy-uniswap-app.sh @@ -0,0 +1,142 @@ +#!/bin/bash + +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +uniswap_app_build='/app-builds/uniswap/build' +uniswap_desk_dir='/urbit/zod/uniswap' + +if [ -d ${uniswap_desk_dir} ]; then + echo "Uniswap desk dir already exists, skipping deployment..." + exit 0 +fi + +# Fire curl requests to perform operations on the ship +dojo () { + curl -s --data '{"source":{"dojo":"'"$1"'"},"sink":{"stdout":null}}' http://localhost:12321 +} + +hood () { + curl -s --data '{"source":{"dojo":"+hood/'"$1"'"},"sink":{"app":"hood"}}' http://localhost:12321 +} + +# Create/mount a uniswap desk +hood "merge %uniswap our %landscape" +hood "mount %uniswap" + +# Loop until the uniswap build appears +while [ ! -d ${uniswap_app_build} ]; do + echo "Uniswap app build not found, retrying in 5s..." + sleep 5 +done +echo "Build found..." + +# Copy over build to desk data dir +cp -r ${uniswap_app_build} ${uniswap_desk_dir} + +# Create a mark file for .map file type +cat << EOF > "${uniswap_desk_dir}/mar/map.hoon" +:: +:::: /hoon/map/mar + :: Mark for js source maps +/? 310 +:: +=, eyre +|_ mud=@ +++ grow + |% + ++ mime [/application/octet-stream (as-octs:mimes:html (@t mud))] + -- +++ grab + |% :: convert from + ++ mime |=([p=mite q=octs] (@t q.q)) + ++ noun cord :: clam from %noun + -- +++ grad %mime +-- +EOF + +# Create a mark file for .woff file type +cat << EOF > "${uniswap_desk_dir}/mar/woff.hoon" +|_ dat=octs +++ grow + |% + ++ mime [/font/woff dat] + -- +++ grab + |% + ++ mime |=([=mite =octs] octs) + ++ noun octs + -- +++ grad %mime +-- +EOF + +# Create a mark file for .ttf file type +cat << EOF > "${uniswap_desk_dir}/mar/ttf.hoon" +|_ dat=octs +++ grow + |% + ++ mime [/font/ttf dat] + -- +++ grab + |% + ++ mime |=([=mite =octs] octs) + ++ noun octs + -- +++ grad %mime +-- +EOF + +rm "${uniswap_desk_dir}/desk.bill" +rm "${uniswap_desk_dir}/desk.ship" + +# Commit changes and create a glob +hood "commit %uniswap" +dojo "-landscape!make-glob %uniswap /build" + +echo "Copying over glob file to mounted volume" +mkdir -p /app-globs/uniswap +cp /urbit/zod/.urb/put/* /app-globs/uniswap/ + +glob_file=$(ls -1 -c zod/.urb/put | head -1) +echo "Glob filename: ${glob_file}" + +# Curl and wait for the glob to be hosted +glob_url="http://uniswap-glob-host:3000/${glob_file}" + +echo "Checking if glob file hosted at ${glob_url}" +while true; do + response=$(curl -sL -w "%{http_code}" -o /dev/null "$glob_url") + + if [ $response -eq 200 ]; then + echo "File found at $glob_url" + break # Exit the loop if the file is found + else + echo "File not found. Retrying in a few seconds..." + sleep 5 + fi +done + +glob_hash=$(echo "$glob_file" | sed "s/glob-\([a-z0-9\.]*\).glob/\1/") + +# Update the docket file +cat << EOF > "${uniswap_desk_dir}/desk.docket-0" +:~ title+'Uniswap' + info+'Self-hosted uniswap frontend.' + color+0xcd.75df + image+'https://logowik.com/content/uploads/images/uniswap-uni7403.jpg' + base+'uniswap' + glob-http+['http://uniswap-glob-host:3000/${glob_file}' ${glob_hash}] + version+[0 0 1] + website+'https://uniswap.org/' + license+'MIT' +== +EOF + +# Commit changes and install the app +hood "commit %uniswap" +hood "install our %uniswap" + +echo "Uniswap app installed" diff --git a/stack_orchestrator/data/config/uniswap-interface/host-uniswap-glob.sh b/stack_orchestrator/data/config/uniswap-interface/host-uniswap-glob.sh new file mode 100755 index 00000000..37605794 --- /dev/null +++ b/stack_orchestrator/data/config/uniswap-interface/host-uniswap-glob.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +# Use config from mounted volume (when running web-app along with watcher stack) +echo "Waiting for uniswap app glob" +while [ ! -d /app-globs/uniswap ]; do + echo "Glob directory not found, retrying in 5 seconds..." + sleep 5 +done + + +# Copy to a new globs directory +mkdir -p globs +cp -r /app-globs/uniswap/* ./globs + +# Serve the glob file +cd globs +echo "Hosting glob file at port 3000" +python3 -m http.server 3000 --bind 0.0.0.0 diff --git a/stack_orchestrator/data/config/uniswap-interface/install-uniswap-app.sh b/stack_orchestrator/data/config/uniswap-interface/install-uniswap-app.sh new file mode 100755 index 00000000..679bc27e --- /dev/null +++ b/stack_orchestrator/data/config/uniswap-interface/install-uniswap-app.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +# $1: Glob file URL (eg. https://xyz.com/glob-abcd.glob) +# $2: Uniswap desk dir (default: ./zod/uniswap) + +if [ -z "$1" ]; then + echo "Glob file URL arg not provided" + exit 0 +fi + +glob_url=$1 +glob_file=$(basename "$glob_url") +glob_hash=$(echo "$glob_file" | sed "s/glob-\([a-z0-9\.]*\).glob/\1/") +echo "Using glob file ${glob_file}" + +# Default desk dir: ./zod/uniswap +uniswap_desk_dir="${2:-./zod/uniswap}" + +echo "Using ${uniswap_desk_dir} as the Uniswap desk dir path" + +# Fire curl requests to perform operations on the ship +dojo () { + curl -s --data '{"source":{"dojo":"'"$1"'"},"sink":{"stdout":null}}' http://localhost:12321 +} + +hood () { + curl -s --data '{"source":{"dojo":"+hood/'"$1"'"},"sink":{"app":"hood"}}' http://localhost:12321 +} + +# Create/mount a uniswap desk +hood "merge %uniswap our %landscape" +hood "mount %uniswap" + +# Create a mark file for .map file type +cat << EOF > "${uniswap_desk_dir}/mar/map.hoon" +:: +:::: /hoon/map/mar + :: Mark for js source maps +/? 310 +:: +=, eyre +|_ mud=@ +++ grow + |% + ++ mime [/application/octet-stream (as-octs:mimes:html (@t mud))] + -- +++ grab + |% :: convert from + ++ mime |=([p=mite q=octs] (@t q.q)) + ++ noun cord :: clam from %noun + -- +++ grad %mime +-- +EOF + +# Create a mark file for .woff file type +cat << EOF > "${uniswap_desk_dir}/mar/woff.hoon" +|_ dat=octs +++ grow + |% + ++ mime [/font/woff dat] + -- +++ grab + |% + ++ mime |=([=mite =octs] octs) + ++ noun octs + -- +++ grad %mime +-- +EOF + +# Create a mark file for .ttf file type +cat << EOF > "${uniswap_desk_dir}/mar/ttf.hoon" +|_ dat=octs +++ grow + |% + ++ mime [/font/ttf dat] + -- +++ grab + |% + ++ mime |=([=mite =octs] octs) + ++ noun octs + -- +++ grad %mime +-- +EOF + +rm "${uniswap_desk_dir}/desk.bill" +rm "${uniswap_desk_dir}/desk.ship" + +# Update the docket file +cat << EOF > "${uniswap_desk_dir}/desk.docket-0" +:~ title+'Uniswap' + info+'Self-hosted uniswap frontend.' + color+0xcd.75df + image+'https://logowik.com/content/uploads/images/uniswap-uni7403.jpg' + base+'uniswap' + glob-http+['${glob_url}' ${glob_hash}] + version+[0 0 1] + website+'https://uniswap.org/' + license+'MIT' +== +EOF + +# Commit changes and install the app +hood "commit %uniswap" +hood "install our %uniswap" + +echo "Uniswap app installed" diff --git a/stack_orchestrator/data/config/uniswap-interface/remote-deploy-uniswap.sh b/stack_orchestrator/data/config/uniswap-interface/remote-deploy-uniswap.sh new file mode 100755 index 00000000..528151e9 --- /dev/null +++ b/stack_orchestrator/data/config/uniswap-interface/remote-deploy-uniswap.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# $1: Remote user host +# $2: Path to run the app installation in (where urbit ship dir is located) +# $3: Glob file URL (eg. https://xyz.com/glob-abcd.glob) + +if [ "$#" -ne 3 ]; then + echo "Usage: $0 " + exit 1 +fi + +remote_user_host="$1" +remote_folder="$2" +glob_url="$3" + +installation_script="./install-uniswap-app.sh" + +ssh "$remote_user_host" "cd $remote_folder && bash -s $glob_url" < "$installation_script" diff --git a/stack_orchestrator/data/config/urbit/run-urbit-ship.sh b/stack_orchestrator/data/config/urbit/run-urbit-ship.sh new file mode 100755 index 00000000..bb301c81 --- /dev/null +++ b/stack_orchestrator/data/config/urbit/run-urbit-ship.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +pier_dir="/urbit/zod" + +# Run urbit ship in daemon mode +# Check if the directory exists +if [ -d "$pier_dir" ]; then + echo "Pier directory already exists, rebooting..." + urbit -d zod +else + echo "Creating a new fake ship..." + urbit -d -F zod +fi diff --git a/stack_orchestrator/data/container-build/cerc-uniswap-interface/Dockerfile b/stack_orchestrator/data/container-build/cerc-uniswap-interface/Dockerfile new file mode 100644 index 00000000..59804896 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-uniswap-interface/Dockerfile @@ -0,0 +1,10 @@ +FROM node:18.17.1-alpine3.18 + +RUN apk --update --no-cache add git make alpine-sdk bash + +WORKDIR /app + +COPY . . + +RUN echo "Building uniswap-interface" && \ + yarn diff --git a/stack_orchestrator/data/container-build/cerc-uniswap-interface/build.sh b/stack_orchestrator/data/container-build/cerc-uniswap-interface/build.sh new file mode 100755 index 00000000..af1971b5 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-uniswap-interface/build.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Build the uniswap-interface image +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh + +# See: https://stackoverflow.com/a/246128/1701505 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/uniswap-interface:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/uniswap-interface diff --git a/stack_orchestrator/data/container-build/cerc-urbit-globs-host/Dockerfile b/stack_orchestrator/data/container-build/cerc-urbit-globs-host/Dockerfile new file mode 100644 index 00000000..7a3ca9b7 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-urbit-globs-host/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.13.0a2-alpine3.18 + +RUN apk --update --no-cache add alpine-sdk jq bash curl wget + +WORKDIR /app + +ENTRYPOINT [ "bash" ] diff --git a/stack_orchestrator/data/container-build/cerc-urbit-globs-host/build.sh b/stack_orchestrator/data/container-build/cerc-urbit-globs-host/build.sh new file mode 100755 index 00000000..ebd396f1 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-urbit-globs-host/build.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Build the urbit-globs-host image + +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh + +# See: https://stackoverflow.com/a/246128/1701505 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/urbit-globs-host:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${SCRIPT_DIR} diff --git a/stack_orchestrator/data/container-image-list.txt b/stack_orchestrator/data/container-image-list.txt index 41dd8b21..fd295be5 100644 --- a/stack_orchestrator/data/container-image-list.txt +++ b/stack_orchestrator/data/container-image-list.txt @@ -59,3 +59,4 @@ cerc/ponder cerc/nitro-rpc-client cerc/watcher-merkl-sushiswap-v3 cerc/watcher-sushiswap-v3 +cerc/uniswap-interface diff --git a/stack_orchestrator/data/repository-list.txt b/stack_orchestrator/data/repository-list.txt index 192a831e..cddaccce 100644 --- a/stack_orchestrator/data/repository-list.txt +++ b/stack_orchestrator/data/repository-list.txt @@ -49,3 +49,4 @@ github.com/cerc-io/mobymask-snap github.com/cerc-io/ponder github.com/cerc-io/merkl-sushiswap-v3-watcher-ts github.com/cerc-io/sushiswap-v3-watcher-ts +github.com/cerc-io/uniswap-interface diff --git a/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md b/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md new file mode 100644 index 00000000..cd6a9f3e --- /dev/null +++ b/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md @@ -0,0 +1,122 @@ +# Self-hosted Uniswap Frontend + +Instructions to setup and deploy Uniswap app on Urbit + +Build and deploy: + +- Urbit +- Uniswap app + +## Setup + +Clone required repositories: + +```bash +laconic-so --stack uniswap-urbit-app setup-repositories --pull + +# If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories mentioned below and re-run the command +``` + +Build the container images: + +```bash +laconic-so --stack uniswap-urbit-app build-containers +``` + +## Create a deployment + +First, create a spec file for the deployment, which will map the stack's ports and volumes to the host: + +```bash +laconic-so --stack uniswap-urbit-app deploy init --output uniswap-urbit-app-spec.yml +``` + +### Ports + +Edit `network` in spec file to map container ports to same ports in host + +``` +... +network: + ports: + urbit-fake-ship: + - '8080:80' + uniswap-glob-host: + - '3000:3000' + uniswap-gql-proxy: + - '4000:4000' +... +``` + +### Data volumes + +Container data volumes are bind-mounted to specified paths in the host filesystem. +The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory. The default mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`. + +--- + +Once you've made any needed changes to the spec file, create a deployment from it: + +```bash +laconic-so --stack uniswap-urbit-app deploy create --spec-file uniswap-urbit-app-spec.yml --deployment-dir uniswap-urbit-app-deployment +``` + +## Set env variables + +Inside the deployment directory, open the file `config.env` and add variable for infura key : + + ```bash + # External RPC endpoints + # https://docs.infura.io/getting-started#2-create-an-api-key + CERC_INFURA_KEY= + + # Uniswap API GQL Endpoint + # Set this to GQL proxy server endpoint for uniswap app + # (Eg. http://localhost:4000/graphql) + CERC_UNISWAP_GQL= + ``` + +## Start the stack + +Start the deployment: + +```bash +laconic-so deployment --dir uniswap-urbit-app-deployment start +``` + +* List and check the health status of all the containers using `docker ps` and wait for them to be `healthy` + +* Run the following to get login password for Urbit web interface: + + ```bash + laconic-so deployment --dir uniswap-urbit-app-deployment exec urbit-fake-ship "curl -s --data '{\"source\":{\"dojo\":\"+code\"},\"sink\":{\"stdout\":null}}' http://localhost:12321" + + # Expected output: "\n"% + ``` + +* Open the Urbit web UI at http://localhost:8080 and use the `PASSWORD` from previous step to login + +* The uniswap app is not available when starting stack for the first time. Check `urbit-fake-ship` logs to see that app has installed + ``` + laconic-so deployment --dir uniswap-urbit-app-deployment logs -f + + # Expected output: + # laconic-3ccf7ee79bdae874-urbit-fake-ship-1 | docket: fetching %http glob for %uniswap desk + # laconic-3ccf7ee79bdae874-urbit-fake-ship-1 | ">="">="Uniswap app installed + ``` + +* The uniswap app will be now visible at http://localhost:8080 + +## Clean up + +To stop all uniswap-urbit-app services running in the background, while preserving chain data: + +```bash +laconic-so deployment --dir uniswap-urbit-app-deployment stop +``` + +To stop all uniswap-urbit-app services and also delete data: + +```bash +laconic-so deployment --dir uniswap-urbit-app-deployment stop --delete-volumes +``` diff --git a/stack_orchestrator/data/stacks/uniswap-urbit-app/stack.yml b/stack_orchestrator/data/stacks/uniswap-urbit-app/stack.yml new file mode 100644 index 00000000..1077b557 --- /dev/null +++ b/stack_orchestrator/data/stacks/uniswap-urbit-app/stack.yml @@ -0,0 +1,10 @@ +version: "0.1" +name: uniswap-urbit-app +repos: + - github.com/cerc-io/uniswap-interface@laconic # TODO: Use release +containers: + - cerc/uniswap-interface + - cerc/urbit-globs-host +pods: + - uniswap-interface + - uniswap-urbit From ab0e70ed83ffcd6baef7d635ea8675dfd0973cee Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Mon, 4 Dec 2023 13:39:14 -0600 Subject: [PATCH 37/75] Change path portion of unique cluster name to point to compose file, not argv[0]. (#678) --- stack_orchestrator/deploy/deploy.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/stack_orchestrator/deploy/deploy.py b/stack_orchestrator/deploy/deploy.py index df231e74..424d112f 100644 --- a/stack_orchestrator/deploy/deploy.py +++ b/stack_orchestrator/deploy/deploy.py @@ -271,8 +271,10 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file): if cluster is None: # Create default unique, stable cluster name from confile file path and stack name if provided - # TODO: change this to the config file path - path = os.path.realpath(sys.argv[0]) + if deployment: + path = os.path.realpath(os.path.abspath(compose_dir)) + else: + path = "internal" unique_cluster_descriptor = f"{path},{stack},{include},{exclude}" if ctx.debug: print(f"pre-hash descriptor: {unique_cluster_descriptor}") From 2dd54892a1fa691b5f1b7b92313cc4d9c6d3846b Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Mon, 4 Dec 2023 21:39:16 -0600 Subject: [PATCH 38/75] Allow specifying the webapp tag explicitly (#675) --- stack_orchestrator/build/build_webapp.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/stack_orchestrator/build/build_webapp.py b/stack_orchestrator/build/build_webapp.py index ace334c4..287347eb 100644 --- a/stack_orchestrator/build/build_webapp.py +++ b/stack_orchestrator/build/build_webapp.py @@ -32,8 +32,9 @@ from stack_orchestrator.build import build_containers @click.option('--source-repo', help="directory containing the webapp to build", required=True) @click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild") @click.option("--extra-build-args", help="Supply extra arguments to build") +@click.option("--tag", help="Container tag (default: cerc/:local)") @click.pass_context -def command(ctx, base_container, source_repo, force_rebuild, extra_build_args): +def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, tag): '''build the specified webapp container''' quiet = ctx.obj.quiet @@ -70,8 +71,11 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args): container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = os.path.join(container_build_dir, base_container.replace("/", "-"), "Dockerfile.webapp") - webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1] - container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local" + if not tag: + webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1] + container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local" + else: + container_build_env["CERC_CONTAINER_BUILD_TAG"] = tag build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet, verbose, dry_run, continue_on_error) From f27da1980890dee660dc6d3b1b9c82a7d7770230 Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Tue, 5 Dec 2023 15:00:03 +0530 Subject: [PATCH 39/75] Use IPFS for hosting glob files for Urbit (#677) * Use IPFS for hosting glob files for Urbit * Add env configuration for IPFS endpoints to instructions * Make ship pier dir configurable in remote deployment script * Update remote deployment script to accept glob hash arg --- .../docker-compose-uniswap-interface.yml | 19 --------------- .../compose/docker-compose-uniswap-urbit.yml | 20 ++++++++++++++++ .../uniswap-interface/deploy-uniswap-app.sh | 21 +++++++++++------ .../uniswap-interface/host-uniswap-glob.sh | 23 ------------------- .../uniswap-interface/install-uniswap-app.sh | 19 +++++++-------- .../remote-deploy-uniswap.sh | 15 +++++++----- .../data/stacks/uniswap-urbit-app/README.md | 15 ++++++++++-- 7 files changed, 66 insertions(+), 66 deletions(-) delete mode 100755 stack_orchestrator/data/config/uniswap-interface/host-uniswap-glob.sh diff --git a/stack_orchestrator/data/compose/docker-compose-uniswap-interface.yml b/stack_orchestrator/data/compose/docker-compose-uniswap-interface.yml index f6a5c53f..6d021961 100644 --- a/stack_orchestrator/data/compose/docker-compose-uniswap-interface.yml +++ b/stack_orchestrator/data/compose/docker-compose-uniswap-interface.yml @@ -12,25 +12,6 @@ services: - app_builds:/app-builds - ../config/uniswap-interface/build-app.sh:/app/build-app.sh - uniswap-glob-host: - image: cerc/urbit-globs-host:local - restart: unless-stopped - depends_on: - uniswap-interface: - condition: service_completed_successfully - command: ["./host-uniswap-glob.sh"] - volumes: - - app_globs:/app-globs - - ../config/uniswap-interface/host-uniswap-glob.sh:/app/host-uniswap-glob.sh - ports: - - "3000" - healthcheck: - test: ["CMD", "nc", "-v", "localhost", "3000"] - interval: 20s - timeout: 5s - retries: 15 - start_period: 10s - uniswap-gql-proxy: image: cerc/uniswap-interface:local restart: on-failure diff --git a/stack_orchestrator/data/compose/docker-compose-uniswap-urbit.yml b/stack_orchestrator/data/compose/docker-compose-uniswap-urbit.yml index ae0b3709..31fa99bf 100644 --- a/stack_orchestrator/data/compose/docker-compose-uniswap-urbit.yml +++ b/stack_orchestrator/data/compose/docker-compose-uniswap-urbit.yml @@ -4,6 +4,9 @@ services: urbit-fake-ship: restart: unless-stopped image: tloncorp/vere + environment: + CERC_IPFS_GLOB_HOST_ENDPOINT: ${CERC_IPFS_GLOB_HOST_ENDPOINT:-http://ipfs-glob-host:5001} + CERC_IPFS_SERVER_ENDPOINT: ${CERC_IPFS_SERVER_ENDPOINT:-http://ipfs-glob-host:8080} entrypoint: ["bash", "-c", "./run-urbit-ship.sh && ./deploy-uniswap-app.sh && tail -f /dev/null"] volumes: - urbit_data:/urbit @@ -20,7 +23,24 @@ services: retries: 15 start_period: 10s + ipfs-glob-host: + image: ipfs/kubo:master-2023-02-20-714a968 + volumes: + - ipfs-import:/import + - ipfs-data:/data/ipfs + ports: + - "8080" + - "5001" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "5001"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 10s + volumes: urbit_data: app_builds: app_globs: + ipfs-import: + ipfs-data: diff --git a/stack_orchestrator/data/config/uniswap-interface/deploy-uniswap-app.sh b/stack_orchestrator/data/config/uniswap-interface/deploy-uniswap-app.sh index 6c147083..f07a205b 100755 --- a/stack_orchestrator/data/config/uniswap-interface/deploy-uniswap-app.sh +++ b/stack_orchestrator/data/config/uniswap-interface/deploy-uniswap-app.sh @@ -4,6 +4,11 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi +echo "Using IPFS endpoint ${CERC_IPFS_GLOB_HOST_ENDPOINT} for hosting globs" +echo "Using IPFS server endpoint ${CERC_IPFS_SERVER_ENDPOINT} for reading glob files" +ipfs_host_endpoint=${CERC_IPFS_GLOB_HOST_ENDPOINT} +ipfs_server_endpoint=${CERC_IPFS_SERVER_ENDPOINT} + uniswap_app_build='/app-builds/uniswap/build' uniswap_desk_dir='/urbit/zod/uniswap' @@ -96,15 +101,17 @@ rm "${uniswap_desk_dir}/desk.ship" hood "commit %uniswap" dojo "-landscape!make-glob %uniswap /build" -echo "Copying over glob file to mounted volume" -mkdir -p /app-globs/uniswap -cp /urbit/zod/.urb/put/* /app-globs/uniswap/ - glob_file=$(ls -1 -c zod/.urb/put | head -1) -echo "Glob filename: ${glob_file}" +echo "Created glob file: ${glob_file}" + +upload_response=$(curl -X POST -F file=@./zod/.urb/put/${glob_file} ${ipfs_host_endpoint}/api/v0/add) +glob_cid=$(echo "$upload_response" | grep -o '"Hash":"[^"]*' | sed 's/"Hash":"//') + +echo "Glob file uploaded to IFPS:" +echo "{ cid: ${glob_cid}, filename: ${glob_file} }" # Curl and wait for the glob to be hosted -glob_url="http://uniswap-glob-host:3000/${glob_file}" +glob_url="${ipfs_server_endpoint}/ipfs/${glob_cid}?filename=${glob_file}" echo "Checking if glob file hosted at ${glob_url}" while true; do @@ -128,7 +135,7 @@ cat << EOF > "${uniswap_desk_dir}/desk.docket-0" color+0xcd.75df image+'https://logowik.com/content/uploads/images/uniswap-uni7403.jpg' base+'uniswap' - glob-http+['http://uniswap-glob-host:3000/${glob_file}' ${glob_hash}] + glob-http+['${glob_url}' ${glob_hash}] version+[0 0 1] website+'https://uniswap.org/' license+'MIT' diff --git a/stack_orchestrator/data/config/uniswap-interface/host-uniswap-glob.sh b/stack_orchestrator/data/config/uniswap-interface/host-uniswap-glob.sh deleted file mode 100755 index 37605794..00000000 --- a/stack_orchestrator/data/config/uniswap-interface/host-uniswap-glob.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -set -e -if [ -n "$CERC_SCRIPT_DEBUG" ]; then - set -x -fi - -# Use config from mounted volume (when running web-app along with watcher stack) -echo "Waiting for uniswap app glob" -while [ ! -d /app-globs/uniswap ]; do - echo "Glob directory not found, retrying in 5 seconds..." - sleep 5 -done - - -# Copy to a new globs directory -mkdir -p globs -cp -r /app-globs/uniswap/* ./globs - -# Serve the glob file -cd globs -echo "Hosting glob file at port 3000" -python3 -m http.server 3000 --bind 0.0.0.0 diff --git a/stack_orchestrator/data/config/uniswap-interface/install-uniswap-app.sh b/stack_orchestrator/data/config/uniswap-interface/install-uniswap-app.sh index 679bc27e..2463e1c7 100755 --- a/stack_orchestrator/data/config/uniswap-interface/install-uniswap-app.sh +++ b/stack_orchestrator/data/config/uniswap-interface/install-uniswap-app.sh @@ -1,21 +1,22 @@ #!/bin/bash -# $1: Glob file URL (eg. https://xyz.com/glob-abcd.glob) -# $2: Uniswap desk dir (default: ./zod/uniswap) +# $1: Glob file URL (eg. https://xyz.com/glob-0vabcd.glob) +# $2: Glob file hash (eg. 0vabcd) +# $3: Urbit ship's pier dir (default: ./zod) -if [ -z "$1" ]; then - echo "Glob file URL arg not provided" +if [ "$#" -lt 2 ]; then + echo "Insufficient arguments" exit 0 fi glob_url=$1 -glob_file=$(basename "$glob_url") -glob_hash=$(echo "$glob_file" | sed "s/glob-\([a-z0-9\.]*\).glob/\1/") -echo "Using glob file ${glob_file}" +glob_hash=$2 +echo "Using glob file from ${glob_url} with hash ${glob_hash}" +# Default pier dir: ./zod # Default desk dir: ./zod/uniswap -uniswap_desk_dir="${2:-./zod/uniswap}" - +pier_dir="${3:-./zod}" +uniswap_desk_dir="${pier_dir}/uniswap" echo "Using ${uniswap_desk_dir} as the Uniswap desk dir path" # Fire curl requests to perform operations on the ship diff --git a/stack_orchestrator/data/config/uniswap-interface/remote-deploy-uniswap.sh b/stack_orchestrator/data/config/uniswap-interface/remote-deploy-uniswap.sh index 528151e9..31f03d72 100755 --- a/stack_orchestrator/data/config/uniswap-interface/remote-deploy-uniswap.sh +++ b/stack_orchestrator/data/config/uniswap-interface/remote-deploy-uniswap.sh @@ -1,18 +1,21 @@ #!/bin/bash # $1: Remote user host -# $2: Path to run the app installation in (where urbit ship dir is located) -# $3: Glob file URL (eg. https://xyz.com/glob-abcd.glob) +# $2: Remote Urbit ship's pier dir path (eg. /home/user/zod) +# $3: Glob file URL (eg. https://xyz.com/glob-0vabcd.glob) +# $4: Glob file hash (eg. 0vabcd) -if [ "$#" -ne 3 ]; then - echo "Usage: $0 " +if [ "$#" -ne 4 ]; then + echo "Incorrect number of arguments" + echo "Usage: $0 " exit 1 fi remote_user_host="$1" -remote_folder="$2" +remote_pier_folder="$2" glob_url="$3" +glob_hash="$4" installation_script="./install-uniswap-app.sh" -ssh "$remote_user_host" "cd $remote_folder && bash -s $glob_url" < "$installation_script" +ssh "$remote_user_host" "bash -s $glob_url $glob_hash $remote_pier_folder" < "$installation_script" diff --git a/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md b/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md index cd6a9f3e..55a37338 100644 --- a/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md +++ b/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md @@ -41,10 +41,11 @@ network: ports: urbit-fake-ship: - '8080:80' - uniswap-glob-host: - - '3000:3000' uniswap-gql-proxy: - '4000:4000' + ipfs-glob-host: + - '8081:8080' + - '5001:5001' ... ``` @@ -74,6 +75,16 @@ Inside the deployment directory, open the file `config.env` and add variable for # Set this to GQL proxy server endpoint for uniswap app # (Eg. http://localhost:4000/graphql) CERC_UNISWAP_GQL= + + # Optional IPFS endpoints: + + # IFPS endpoint to host the glob file on + # (Default: http://ipfs-glob-host:5001 pointing to in-stack IPFS node) + CERC_IPFS_GLOB_HOST_ENDPOINT= + + # IFPS endpoint to fetch the glob file from + # (Default: http://ipfs-glob-host:8080 pointing to in-stack IPFS node) + CERC_IPFS_SERVER_ENDPOINT= ``` ## Start the stack From 6bef0c5b2f83c5e6710b0efb5ad685d6015ffa1f Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Wed, 6 Dec 2023 10:41:10 +0530 Subject: [PATCH 40/75] Separate out GQL proxy server from uniswap-urbit stack (#681) * Separate out uniswap gql proxy in a stack * Use proxy server from watcher-ts * Add a flag to enable/disable the proxy server * Update env configuratoin for uniswap urbit app stack * Update stack file for uniswap urbit app stack * Fix env variables in instructions --- .../compose/docker-compose-proxy-server.yml | 22 ++++++ .../docker-compose-uniswap-interface.yml | 13 --- .../data/config/proxy-server/run.sh | 9 +++ .../data/stacks/proxy-server/README.md | 79 +++++++++++++++++++ .../data/stacks/proxy-server/stack.yml | 8 ++ .../data/stacks/uniswap-urbit-app/README.md | 28 +++++-- .../data/stacks/uniswap-urbit-app/stack.yml | 3 + 7 files changed, 144 insertions(+), 18 deletions(-) create mode 100644 stack_orchestrator/data/compose/docker-compose-proxy-server.yml create mode 100755 stack_orchestrator/data/config/proxy-server/run.sh create mode 100644 stack_orchestrator/data/stacks/proxy-server/README.md create mode 100644 stack_orchestrator/data/stacks/proxy-server/stack.yml diff --git a/stack_orchestrator/data/compose/docker-compose-proxy-server.yml b/stack_orchestrator/data/compose/docker-compose-proxy-server.yml new file mode 100644 index 00000000..607e8d23 --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-proxy-server.yml @@ -0,0 +1,22 @@ +version: "3.2" + +services: + proxy-server: + image: cerc/watcher-ts:local + restart: on-failure + working_dir: /app/packages/cli + environment: + ENABLE_PROXY: ${ENABLE_PROXY:-true} + PROXY_UPSTREAM: ${CERC_PROXY_UPSTREAM} + PROXY_ORIGIN_HEADER: ${CERC_PROXY_ORIGIN_HEADER} + command: ["sh", "-c", "./run.sh"] + volumes: + - ../config/proxy-server/run.sh:/app/packages/cli/run.sh + ports: + - "4000" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "4000"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 10s diff --git a/stack_orchestrator/data/compose/docker-compose-uniswap-interface.yml b/stack_orchestrator/data/compose/docker-compose-uniswap-interface.yml index 6d021961..85b71af2 100644 --- a/stack_orchestrator/data/compose/docker-compose-uniswap-interface.yml +++ b/stack_orchestrator/data/compose/docker-compose-uniswap-interface.yml @@ -12,19 +12,6 @@ services: - app_builds:/app-builds - ../config/uniswap-interface/build-app.sh:/app/build-app.sh - uniswap-gql-proxy: - image: cerc/uniswap-interface:local - restart: on-failure - command: ["bash", "-c", "yarn proxy-gql"] - ports: - - "4000" - healthcheck: - test: ["CMD", "nc", "-v", "localhost", "4000"] - interval: 20s - timeout: 5s - retries: 15 - start_period: 10s - volumes: app_builds: app_globs: diff --git a/stack_orchestrator/data/config/proxy-server/run.sh b/stack_orchestrator/data/config/proxy-server/run.sh new file mode 100755 index 00000000..9e8dc7f5 --- /dev/null +++ b/stack_orchestrator/data/config/proxy-server/run.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +if [ "$ENABLE_PROXY" = "true" ]; then + echo "Proxy server enabled" + yarn proxy +else + echo "Proxy server disabled, exiting" + exit 0 +fi diff --git a/stack_orchestrator/data/stacks/proxy-server/README.md b/stack_orchestrator/data/stacks/proxy-server/README.md new file mode 100644 index 00000000..f0ccdb0f --- /dev/null +++ b/stack_orchestrator/data/stacks/proxy-server/README.md @@ -0,0 +1,79 @@ +# Proxy Server + +Instructions to setup and deploy a HTTP proxy server + +## Setup + +Clone required repository: + +```bash +laconic-so --stack proxy-server setup-repositories --pull + +# If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories mentioned below and re-run the command +``` + +Build the container image: + +```bash +laconic-so --stack proxy-server build-containers +``` + +## Create a deployment + +* First, create a spec file for the deployment, which will allow mapping the stack's ports and volumes to the host: + + ```bash + laconic-so --stack proxy-server deploy init --output proxy-server-spec.yml + ``` + +* Edit `network` in spec file to map container ports to same ports in host: + + ```yml + ... + network: + ports: + proxy-server: + - '4000:4000' + ... + ``` + +* Once you've made any needed changes to the spec file, create a deployment from it: + + ```bash + laconic-so --stack proxy-server deploy create --spec-file proxy-server-spec.yml --deployment-dir proxy-server-deployment + ``` + +* Inside the deployment directory, open the file `config.env` and set the following env variables: + + ```bash + # Whether to run the proxy server (Optional) (Default: true) + ENABLE_PROXY= + + # Upstream endpoint + # (Eg. https://api.example.org) + CERC_PROXY_UPSTREAM= + + # Origin header to be used (Optional) + # (Eg. https://app.example.org) + CERC_PROXY_ORIGIN_HEADER= + ``` + +## Start the stack + +Start the deployment: + +```bash +laconic-so deployment --dir proxy-server-deployment start +``` + +* List and check the health status of the container using `docker ps` + +* The proxy server will now be listening at http://localhost:4000 + +## Clean up + +To stop the service running in background: + +```bash +laconic-so deployment --dir proxy-server-deployment stop +``` diff --git a/stack_orchestrator/data/stacks/proxy-server/stack.yml b/stack_orchestrator/data/stacks/proxy-server/stack.yml new file mode 100644 index 00000000..313a7f91 --- /dev/null +++ b/stack_orchestrator/data/stacks/proxy-server/stack.yml @@ -0,0 +1,8 @@ +version: "0.1" +name: proxy-server +repos: + - github.com/cerc-io/watcher-ts@v0.2.78 +containers: + - cerc/watcher-ts +pods: + - proxy-server diff --git a/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md b/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md index 55a37338..7499f5fc 100644 --- a/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md +++ b/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md @@ -41,7 +41,7 @@ network: ports: urbit-fake-ship: - '8080:80' - uniswap-gql-proxy: + proxy-server: - '4000:4000' ipfs-glob-host: - '8081:8080' @@ -64,7 +64,7 @@ laconic-so --stack uniswap-urbit-app deploy create --spec-file uniswap-urbit-app ## Set env variables -Inside the deployment directory, open the file `config.env` and add variable for infura key : +Inside the deployment directory, open the file `config.env` and set the following env variables: ```bash # External RPC endpoints @@ -73,10 +73,28 @@ Inside the deployment directory, open the file `config.env` and add variable for # Uniswap API GQL Endpoint # Set this to GQL proxy server endpoint for uniswap app - # (Eg. http://localhost:4000/graphql) + # (Eg. http://localhost:4000/v1/graphql) + # (Eg. https://abc.xyz.com/v1/graphql) CERC_UNISWAP_GQL= - # Optional IPFS endpoints: + # Optional + + # Whether to run the proxy GQL server + # (Disable only if proxy not required to be run) (Default: true) + ENABLE_PROXY= + + # Proxy server configuration + # Used only if proxy is enabled + + # Upstream API URL + # (Eg. https://api.example.org) + CERC_PROXY_UPSTREAM=https://api.uniswap.org + + # Origin header to be used in the proxy + # (Eg. https://app.example.org) + CERC_PROXY_ORIGIN_HEADER=https://app.uniswap.org + + # IPFS configuration # IFPS endpoint to host the glob file on # (Default: http://ipfs-glob-host:5001 pointing to in-stack IPFS node) @@ -120,7 +138,7 @@ laconic-so deployment --dir uniswap-urbit-app-deployment start ## Clean up -To stop all uniswap-urbit-app services running in the background, while preserving chain data: +To stop all uniswap-urbit-app services running in the background, while preserving data: ```bash laconic-so deployment --dir uniswap-urbit-app-deployment stop diff --git a/stack_orchestrator/data/stacks/uniswap-urbit-app/stack.yml b/stack_orchestrator/data/stacks/uniswap-urbit-app/stack.yml index 1077b557..3f77098f 100644 --- a/stack_orchestrator/data/stacks/uniswap-urbit-app/stack.yml +++ b/stack_orchestrator/data/stacks/uniswap-urbit-app/stack.yml @@ -2,9 +2,12 @@ version: "0.1" name: uniswap-urbit-app repos: - github.com/cerc-io/uniswap-interface@laconic # TODO: Use release + - github.com/cerc-io/watcher-ts@v0.2.78 containers: - cerc/uniswap-interface + - cerc/watcher-ts - cerc/urbit-globs-host pods: - uniswap-interface + - proxy-server - uniswap-urbit From 15faed00de8e510b4bdd5150eef1c26248ae3d19 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 5 Dec 2023 22:56:58 -0700 Subject: [PATCH 41/75] Generate a unique deployment id for each deployment (#680) * Move cluster name generation into a function * Generate a unique deployment id for each deployment --- stack_orchestrator/constants.py | 4 +++ stack_orchestrator/deploy/deploy.py | 36 ++++++++++++------- stack_orchestrator/deploy/deployment.py | 2 +- .../deploy/deployment_context.py | 28 +++++++++++++-- .../deploy/deployment_create.py | 18 +++++++--- 5 files changed, 68 insertions(+), 20 deletions(-) diff --git a/stack_orchestrator/constants.py b/stack_orchestrator/constants.py index 54cfe355..596b0c1b 100644 --- a/stack_orchestrator/constants.py +++ b/stack_orchestrator/constants.py @@ -13,12 +13,16 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +cluster_name_prefix = "laconic-" stack_file_name = "stack.yml" spec_file_name = "spec.yml" config_file_name = "config.env" +deployment_file_name = "deployment.yml" +compose_dir_name = "compose" compose_deploy_type = "compose" k8s_kind_deploy_type = "k8s-kind" k8s_deploy_type = "k8s" +cluster_id_key = "cluster-id" kube_config_key = "kube-config" deploy_to_key = "deploy-to" network_key = "network" diff --git a/stack_orchestrator/deploy/deploy.py b/stack_orchestrator/deploy/deploy.py index 424d112f..d1b64743 100644 --- a/stack_orchestrator/deploy/deploy.py +++ b/stack_orchestrator/deploy/deploy.py @@ -24,6 +24,8 @@ from importlib import resources import subprocess import click from pathlib import Path +from stack_orchestrator import constants +from stack_orchestrator.opts import opts from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, global_options2, get_dev_root_path from stack_orchestrator.deploy.deployer import Deployer, DeployerException from stack_orchestrator.deploy.deployer_factory import getDeployer @@ -70,6 +72,9 @@ def create_deploy_context( cluster, env_file, deploy_to) -> DeployCommandContext: + # Extract the cluster name from the deployment, if we have one + if deployment_context and cluster is None: + cluster = deployment_context.get_cluster_id() cluster_context = _make_cluster_context(global_context, stack, include, exclude, cluster, env_file) deployer = getDeployer(deploy_to, deployment_context, compose_files=cluster_context.compose_files, compose_project_name=cluster_context.cluster, @@ -253,6 +258,22 @@ def _make_runtime_env(ctx): return container_exec_env +def _make_default_cluster_name(deployment, compose_dir, stack, include, exclude): + # Create default unique, stable cluster name from confile file path and stack name if provided + if deployment: + path = os.path.realpath(os.path.abspath(compose_dir)) + else: + path = "internal" + unique_cluster_descriptor = f"{path},{stack},{include},{exclude}" + if opts.o.debug: + print(f"pre-hash descriptor: {unique_cluster_descriptor}") + hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest()[:16] + cluster = f"{constants.cluster_name_prefix}{hash}" + if opts.o.debug: + print(f"Using cluster name: {cluster}") + return cluster + + # stack has to be either PathLike pointing to a stack yml file, or a string with the name of a known stack def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file): @@ -270,18 +291,9 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file): compose_dir = Path(__file__).absolute().parent.parent.joinpath("data", "compose") if cluster is None: - # Create default unique, stable cluster name from confile file path and stack name if provided - if deployment: - path = os.path.realpath(os.path.abspath(compose_dir)) - else: - path = "internal" - unique_cluster_descriptor = f"{path},{stack},{include},{exclude}" - if ctx.debug: - print(f"pre-hash descriptor: {unique_cluster_descriptor}") - hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest()[:16] - cluster = f"laconic-{hash}" - if ctx.verbose: - print(f"Using cluster name: {cluster}") + cluster = _make_default_cluster_name(deployment, compose_dir, stack, include, exclude) + else: + _make_default_cluster_name(deployment, compose_dir, stack, include, exclude) # See: https://stackoverflow.com/a/20885799/1701505 from stack_orchestrator import data diff --git a/stack_orchestrator/deploy/deployment.py b/stack_orchestrator/deploy/deployment.py index 8d74a62d..cc70519e 100644 --- a/stack_orchestrator/deploy/deployment.py +++ b/stack_orchestrator/deploy/deployment.py @@ -52,7 +52,7 @@ def make_deploy_context(ctx) -> DeployCommandContext: context: DeploymentContext = ctx.obj stack_file_path = context.get_stack_file() env_file = context.get_env_file() - cluster_name = context.get_cluster_name() + cluster_name = context.get_cluster_id() if constants.deploy_to_key in context.spec.obj: deployment_type = context.spec.obj[constants.deploy_to_key] else: diff --git a/stack_orchestrator/deploy/deployment_context.py b/stack_orchestrator/deploy/deployment_context.py index cbee4151..27e32812 100644 --- a/stack_orchestrator/deploy/deployment_context.py +++ b/stack_orchestrator/deploy/deployment_context.py @@ -14,15 +14,19 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import hashlib +import os from pathlib import Path from stack_orchestrator import constants +from stack_orchestrator.util import get_yaml from stack_orchestrator.deploy.stack import Stack from stack_orchestrator.deploy.spec import Spec class DeploymentContext: deployment_dir: Path + id: str spec: Spec stack: Stack @@ -35,9 +39,14 @@ class DeploymentContext: def get_env_file(self): return self.deployment_dir.joinpath(constants.config_file_name) - # TODO: implement me - def get_cluster_name(self): - return None + def get_deployment_file(self): + return self.deployment_dir.joinpath(constants.deployment_file_name) + + def get_compose_dir(self): + return self.deployment_dir.joinpath(constants.compose_dir_name) + + def get_cluster_id(self): + return self.id def init(self, dir): self.deployment_dir = dir @@ -45,3 +54,16 @@ class DeploymentContext: self.spec.init_from_file(self.get_spec_file()) self.stack = Stack(self.spec.obj["stack"]) self.stack.init_from_file(self.get_stack_file()) + deployment_file_path = self.get_deployment_file() + if deployment_file_path.exists(): + with deployment_file_path: + obj = get_yaml().load(open(deployment_file_path, "r")) + self.id = obj[constants.cluster_id_key] + # Handle the case of a legacy deployment with no file + # Code below is intended to match the output from _make_default_cluster_name() + # TODO: remove when we no longer need to support legacy deployments + else: + path = os.path.realpath(os.path.abspath(self.get_compose_dir())) + unique_cluster_descriptor = f"{path},{self.get_stack_file()},None,None" + hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest()[:16] + self.id = f"{constants.cluster_name_prefix}{hash}" diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py index 88ce0b2a..9eaea30c 100644 --- a/stack_orchestrator/deploy/deployment_create.py +++ b/stack_orchestrator/deploy/deployment_create.py @@ -20,6 +20,7 @@ from pathlib import Path from typing import List import random from shutil import copy, copyfile, copytree +from secrets import token_hex import sys from stack_orchestrator import constants from stack_orchestrator.opts import opts @@ -276,7 +277,7 @@ def init(ctx, config, config_file, kube_config, image_registry, output, map_port # call it from other commands, bypassing the click decoration stuff def init_operation(deploy_command_context, stack, deployer_type, config, config_file, kube_config, image_registry, output, map_ports_to_host): - yaml = get_yaml() + default_spec_file_content = call_stack_deploy_init(deploy_command_context) spec_file_content = {"stack": stack, constants.deploy_to_key: deployer_type} if deployer_type == "k8s": @@ -311,8 +312,6 @@ def init_operation(deploy_command_context, stack, deployer_type, config, new_config = config_file_variables merged_config = {**new_config, **orig_config} spec_file_content.update({"config": merged_config}) - if opts.o.debug: - print(f"Creating spec file for stack: {stack} with content: {spec_file_content}") ports = _get_mapped_ports(stack, map_ports_to_host) spec_file_content.update({"network": {"ports": ports}}) @@ -324,8 +323,11 @@ def init_operation(deploy_command_context, stack, deployer_type, config, volume_descriptors[named_volume] = f"./data/{named_volume}" spec_file_content["volumes"] = volume_descriptors + if opts.o.debug: + print(f"Creating spec file for stack: {stack} with content: {spec_file_content}") + with open(output, "w") as output_file: - yaml.dump(spec_file_content, output_file) + get_yaml().dump(spec_file_content, output_file) def _write_config_file(spec_file: Path, config_env_file: Path): @@ -351,6 +353,13 @@ def _copy_files_to_directory(file_paths: List[Path], directory: Path): copy(path, os.path.join(directory, os.path.basename(path))) +def _create_deployment_file(deployment_dir: Path): + deployment_file_path = deployment_dir.joinpath(constants.deployment_file_name) + cluster = f"{constants.cluster_name_prefix}{token_hex(8)}" + with open(deployment_file_path, "w") as output_file: + output_file.write(f"{constants.cluster_id_key}: {cluster}\n") + + @click.command() @click.option("--spec-file", required=True, help="Spec file to use to create this deployment") @click.option("--deployment-dir", help="Create deployment files in this directory") @@ -383,6 +392,7 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw # Copy spec file and the stack file into the deployment dir copyfile(spec_file, deployment_dir_path.joinpath(constants.spec_file_name)) copyfile(stack_file, deployment_dir_path.joinpath(os.path.basename(stack_file))) + _create_deployment_file(deployment_dir_path) # Copy any config varibles from the spec file into an env file suitable for compose _write_config_file(spec_file, deployment_dir_path.joinpath(constants.config_file_name)) # Copy any k8s config file into the deployment dir From 077ea80c7030b000bbe0775ff944ab977c4c18bc Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 6 Dec 2023 10:27:47 -0600 Subject: [PATCH 42/75] Add `deployment status` command and fix k8s output for `deployment ps` (#679) --- .../deploy/compose/deploy_docker.py | 7 ++ stack_orchestrator/deploy/deploy.py | 8 ++ stack_orchestrator/deploy/deployer.py | 4 + stack_orchestrator/deploy/deployment.py | 5 +- stack_orchestrator/deploy/k8s/deploy_k8s.py | 90 +++++++++++++++++-- 5 files changed, 104 insertions(+), 10 deletions(-) diff --git a/stack_orchestrator/deploy/compose/deploy_docker.py b/stack_orchestrator/deploy/compose/deploy_docker.py index 04f24df5..d34d1e6f 100644 --- a/stack_orchestrator/deploy/compose/deploy_docker.py +++ b/stack_orchestrator/deploy/compose/deploy_docker.py @@ -40,6 +40,13 @@ class DockerDeployer(Deployer): except DockerException as e: raise DeployerException(e) + def status(self): + try: + for p in self.docker.compose.ps(): + print(f"{p.name}\t{p.state.status}") + except DockerException as e: + raise DeployerException(e) + def ps(self): try: return self.docker.compose.ps() diff --git a/stack_orchestrator/deploy/deploy.py b/stack_orchestrator/deploy/deploy.py index d1b64743..da96a500 100644 --- a/stack_orchestrator/deploy/deploy.py +++ b/stack_orchestrator/deploy/deploy.py @@ -112,6 +112,14 @@ def down_operation(ctx, delete_volumes, extra_args_list): ctx.obj.deployer.down(timeout=timeout_arg, volumes=delete_volumes) +def status_operation(ctx): + global_context = ctx.parent.parent.obj + if not global_context.dry_run: + if global_context.verbose: + print("Running compose status") + ctx.obj.deployer.status() + + def ps_operation(ctx): global_context = ctx.parent.parent.obj if not global_context.dry_run: diff --git a/stack_orchestrator/deploy/deployer.py b/stack_orchestrator/deploy/deployer.py index 984945ed..2806044b 100644 --- a/stack_orchestrator/deploy/deployer.py +++ b/stack_orchestrator/deploy/deployer.py @@ -31,6 +31,10 @@ class Deployer(ABC): def ps(self): pass + @abstractmethod + def status(self): + pass + @abstractmethod def port(self, service, private_port): pass diff --git a/stack_orchestrator/deploy/deployment.py b/stack_orchestrator/deploy/deployment.py index cc70519e..366a83f6 100644 --- a/stack_orchestrator/deploy/deployment.py +++ b/stack_orchestrator/deploy/deployment.py @@ -18,7 +18,7 @@ from pathlib import Path import sys from stack_orchestrator import constants from stack_orchestrator.deploy.images import push_images_operation -from stack_orchestrator.deploy.deploy import up_operation, down_operation, ps_operation, port_operation +from stack_orchestrator.deploy.deploy import up_operation, down_operation, ps_operation, port_operation, status_operation from stack_orchestrator.deploy.deploy import exec_operation, logs_operation, create_deploy_context from stack_orchestrator.deploy.deploy_types import DeployCommandContext from stack_orchestrator.deploy.deployment_context import DeploymentContext @@ -147,4 +147,5 @@ def logs(ctx, tail, follow, extra_args): @command.command() @click.pass_context def status(ctx): - print(f"Context: {ctx.parent.obj}") + ctx.obj = make_deploy_context(ctx) + status_operation(ctx) diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index c84aa34a..95131966 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -26,6 +26,12 @@ from stack_orchestrator.deploy.deployment_context import DeploymentContext from stack_orchestrator.util import error_exit +class AttrDict(dict): + def __init__(self, *args, **kwargs): + super(AttrDict, self).__init__(*args, **kwargs) + self.__dict__ = self + + def _check_delete_exception(e: client.exceptions.ApiException): if e.status == 404: if opts.o.debug: @@ -42,7 +48,7 @@ class K8sDeployer(Deployer): networking_api: client.NetworkingV1Api k8s_namespace: str = "default" kind_cluster_name: str - cluster_info : ClusterInfo + cluster_info: ClusterInfo deployment_dir: Path deployment_context: DeploymentContext @@ -72,6 +78,7 @@ class K8sDeployer(Deployer): self.core_api = client.CoreV1Api() self.networking_api = client.NetworkingV1Api() self.apps_api = client.AppsV1Api() + self.custom_obj_api = client.CustomObjectsApi() def up(self, detach, services): @@ -202,15 +209,82 @@ class K8sDeployer(Deployer): # Destroy the kind cluster destroy_cluster(self.kind_cluster_name) - def ps(self): + def status(self): self.connect_api() # Call whatever API we need to get the running container list - ret = self.core_api.list_pod_for_all_namespaces(watch=False) - if ret.items: - for i in ret.items: - print("%s\t%s\t%s" % (i.status.pod_ip, i.metadata.namespace, i.metadata.name)) - ret = self.core_api.list_node(pretty=True, watch=False) - return [] + all_pods = self.core_api.list_pod_for_all_namespaces(watch=False) + pods = [] + + if all_pods.items: + for p in all_pods.items: + if self.cluster_info.app_name in p.metadata.name: + pods.append(p) + + if not pods: + return + + hostname = "?" + ip = "?" + tls = "?" + try: + ingress = self.networking_api.read_namespaced_ingress(namespace=self.k8s_namespace, + name=self.cluster_info.get_ingress().metadata.name) + + cert = self.custom_obj_api.get_namespaced_custom_object( + group="cert-manager.io", + version="v1", + namespace=self.k8s_namespace, + plural="certificates", + name=ingress.spec.tls[0].secret_name + ) + + hostname = ingress.spec.tls[0].hosts[0] + ip = ingress.status.load_balancer.ingress[0].ip + tls = "notBefore: %s, notAfter: %s" % (cert["status"]["notBefore"], cert["status"]["notAfter"]) + except: # noqa: E722 + pass + + print("Ingress:") + print("\tHostname:", hostname) + print("\tIP:", ip) + print("\tTLS:", tls) + print("") + print("Pods:") + + for p in pods: + if p.metadata.deletion_timestamp: + print(f"\t{p.metadata.namespace}/{p.metadata.name}: Terminating ({p.metadata.deletion_timestamp})") + else: + print(f"\t{p.metadata.namespace}/{p.metadata.name}: Running ({p.metadata.creation_timestamp})") + + def ps(self): + self.connect_api() + pods = self.core_api.list_pod_for_all_namespaces(watch=False) + + ret = [] + + for p in pods.items: + if self.cluster_info.app_name in p.metadata.name: + pod_ip = p.status.pod_ip + ports = AttrDict() + for c in p.spec.containers: + if c.ports: + for prt in c.ports: + ports[str(prt.container_port)] = [AttrDict({ + "HostIp": pod_ip, + "HostPort": prt.container_port + })] + + ret.append(AttrDict({ + "id": f"{p.metadata.namespace}/{p.metadata.name}", + "name": p.metadata.name, + "namespace": p.metadata.namespace, + "network_settings": AttrDict({ + "ports": ports + }) + })) + + return ret def port(self, service, private_port): # Since we handle the port mapping, need to figure out where this comes from From f9eb5a4ba87de1639dff7bf564537753e9897e17 Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Fri, 8 Dec 2023 09:35:00 +0530 Subject: [PATCH 43/75] Refactor to make Urbit setup generic (#682) * Refactor to make Urbit app deployment script generic * Rename urbit pod and update instructions * Add a flag to allow skipping app installation on Urbit * Make remote Urbit app deployment scripts generic * Move remote deployment scripts to urbit fixturenet * Update and use existing kubo pod for Urbit glob hosting --- .../docker-compose-fixturenet-urbit.yml | 36 +++++ .../data/compose/docker-compose-kubo.yml | 19 ++- .../compose/docker-compose-proxy-server.yml | 2 +- .../docker-compose-uniswap-interface.yml | 7 +- .../compose/docker-compose-uniswap-urbit.yml | 46 ------ .../config/uniswap-interface/build-app.sh | 7 +- .../uniswap-interface/deploy-uniswap-app.sh | 149 ------------------ .../uniswap-interface/install-uniswap-app.sh | 110 ------------- .../remote-deploy-uniswap.sh | 21 --- .../urbit-files/desk.docket-0 | 10 ++ .../urbit-files/mar/map.hoon | 18 +++ .../urbit-files/mar/ttf.hoon | 12 ++ .../urbit-files/mar/woff.hoon | 12 ++ .../urbit/deploy-app-to-remote-urbit.sh | 34 ++++ .../data/config/urbit/deploy-app.sh | 110 +++++++++++++ .../data/config/urbit/install-urbit-app.sh | 60 +++++++ .../data/config/urbit/run-urbit-ship.sh | 4 +- .../data/stacks/proxy-server/README.md | 2 +- .../data/stacks/uniswap-urbit-app/README.md | 21 ++- .../data/stacks/uniswap-urbit-app/stack.yml | 3 +- 20 files changed, 338 insertions(+), 345 deletions(-) create mode 100644 stack_orchestrator/data/compose/docker-compose-fixturenet-urbit.yml delete mode 100644 stack_orchestrator/data/compose/docker-compose-uniswap-urbit.yml delete mode 100755 stack_orchestrator/data/config/uniswap-interface/deploy-uniswap-app.sh delete mode 100755 stack_orchestrator/data/config/uniswap-interface/install-uniswap-app.sh delete mode 100755 stack_orchestrator/data/config/uniswap-interface/remote-deploy-uniswap.sh create mode 100644 stack_orchestrator/data/config/uniswap-interface/urbit-files/desk.docket-0 create mode 100644 stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/map.hoon create mode 100644 stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/ttf.hoon create mode 100644 stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/woff.hoon create mode 100755 stack_orchestrator/data/config/urbit/deploy-app-to-remote-urbit.sh create mode 100755 stack_orchestrator/data/config/urbit/deploy-app.sh create mode 100755 stack_orchestrator/data/config/urbit/install-urbit-app.sh diff --git a/stack_orchestrator/data/compose/docker-compose-fixturenet-urbit.yml b/stack_orchestrator/data/compose/docker-compose-fixturenet-urbit.yml new file mode 100644 index 00000000..bc64cd90 --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-fixturenet-urbit.yml @@ -0,0 +1,36 @@ +version: '3.7' + +services: + # Runs an Urbit fake ship and attempts an app installation using given data + # Uploads the app glob to given IPFS endpoint + # From urbit_app_builds volume: + # - takes app build from ${CERC_URBIT_APP}/build (waits for it to appear) + # - takes additional mark files from ${CERC_URBIT_APP}/mar + # - takes the docket file from ${CERC_URBIT_APP}/desk.docket-0 + urbit-fake-ship: + restart: unless-stopped + image: tloncorp/vere + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_URBIT_APP: ${CERC_URBIT_APP} + CERC_ENABLE_APP_INSTALL: ${CERC_ENABLE_APP_INSTALL:-true} + CERC_IPFS_GLOB_HOST_ENDPOINT: ${CERC_IPFS_GLOB_HOST_ENDPOINT:-http://ipfs:5001} + CERC_IPFS_SERVER_ENDPOINT: ${CERC_IPFS_SERVER_ENDPOINT:-http://ipfs:8080} + entrypoint: ["bash", "-c", "./run-urbit-ship.sh && ./deploy-app.sh && tail -f /dev/null"] + volumes: + - urbit_data:/urbit + - urbit_app_builds:/app-builds + - ../config/urbit/run-urbit-ship.sh:/urbit/run-urbit-ship.sh + - ../config/urbit/deploy-app.sh:/urbit/deploy-app.sh + ports: + - "80" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "80"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 10s + +volumes: + urbit_data: + urbit_app_builds: diff --git a/stack_orchestrator/data/compose/docker-compose-kubo.yml b/stack_orchestrator/data/compose/docker-compose-kubo.yml index f5f8b06e..2e4ae419 100644 --- a/stack_orchestrator/data/compose/docker-compose-kubo.yml +++ b/stack_orchestrator/data/compose/docker-compose-kubo.yml @@ -1,13 +1,24 @@ version: "3.2" + # See: https://docs.ipfs.tech/install/run-ipfs-inside-docker/#set-up services: ipfs: image: ipfs/kubo:master-2023-02-20-714a968 restart: always volumes: - - ./ipfs/import:/import - - ./ipfs/data:/data/ipfs + - ipfs-import:/import + - ipfs-data:/data/ipfs ports: - - "0.0.0.0:8080:8080" - - "0.0.0.0:4001:4001" + - "4001" + - "8080" - "0.0.0.0:5001:5001" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "5001"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 10s + +volumes: + ipfs-import: + ipfs-data: diff --git a/stack_orchestrator/data/compose/docker-compose-proxy-server.yml b/stack_orchestrator/data/compose/docker-compose-proxy-server.yml index 607e8d23..dc672e0a 100644 --- a/stack_orchestrator/data/compose/docker-compose-proxy-server.yml +++ b/stack_orchestrator/data/compose/docker-compose-proxy-server.yml @@ -6,7 +6,7 @@ services: restart: on-failure working_dir: /app/packages/cli environment: - ENABLE_PROXY: ${ENABLE_PROXY:-true} + ENABLE_PROXY: ${CERC_ENABLE_PROXY:-true} PROXY_UPSTREAM: ${CERC_PROXY_UPSTREAM} PROXY_ORIGIN_HEADER: ${CERC_PROXY_ORIGIN_HEADER} command: ["sh", "-c", "./run.sh"] diff --git a/stack_orchestrator/data/compose/docker-compose-uniswap-interface.yml b/stack_orchestrator/data/compose/docker-compose-uniswap-interface.yml index 85b71af2..334ab0cd 100644 --- a/stack_orchestrator/data/compose/docker-compose-uniswap-interface.yml +++ b/stack_orchestrator/data/compose/docker-compose-uniswap-interface.yml @@ -9,9 +9,10 @@ services: - REACT_APP_AWS_API_ENDPOINT=${CERC_UNISWAP_GQL} command: ["./build-app.sh"] volumes: - - app_builds:/app-builds - ../config/uniswap-interface/build-app.sh:/app/build-app.sh + - urbit_app_builds:/app-builds + - ../config/uniswap-interface/urbit-files/mar:/app/mar + - ../config/uniswap-interface/urbit-files/desk.docket-0:/app/desk.docket-0 volumes: - app_builds: - app_globs: + urbit_app_builds: diff --git a/stack_orchestrator/data/compose/docker-compose-uniswap-urbit.yml b/stack_orchestrator/data/compose/docker-compose-uniswap-urbit.yml deleted file mode 100644 index 31fa99bf..00000000 --- a/stack_orchestrator/data/compose/docker-compose-uniswap-urbit.yml +++ /dev/null @@ -1,46 +0,0 @@ -version: '3.7' - -services: - urbit-fake-ship: - restart: unless-stopped - image: tloncorp/vere - environment: - CERC_IPFS_GLOB_HOST_ENDPOINT: ${CERC_IPFS_GLOB_HOST_ENDPOINT:-http://ipfs-glob-host:5001} - CERC_IPFS_SERVER_ENDPOINT: ${CERC_IPFS_SERVER_ENDPOINT:-http://ipfs-glob-host:8080} - entrypoint: ["bash", "-c", "./run-urbit-ship.sh && ./deploy-uniswap-app.sh && tail -f /dev/null"] - volumes: - - urbit_data:/urbit - - app_builds:/app-builds - - app_globs:/app-globs - - ../config/urbit/run-urbit-ship.sh:/urbit/run-urbit-ship.sh - - ../config/uniswap-interface/deploy-uniswap-app.sh:/urbit/deploy-uniswap-app.sh - ports: - - "80" - healthcheck: - test: ["CMD", "nc", "-v", "localhost", "80"] - interval: 20s - timeout: 5s - retries: 15 - start_period: 10s - - ipfs-glob-host: - image: ipfs/kubo:master-2023-02-20-714a968 - volumes: - - ipfs-import:/import - - ipfs-data:/data/ipfs - ports: - - "8080" - - "5001" - healthcheck: - test: ["CMD", "nc", "-v", "localhost", "5001"] - interval: 20s - timeout: 5s - retries: 15 - start_period: 10s - -volumes: - urbit_data: - app_builds: - app_globs: - ipfs-import: - ipfs-data: diff --git a/stack_orchestrator/data/config/uniswap-interface/build-app.sh b/stack_orchestrator/data/config/uniswap-interface/build-app.sh index d3b012e6..81e306b6 100755 --- a/stack_orchestrator/data/config/uniswap-interface/build-app.sh +++ b/stack_orchestrator/data/config/uniswap-interface/build-app.sh @@ -13,6 +13,9 @@ fi yarn build -# Move build to app-builds so urbit can deploy it -mkdir /app-builds/uniswap +# Copy over build and other files to app-builds for urbit deployment +mkdir -p /app-builds/uniswap cp -r ./build /app-builds/uniswap/ + +cp -r mar /app-builds/uniswap/ +cp desk.docket-0 /app-builds/uniswap/ diff --git a/stack_orchestrator/data/config/uniswap-interface/deploy-uniswap-app.sh b/stack_orchestrator/data/config/uniswap-interface/deploy-uniswap-app.sh deleted file mode 100755 index f07a205b..00000000 --- a/stack_orchestrator/data/config/uniswap-interface/deploy-uniswap-app.sh +++ /dev/null @@ -1,149 +0,0 @@ -#!/bin/bash - -if [ -n "$CERC_SCRIPT_DEBUG" ]; then - set -x -fi - -echo "Using IPFS endpoint ${CERC_IPFS_GLOB_HOST_ENDPOINT} for hosting globs" -echo "Using IPFS server endpoint ${CERC_IPFS_SERVER_ENDPOINT} for reading glob files" -ipfs_host_endpoint=${CERC_IPFS_GLOB_HOST_ENDPOINT} -ipfs_server_endpoint=${CERC_IPFS_SERVER_ENDPOINT} - -uniswap_app_build='/app-builds/uniswap/build' -uniswap_desk_dir='/urbit/zod/uniswap' - -if [ -d ${uniswap_desk_dir} ]; then - echo "Uniswap desk dir already exists, skipping deployment..." - exit 0 -fi - -# Fire curl requests to perform operations on the ship -dojo () { - curl -s --data '{"source":{"dojo":"'"$1"'"},"sink":{"stdout":null}}' http://localhost:12321 -} - -hood () { - curl -s --data '{"source":{"dojo":"+hood/'"$1"'"},"sink":{"app":"hood"}}' http://localhost:12321 -} - -# Create/mount a uniswap desk -hood "merge %uniswap our %landscape" -hood "mount %uniswap" - -# Loop until the uniswap build appears -while [ ! -d ${uniswap_app_build} ]; do - echo "Uniswap app build not found, retrying in 5s..." - sleep 5 -done -echo "Build found..." - -# Copy over build to desk data dir -cp -r ${uniswap_app_build} ${uniswap_desk_dir} - -# Create a mark file for .map file type -cat << EOF > "${uniswap_desk_dir}/mar/map.hoon" -:: -:::: /hoon/map/mar - :: Mark for js source maps -/? 310 -:: -=, eyre -|_ mud=@ -++ grow - |% - ++ mime [/application/octet-stream (as-octs:mimes:html (@t mud))] - -- -++ grab - |% :: convert from - ++ mime |=([p=mite q=octs] (@t q.q)) - ++ noun cord :: clam from %noun - -- -++ grad %mime --- -EOF - -# Create a mark file for .woff file type -cat << EOF > "${uniswap_desk_dir}/mar/woff.hoon" -|_ dat=octs -++ grow - |% - ++ mime [/font/woff dat] - -- -++ grab - |% - ++ mime |=([=mite =octs] octs) - ++ noun octs - -- -++ grad %mime --- -EOF - -# Create a mark file for .ttf file type -cat << EOF > "${uniswap_desk_dir}/mar/ttf.hoon" -|_ dat=octs -++ grow - |% - ++ mime [/font/ttf dat] - -- -++ grab - |% - ++ mime |=([=mite =octs] octs) - ++ noun octs - -- -++ grad %mime --- -EOF - -rm "${uniswap_desk_dir}/desk.bill" -rm "${uniswap_desk_dir}/desk.ship" - -# Commit changes and create a glob -hood "commit %uniswap" -dojo "-landscape!make-glob %uniswap /build" - -glob_file=$(ls -1 -c zod/.urb/put | head -1) -echo "Created glob file: ${glob_file}" - -upload_response=$(curl -X POST -F file=@./zod/.urb/put/${glob_file} ${ipfs_host_endpoint}/api/v0/add) -glob_cid=$(echo "$upload_response" | grep -o '"Hash":"[^"]*' | sed 's/"Hash":"//') - -echo "Glob file uploaded to IFPS:" -echo "{ cid: ${glob_cid}, filename: ${glob_file} }" - -# Curl and wait for the glob to be hosted -glob_url="${ipfs_server_endpoint}/ipfs/${glob_cid}?filename=${glob_file}" - -echo "Checking if glob file hosted at ${glob_url}" -while true; do - response=$(curl -sL -w "%{http_code}" -o /dev/null "$glob_url") - - if [ $response -eq 200 ]; then - echo "File found at $glob_url" - break # Exit the loop if the file is found - else - echo "File not found. Retrying in a few seconds..." - sleep 5 - fi -done - -glob_hash=$(echo "$glob_file" | sed "s/glob-\([a-z0-9\.]*\).glob/\1/") - -# Update the docket file -cat << EOF > "${uniswap_desk_dir}/desk.docket-0" -:~ title+'Uniswap' - info+'Self-hosted uniswap frontend.' - color+0xcd.75df - image+'https://logowik.com/content/uploads/images/uniswap-uni7403.jpg' - base+'uniswap' - glob-http+['${glob_url}' ${glob_hash}] - version+[0 0 1] - website+'https://uniswap.org/' - license+'MIT' -== -EOF - -# Commit changes and install the app -hood "commit %uniswap" -hood "install our %uniswap" - -echo "Uniswap app installed" diff --git a/stack_orchestrator/data/config/uniswap-interface/install-uniswap-app.sh b/stack_orchestrator/data/config/uniswap-interface/install-uniswap-app.sh deleted file mode 100755 index 2463e1c7..00000000 --- a/stack_orchestrator/data/config/uniswap-interface/install-uniswap-app.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/bash - -# $1: Glob file URL (eg. https://xyz.com/glob-0vabcd.glob) -# $2: Glob file hash (eg. 0vabcd) -# $3: Urbit ship's pier dir (default: ./zod) - -if [ "$#" -lt 2 ]; then - echo "Insufficient arguments" - exit 0 -fi - -glob_url=$1 -glob_hash=$2 -echo "Using glob file from ${glob_url} with hash ${glob_hash}" - -# Default pier dir: ./zod -# Default desk dir: ./zod/uniswap -pier_dir="${3:-./zod}" -uniswap_desk_dir="${pier_dir}/uniswap" -echo "Using ${uniswap_desk_dir} as the Uniswap desk dir path" - -# Fire curl requests to perform operations on the ship -dojo () { - curl -s --data '{"source":{"dojo":"'"$1"'"},"sink":{"stdout":null}}' http://localhost:12321 -} - -hood () { - curl -s --data '{"source":{"dojo":"+hood/'"$1"'"},"sink":{"app":"hood"}}' http://localhost:12321 -} - -# Create/mount a uniswap desk -hood "merge %uniswap our %landscape" -hood "mount %uniswap" - -# Create a mark file for .map file type -cat << EOF > "${uniswap_desk_dir}/mar/map.hoon" -:: -:::: /hoon/map/mar - :: Mark for js source maps -/? 310 -:: -=, eyre -|_ mud=@ -++ grow - |% - ++ mime [/application/octet-stream (as-octs:mimes:html (@t mud))] - -- -++ grab - |% :: convert from - ++ mime |=([p=mite q=octs] (@t q.q)) - ++ noun cord :: clam from %noun - -- -++ grad %mime --- -EOF - -# Create a mark file for .woff file type -cat << EOF > "${uniswap_desk_dir}/mar/woff.hoon" -|_ dat=octs -++ grow - |% - ++ mime [/font/woff dat] - -- -++ grab - |% - ++ mime |=([=mite =octs] octs) - ++ noun octs - -- -++ grad %mime --- -EOF - -# Create a mark file for .ttf file type -cat << EOF > "${uniswap_desk_dir}/mar/ttf.hoon" -|_ dat=octs -++ grow - |% - ++ mime [/font/ttf dat] - -- -++ grab - |% - ++ mime |=([=mite =octs] octs) - ++ noun octs - -- -++ grad %mime --- -EOF - -rm "${uniswap_desk_dir}/desk.bill" -rm "${uniswap_desk_dir}/desk.ship" - -# Update the docket file -cat << EOF > "${uniswap_desk_dir}/desk.docket-0" -:~ title+'Uniswap' - info+'Self-hosted uniswap frontend.' - color+0xcd.75df - image+'https://logowik.com/content/uploads/images/uniswap-uni7403.jpg' - base+'uniswap' - glob-http+['${glob_url}' ${glob_hash}] - version+[0 0 1] - website+'https://uniswap.org/' - license+'MIT' -== -EOF - -# Commit changes and install the app -hood "commit %uniswap" -hood "install our %uniswap" - -echo "Uniswap app installed" diff --git a/stack_orchestrator/data/config/uniswap-interface/remote-deploy-uniswap.sh b/stack_orchestrator/data/config/uniswap-interface/remote-deploy-uniswap.sh deleted file mode 100755 index 31f03d72..00000000 --- a/stack_orchestrator/data/config/uniswap-interface/remote-deploy-uniswap.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -# $1: Remote user host -# $2: Remote Urbit ship's pier dir path (eg. /home/user/zod) -# $3: Glob file URL (eg. https://xyz.com/glob-0vabcd.glob) -# $4: Glob file hash (eg. 0vabcd) - -if [ "$#" -ne 4 ]; then - echo "Incorrect number of arguments" - echo "Usage: $0 " - exit 1 -fi - -remote_user_host="$1" -remote_pier_folder="$2" -glob_url="$3" -glob_hash="$4" - -installation_script="./install-uniswap-app.sh" - -ssh "$remote_user_host" "bash -s $glob_url $glob_hash $remote_pier_folder" < "$installation_script" diff --git a/stack_orchestrator/data/config/uniswap-interface/urbit-files/desk.docket-0 b/stack_orchestrator/data/config/uniswap-interface/urbit-files/desk.docket-0 new file mode 100644 index 00000000..6256c41b --- /dev/null +++ b/stack_orchestrator/data/config/uniswap-interface/urbit-files/desk.docket-0 @@ -0,0 +1,10 @@ +:~ title+'Uniswap' + info+'Self-hosted uniswap frontend.' + color+0xcd.75df + image+'https://logowik.com/content/uploads/images/uniswap-uni7403.jpg' + base+'uniswap' + glob-http+['REPLACE_WITH_GLOB_URL' REPLACE_WITH_GLOB_HASH] + version+[0 0 1] + website+'https://uniswap.org/' + license+'MIT' +== diff --git a/stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/map.hoon b/stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/map.hoon new file mode 100644 index 00000000..e5d61eee --- /dev/null +++ b/stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/map.hoon @@ -0,0 +1,18 @@ +:: +:::: /hoon/map/mar + :: Mark for js source maps +/? 310 +:: +=, eyre +|_ mud=@ +++ grow + |% + ++ mime [/application/octet-stream (as-octs:mimes:html (@t mud))] + -- +++ grab + |% :: convert from + ++ mime |=([p=mite q=octs] (@t q.q)) + ++ noun cord :: clam from %noun + -- +++ grad %mime +-- diff --git a/stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/ttf.hoon b/stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/ttf.hoon new file mode 100644 index 00000000..2d29193e --- /dev/null +++ b/stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/ttf.hoon @@ -0,0 +1,12 @@ +|_ dat=octs +++ grow + |% + ++ mime [/font/ttf dat] + -- +++ grab + |% + ++ mime |=([=mite =octs] octs) + ++ noun octs + -- +++ grad %mime +-- diff --git a/stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/woff.hoon b/stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/woff.hoon new file mode 100644 index 00000000..2933aea7 --- /dev/null +++ b/stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/woff.hoon @@ -0,0 +1,12 @@ +|_ dat=octs +++ grow + |% + ++ mime [/font/woff dat] + -- +++ grab + |% + ++ mime |=([=mite =octs] octs) + ++ noun octs + -- +++ grad %mime +-- diff --git a/stack_orchestrator/data/config/urbit/deploy-app-to-remote-urbit.sh b/stack_orchestrator/data/config/urbit/deploy-app-to-remote-urbit.sh new file mode 100755 index 00000000..18019949 --- /dev/null +++ b/stack_orchestrator/data/config/urbit/deploy-app-to-remote-urbit.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# $1: Remote user host +# $2: App name (eg. uniswap) +# $3: Assets dir path (local) for app (eg. /home/user/myapp/urbit-files) +# $4: Remote Urbit ship's pier dir path (eg. /home/user/zod) +# $5: Glob file URL (eg. https://xyz.com/glob-0vabcd.glob) +# $6: Glob file hash (eg. 0vabcd) + +if [ "$#" -ne 6 ]; then + echo "Incorrect number of arguments" + echo "Usage: $0 " + exit 1 +fi + +remote_user_host="$1" +app_name=$2 +app_assets_folder=$3 +remote_pier_folder="$4" +glob_url="$5" +glob_hash="$6" + +installation_script="./install-urbit-app.sh" + +# Copy over the assets to remote machine in a tmp dir +remote_app_assets_folder=/tmp/urbit-app-assets/$app_name +ssh "$remote_user_host" "mkdir -p $remote_app_assets_folder" +scp -r $app_assets_folder/* $remote_user_host:$remote_app_assets_folder + +# Run the installation script +ssh "$remote_user_host" "bash -s $app_name $remote_app_assets_folder '${glob_url}' $glob_hash $remote_pier_folder" < "$installation_script" + +# Remove the tmp assets dir +ssh "$remote_user_host" "rm -rf $remote_app_assets_folder" diff --git a/stack_orchestrator/data/config/urbit/deploy-app.sh b/stack_orchestrator/data/config/urbit/deploy-app.sh new file mode 100755 index 00000000..bde6cd6f --- /dev/null +++ b/stack_orchestrator/data/config/urbit/deploy-app.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +if [ -z "$CERC_URBIT_APP" ]; then + echo "CERC_URBIT_APP not set, exiting" + exit 0 +fi + +echo "Creating Urbit application for ${CERC_URBIT_APP}" + +app_desk_dir=/urbit/zod/${CERC_URBIT_APP} +if [ -d ${app_desk_dir} ]; then + echo "Desk dir already exists for ${CERC_URBIT_APP}, skipping deployment..." + exit 0 +fi + +app_build=/app-builds/${CERC_URBIT_APP}/build +app_mark_files=/app-builds/${CERC_URBIT_APP}/mar +app_docket_file=/app-builds/${CERC_URBIT_APP}/desk.docket-0 + +echo "Reading app build from ${app_build}" +echo "Reading additional mark files from ${app_mark_files}" +echo "Reading docket file ${app_docket_file}" + +# Loop until the app's build appears +while [ ! -d ${app_build} ]; do + echo "${CERC_URBIT_APP} app build not found, retrying in 5s..." + sleep 5 +done +echo "Build found..." + +echo "Using IPFS endpoint ${CERC_IPFS_GLOB_HOST_ENDPOINT} for hosting the ${CERC_URBIT_APP} glob" +echo "Using IPFS server endpoint ${CERC_IPFS_SERVER_ENDPOINT} for reading ${CERC_URBIT_APP} glob" +ipfs_host_endpoint=${CERC_IPFS_GLOB_HOST_ENDPOINT} +ipfs_server_endpoint=${CERC_IPFS_SERVER_ENDPOINT} + +# Fire curl requests to perform operations on the ship +dojo () { + curl -s --data '{"source":{"dojo":"'"$1"'"},"sink":{"stdout":null}}' http://localhost:12321 +} + +hood () { + curl -s --data '{"source":{"dojo":"+hood/'"$1"'"},"sink":{"app":"hood"}}' http://localhost:12321 +} + +# Create / mount the app's desk +hood "merge %${CERC_URBIT_APP} our %landscape" +hood "mount %${CERC_URBIT_APP}" + +# Copy over build to desk data dir +cp -r ${app_build} ${app_desk_dir} + +# Copy over the additional mark files +cp ${app_mark_files}/* ${app_desk_dir}/mar/ + +rm "${app_desk_dir}/desk.bill" +rm "${app_desk_dir}/desk.ship" + +# Commit changes and create a glob +hood "commit %${CERC_URBIT_APP}" +dojo "-landscape!make-glob %${CERC_URBIT_APP} /build" + +glob_file=$(ls -1 -c zod/.urb/put | head -1) +echo "Created glob file: ${glob_file}" + +# Upload the glob file to IPFS +echo "Uploading glob file to ${ipfs_host_endpoint}" +upload_response=$(curl -X POST -F file=@./zod/.urb/put/${glob_file} ${ipfs_host_endpoint}/api/v0/add) +glob_cid=$(echo "$upload_response" | grep -o '"Hash":"[^"]*' | sed 's/"Hash":"//') + +glob_url="${ipfs_server_endpoint}/ipfs/${glob_cid}?filename=${glob_file}" +glob_hash=$(echo "$glob_file" | sed "s/glob-\([a-z0-9\.]*\).glob/\1/") + +echo "Glob file uploaded to IFPS:" +echo "{ cid: ${glob_cid}, filename: ${glob_file} }" +echo "{ url: ${glob_url}, hash: ${glob_hash} }" + +# Exit if the installation not required +if [ "$CERC_ENABLE_APP_INSTALL" = "false" ]; then + echo "CERC_ENABLE_APP_INSTALL set to false, skipping app installation" + exit 0 +fi + +# Curl and wait for the glob to be hosted +echo "Checking if glob file hosted at ${glob_url}" +while true; do + response=$(curl -sL -w "%{http_code}" -o /dev/null "$glob_url") + + if [ $response -eq 200 ]; then + echo "File found at $glob_url" + break # Exit the loop if the file is found + else + echo "File not found, retrying in a 5s..." + sleep 5 + fi +done + +# Replace the docket file for app +# Substitue the glob URL and hash +cp ${app_docket_file} ${app_desk_dir}/ +sed -i "s|REPLACE_WITH_GLOB_URL|${glob_url}|g; s|REPLACE_WITH_GLOB_HASH|${glob_hash}|g" ${app_desk_dir}/desk.docket-0 + +# Commit changes and install the app +hood "commit %${CERC_URBIT_APP}" +hood "install our %${CERC_URBIT_APP}" + +echo "${CERC_URBIT_APP} app installed" diff --git a/stack_orchestrator/data/config/urbit/install-urbit-app.sh b/stack_orchestrator/data/config/urbit/install-urbit-app.sh new file mode 100755 index 00000000..444ff45e --- /dev/null +++ b/stack_orchestrator/data/config/urbit/install-urbit-app.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# $1: App name (eg. uniswap) +# $2: Assets dir path (local) for app (eg. /home/user/myapp/urbit-files) +# $3: Glob file URL (eg. https://xyz.com/glob-0vabcd.glob) +# $4: Glob file hash (eg. 0vabcd) +# $5: Urbit ship's pier dir (default: ./zod) + +if [ "$#" -lt 4 ]; then + echo "Insufficient arguments" + echo "Usage: $0 [/path/to/remote/pier/folder]" + exit 1 +fi + +app_name=$1 +app_mark_files=$2/mar +app_docket_file=$2/desk.docket-0 +echo "Creating Urbit application for ${app_name}" +echo "Reading additional mark files from ${app_mark_files}" +echo "Reading docket file ${app_docket_file}" + +glob_url=$3 +glob_hash=$4 +echo "Using glob file from ${glob_url} with hash ${glob_hash}" + +# Default pier dir: ./zod +# Default desk dir: ./zod/ +pier_dir="${5:-./zod}" +app_desk_dir="${pier_dir}/${app_name}" +echo "Using ${app_desk_dir} as the ${app_name} desk dir path" + +# Fire curl requests to perform operations on the ship +dojo () { + curl -s --data '{"source":{"dojo":"'"$1"'"},"sink":{"stdout":null}}' http://localhost:12321 +} + +hood () { + curl -s --data '{"source":{"dojo":"+hood/'"$1"'"},"sink":{"app":"hood"}}' http://localhost:12321 +} + +# Create / mount the app's desk +hood "merge %${app_name} our %landscape" +hood "mount %${app_name}" + +# Copy over the additional mark files +cp ${app_mark_files}/* ${app_desk_dir}/mar/ + +rm "${app_desk_dir}/desk.bill" +rm "${app_desk_dir}/desk.ship" + +# Replace the docket file for app +# Substitue the glob URL and hash +cp ${app_docket_file} ${app_desk_dir}/ +sed -i "s|REPLACE_WITH_GLOB_URL|${glob_url}|g; s|REPLACE_WITH_GLOB_HASH|${glob_hash}|g" ${app_desk_dir}/desk.docket-0 + +# Commit changes and install the app +hood "commit %${app_name}" +hood "install our %${app_name}" + +echo "${app_name} app installed" diff --git a/stack_orchestrator/data/config/urbit/run-urbit-ship.sh b/stack_orchestrator/data/config/urbit/run-urbit-ship.sh index bb301c81..0b219803 100755 --- a/stack_orchestrator/data/config/urbit/run-urbit-ship.sh +++ b/stack_orchestrator/data/config/urbit/run-urbit-ship.sh @@ -7,11 +7,13 @@ fi pier_dir="/urbit/zod" +# TODO: Bootstrap fake ship on the first run + # Run urbit ship in daemon mode # Check if the directory exists if [ -d "$pier_dir" ]; then echo "Pier directory already exists, rebooting..." - urbit -d zod + /urbit/zod/.run -d else echo "Creating a new fake ship..." urbit -d -F zod diff --git a/stack_orchestrator/data/stacks/proxy-server/README.md b/stack_orchestrator/data/stacks/proxy-server/README.md index f0ccdb0f..294b01a9 100644 --- a/stack_orchestrator/data/stacks/proxy-server/README.md +++ b/stack_orchestrator/data/stacks/proxy-server/README.md @@ -47,7 +47,7 @@ laconic-so --stack proxy-server build-containers ```bash # Whether to run the proxy server (Optional) (Default: true) - ENABLE_PROXY= + CERC_ENABLE_PROXY= # Upstream endpoint # (Eg. https://api.example.org) diff --git a/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md b/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md index 7499f5fc..6818dd94 100644 --- a/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md +++ b/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md @@ -33,7 +33,7 @@ laconic-so --stack uniswap-urbit-app deploy init --output uniswap-urbit-app-spec ### Ports -Edit `network` in spec file to map container ports to same ports in host +Edit `network` in spec file to map container ports to same ports in host: ``` ... @@ -43,12 +43,14 @@ network: - '8080:80' proxy-server: - '4000:4000' - ipfs-glob-host: + ipfs: - '8081:8080' - '5001:5001' ... ``` +Note: Skip the `ipfs` ports if need to use an externally running IPFS node + ### Data volumes Container data volumes are bind-mounted to specified paths in the host filesystem. @@ -67,6 +69,9 @@ laconic-so --stack uniswap-urbit-app deploy create --spec-file uniswap-urbit-app Inside the deployment directory, open the file `config.env` and set the following env variables: ```bash + # App to be installed (Do not change) + CERC_URBIT_APP=uniswap + # External RPC endpoints # https://docs.infura.io/getting-started#2-create-an-api-key CERC_INFURA_KEY= @@ -79,9 +84,13 @@ Inside the deployment directory, open the file `config.env` and set the followin # Optional + # Whether to enable app installation on Urbit + # (just builds and uploads the glob file if disabled) (Default: true) + CERC_ENABLE_APP_INSTALL= + # Whether to run the proxy GQL server - # (Disable only if proxy not required to be run) (Default: true) - ENABLE_PROXY= + # (disable only if proxy not required to be run) (Default: true) + CERC_ENABLE_PROXY= # Proxy server configuration # Used only if proxy is enabled @@ -97,11 +106,11 @@ Inside the deployment directory, open the file `config.env` and set the followin # IPFS configuration # IFPS endpoint to host the glob file on - # (Default: http://ipfs-glob-host:5001 pointing to in-stack IPFS node) + # (Default: http://ipfs:5001 pointing to in-stack IPFS node) CERC_IPFS_GLOB_HOST_ENDPOINT= # IFPS endpoint to fetch the glob file from - # (Default: http://ipfs-glob-host:8080 pointing to in-stack IPFS node) + # (Default: http://ipfs:8080 pointing to in-stack IPFS node) CERC_IPFS_SERVER_ENDPOINT= ``` diff --git a/stack_orchestrator/data/stacks/uniswap-urbit-app/stack.yml b/stack_orchestrator/data/stacks/uniswap-urbit-app/stack.yml index 3f77098f..31499406 100644 --- a/stack_orchestrator/data/stacks/uniswap-urbit-app/stack.yml +++ b/stack_orchestrator/data/stacks/uniswap-urbit-app/stack.yml @@ -10,4 +10,5 @@ containers: pods: - uniswap-interface - proxy-server - - uniswap-urbit + - fixturenet-urbit + - kubo From fa6b570f4a1f221eb396df9d4dd654df8fc9ec15 Mon Sep 17 00:00:00 2001 From: Nabarun Gogoi Date: Mon, 11 Dec 2023 14:10:54 +0530 Subject: [PATCH 44/75] Add stack for running osmosis frontend app (#673) * osmosis FE stack * chmod * dont use 3000 * fix for neww stack format * updates * update osmosis readme * Update stack.yml * Update osmosis frontend stack to serve app * Host osmosis app static build using python server * Fix mapped ports in deployment for containers * Update instructions * Use nginx server to host files and handle page reloads * Fix typo --------- Co-authored-by: zramsay Co-authored-by: Prathamesh Musale --- .../data/compose/docker-compose-kubo.yml | 2 +- .../docker-compose-osmosis-front-end.yml | 34 +++++ .../data/config/osmosis/build-app.sh | 18 +++ .../data/config/osmosis/nginx/web-app.conf | 16 ++ .../cerc-osmosis-front-end/build.sh | 4 + .../data/stacks/osmosis/README.md | 141 ++++++++++++++++++ .../data/stacks/osmosis/stack.yml | 30 ++++ .../data/stacks/uniswap-urbit-app/README.md | 13 +- 8 files changed, 254 insertions(+), 4 deletions(-) create mode 100644 stack_orchestrator/data/compose/docker-compose-osmosis-front-end.yml create mode 100755 stack_orchestrator/data/config/osmosis/build-app.sh create mode 100644 stack_orchestrator/data/config/osmosis/nginx/web-app.conf create mode 100755 stack_orchestrator/data/container-build/cerc-osmosis-front-end/build.sh create mode 100644 stack_orchestrator/data/stacks/osmosis/README.md create mode 100644 stack_orchestrator/data/stacks/osmosis/stack.yml diff --git a/stack_orchestrator/data/compose/docker-compose-kubo.yml b/stack_orchestrator/data/compose/docker-compose-kubo.yml index 2e4ae419..b50dd15a 100644 --- a/stack_orchestrator/data/compose/docker-compose-kubo.yml +++ b/stack_orchestrator/data/compose/docker-compose-kubo.yml @@ -3,7 +3,7 @@ version: "3.2" # See: https://docs.ipfs.tech/install/run-ipfs-inside-docker/#set-up services: ipfs: - image: ipfs/kubo:master-2023-02-20-714a968 + image: ipfs/kubo:v0.24.0 restart: always volumes: - ipfs-import:/import diff --git a/stack_orchestrator/data/compose/docker-compose-osmosis-front-end.yml b/stack_orchestrator/data/compose/docker-compose-osmosis-front-end.yml new file mode 100644 index 00000000..f4a25b54 --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-osmosis-front-end.yml @@ -0,0 +1,34 @@ +version: "3.2" + +services: + osmosis-front-end: + image: cerc/osmosis-front-end:local + restart: on-failure + environment: + - NEXT_PUBLIC_WEB_API_BASE_URL=${CERC_WEB_API_BASE_URL} + working_dir: /app/packages/web + command: ["./build-app.sh"] + volumes: + - ../config/osmosis/build-app.sh:/app/packages/web/build-app.sh + - app_builds:/app-builds + + nginx: + image: nginx:1.23-alpine + restart: always + depends_on: + osmosis-front-end: + condition: service_completed_successfully + volumes: + - ../config/osmosis/nginx:/etc/nginx/conf.d + - app_builds:/usr/share/nginx + ports: + - "80" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "80"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + +volumes: + app_builds: diff --git a/stack_orchestrator/data/config/osmosis/build-app.sh b/stack_orchestrator/data/config/osmosis/build-app.sh new file mode 100755 index 00000000..2f3cfd01 --- /dev/null +++ b/stack_orchestrator/data/config/osmosis/build-app.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +# Check and exit if a deployment already exists (on restarts) +if [ -d /app-builds/osmosis/build ]; then + echo "Build already exists, remove volume to rebuild" + exit 0 +fi + +yarn build:static + +# Move build to app-builds +mkdir -p /app-builds/osmosis +cp -r ./out /app-builds/osmosis/build diff --git a/stack_orchestrator/data/config/osmosis/nginx/web-app.conf b/stack_orchestrator/data/config/osmosis/nginx/web-app.conf new file mode 100644 index 00000000..784df3b2 --- /dev/null +++ b/stack_orchestrator/data/config/osmosis/nginx/web-app.conf @@ -0,0 +1,16 @@ +server { + listen 80; + listen [::]:80; + server_name _; + + error_page 500 502 503 504 /50x.html; + + location / { + root /usr/share/nginx/osmosis/build; + index index.html index.htm index.nginx-debian.html; + + # First attempt to serve request as file, then as html, + # then as directory, then fall back to displaying a 404. + try_files $uri $uri.html $uri/ /index.html =404; + } +} diff --git a/stack_orchestrator/data/container-build/cerc-osmosis-front-end/build.sh b/stack_orchestrator/data/container-build/cerc-osmosis-front-end/build.sh new file mode 100755 index 00000000..c4a9f27e --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-osmosis-front-end/build.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Build the osmosis front end image +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh +docker build -t cerc/osmosis-front-end:local -f ${CERC_REPO_BASE_DIR}/osmosis-frontend/docker/Dockerfile.static ${build_command_args} ${CERC_REPO_BASE_DIR}/osmosis-frontend diff --git a/stack_orchestrator/data/stacks/osmosis/README.md b/stack_orchestrator/data/stacks/osmosis/README.md new file mode 100644 index 00000000..4c13c93c --- /dev/null +++ b/stack_orchestrator/data/stacks/osmosis/README.md @@ -0,0 +1,141 @@ +# self-hosted osmosis + +Instructions to build and deploy: +- self-hosted gitea +- an ipfs node +- the osmosis front end +- a laconicd chain + +## Setup + +Clone required repositories: + +```bash +laconic-so --stack osmosis setup-repositories + +# If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories and re-run the command +``` + +Build the container images: + +``` +# support image for the gitea package registry +laconic-so --stack build-support build-containers + +laconic-so --stack osmosis build-containers +``` + +## Create a deployment + +First, create a spec file for the deployment, which will map the stack's ports and volumes to the host: +```bash +laconic-so --stack osmosis deploy init --output osmosis-spec.yml +``` + +### Ports + +Edit `network` in spec file to map container ports to same ports in host + +``` +... +network: + ports: + proxy-server: + - '4000:4000' + nginx: + - '3000:80' +``` + +### Data volumes + +Container data volumes are bind-mounted to specified paths in the host filesystem. +The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory. The default mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`. + +--- + +Once you've made any needed changes to the spec file, create a deployment from it: +```bash +laconic-so --stack osmosis deploy create --spec-file osmosis-spec.yml --deployment-dir osmosis-deployment +``` + +## Set env variables + +Inside the deployment directory, open the file `config.env` and set the following env variables: + + ```bash + # Osmosis API base URL + # Set this to proxy server endpoint for uniswap app + # (Eg. http://localhost:4000 - in case stack is being run locally with proxy enabled) + # (Eg. https://abc.xyz.com - in case https://abc.xyz.com is pointed to the proxy endpoint) + CERC_WEB_API_BASE_URL=http://localhost:4000 + + # Optional + + # Whether to run the proxy server + # (Disable only if proxy not required to be run) (Default: true) + CERC_ENABLE_PROXY= + + # Proxy server configuration + # Used only if proxy is enabled + + # Upstream API URL + CERC_PROXY_UPSTREAM=https://app.osmosis.zone + ``` + +## Start the stack + +Start the deployment: +```bash +laconic-so deployment --dir osmosis-deployment start +``` + +* List and check the health status of all the containers using `docker ps` and wait for them to be `healthy` + +* The web app can be accessed at http://localhost:3000 + +## Laconic registry + +Setup a test chain: +``` +export CERC_NPM_REGISTRY_URL=https://git.vdb.to/api/packages/cerc-io/npm/ + +laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd,git.vdb.to/cerc-io/laconic-sdk,git.vdb.to/cerc-io/laconic-registry-cli,git.vdb.to/cerc-io/laconic-console + +laconic-so --stack fixturenet-laconic-loaded build-containers + +export LACONIC_HOSTED_ENDPOINT=http:// + +laconic-so --stack fixturenet-laconic-loaded deploy up +``` + +then `docker exec` into the `laconicd` container and either export the private key or create a new one and send funds to it. Use that private key for `LACONIC_HOTWALLET_KEY`. + +## Clean up + +To stop all osmosis services running in the background, while preserving data: + +```bash +# Only stop the docker containers +laconic-so deployment --dir osmosis-deployment stop + +# Run 'start' to restart the deployment +``` + +To stop all osmosis services and also delete data: + +```bash +# Stop the docker containers +laconic-so deployment --dir osmosis-deployment stop --delete-volumes + +# Remove deployment directory (deployment will have to be recreated for a re-run) +rm -r osmosis-deployment +``` + +To stop stack running Laconic registry + +```bash +laconic-so --stack fixturenet-laconic-loaded deploy down + +# Delete volumes +laconic-so --stack fixturenet-laconic-loaded deploy down --delete-volumes +``` diff --git a/stack_orchestrator/data/stacks/osmosis/stack.yml b/stack_orchestrator/data/stacks/osmosis/stack.yml new file mode 100644 index 00000000..a0669b71 --- /dev/null +++ b/stack_orchestrator/data/stacks/osmosis/stack.yml @@ -0,0 +1,30 @@ +version: "0.1" +name: osmosis +repos: + # For gitea + # - git.vdb.to/cerc-io/hosting@names-for-so + # - gitea.com/gitea/act_runner + # For osmosis frontend + - github.com/cerc-io/watcher-ts@v0.2.78 + - github.com/cerc-io/osmosis-frontend@laconic +containers: + # - cerc/act-runner + # - cerc/act-runner-task-executor + - cerc/watcher-ts + - cerc/osmosis-front-end +pods: + - kubo + - proxy-server + - osmosis-front-end + # TODO: Fix pod config for deployment command + # TODO: mirroring all of osmosis repos: https://git.vdb.to/cerc-io/hosting/pulls/42 + # - name: gitea + # repository: cerc-io/hosting + # path: gitea + # pre_start_command: "run-this-first.sh" + # post_start_command: "initialize-gitea.sh" + # - name: act-runner + # repository: cerc-io/hosting + # path: act-runner + # pre_start_command: "pre_start.sh" + # post_start_command: "post_start.sh" diff --git a/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md b/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md index 6818dd94..7f49c16f 100644 --- a/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md +++ b/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md @@ -78,9 +78,9 @@ Inside the deployment directory, open the file `config.env` and set the followin # Uniswap API GQL Endpoint # Set this to GQL proxy server endpoint for uniswap app - # (Eg. http://localhost:4000/v1/graphql) - # (Eg. https://abc.xyz.com/v1/graphql) - CERC_UNISWAP_GQL= + # (Eg. http://localhost:4000/v1/graphql - in case stack is being run locally with proxy enabled) + # (Eg. https://abc.xyz.com/v1/graphql - in case https://abc.xyz.com is pointed to the proxy endpoint) + CERC_UNISWAP_GQL=http://localhost:4000/v1/graphql # Optional @@ -150,11 +150,18 @@ laconic-so deployment --dir uniswap-urbit-app-deployment start To stop all uniswap-urbit-app services running in the background, while preserving data: ```bash +# Only stop the docker containers laconic-so deployment --dir uniswap-urbit-app-deployment stop + +# Run 'start' to restart the deployment ``` To stop all uniswap-urbit-app services and also delete data: ```bash +# Stop the docker containers laconic-so deployment --dir uniswap-urbit-app-deployment stop --delete-volumes + +# Remove deployment directory (deployment will have to be recreated for a re-run) +rm -r uniswap-urbit-app-deployment ``` From 232d5618cb29a8f581d5699d2a3267557561c90a Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Tue, 12 Dec 2023 13:54:00 +0530 Subject: [PATCH 45/75] Update instructions in osmosis stack (#684) --- .../data/stacks/osmosis/README.md | 20 ++++++++++++++----- .../data/stacks/osmosis/stack.yml | 2 +- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/stack_orchestrator/data/stacks/osmosis/README.md b/stack_orchestrator/data/stacks/osmosis/README.md index 4c13c93c..ef5504ae 100644 --- a/stack_orchestrator/data/stacks/osmosis/README.md +++ b/stack_orchestrator/data/stacks/osmosis/README.md @@ -18,9 +18,9 @@ laconic-so --stack osmosis setup-repositories Build the container images: -``` -# support image for the gitea package registry -laconic-so --stack build-support build-containers +```bash +# TODO: support image for the gitea package registry +# laconic-so --stack build-support build-containers laconic-so --stack osmosis build-containers ``` @@ -62,9 +62,11 @@ laconic-so --stack osmosis deploy create --spec-file osmosis-spec.yml --deployme Inside the deployment directory, open the file `config.env` and set the following env variables: + (Note: Following config can be used as is if the stack is being run locally) + ```bash # Osmosis API base URL - # Set this to proxy server endpoint for uniswap app + # Set this to proxy server endpoint for osmosis app # (Eg. http://localhost:4000 - in case stack is being run locally with proxy enabled) # (Eg. https://abc.xyz.com - in case https://abc.xyz.com is pointed to the proxy endpoint) CERC_WEB_API_BASE_URL=http://localhost:4000 @@ -85,10 +87,17 @@ Inside the deployment directory, open the file `config.env` and set the followin ## Start the stack Start the deployment: + ```bash laconic-so deployment --dir osmosis-deployment start ``` +* The `osmosis-front-end` container will take some time to run to completion as it builds the front-end app with given configuration; same can be tracked in the logs: + + ```bash + laconic-so deployment --dir osmosis-deployment logs -f osmosis-front-end + ``` + * List and check the health status of all the containers using `docker ps` and wait for them to be `healthy` * The web app can be accessed at http://localhost:3000 @@ -96,7 +105,8 @@ laconic-so deployment --dir osmosis-deployment start ## Laconic registry Setup a test chain: -``` + +```bash export CERC_NPM_REGISTRY_URL=https://git.vdb.to/api/packages/cerc-io/npm/ laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd,git.vdb.to/cerc-io/laconic-sdk,git.vdb.to/cerc-io/laconic-registry-cli,git.vdb.to/cerc-io/laconic-console diff --git a/stack_orchestrator/data/stacks/osmosis/stack.yml b/stack_orchestrator/data/stacks/osmosis/stack.yml index a0669b71..b0e51a1d 100644 --- a/stack_orchestrator/data/stacks/osmosis/stack.yml +++ b/stack_orchestrator/data/stacks/osmosis/stack.yml @@ -6,7 +6,7 @@ repos: # - gitea.com/gitea/act_runner # For osmosis frontend - github.com/cerc-io/watcher-ts@v0.2.78 - - github.com/cerc-io/osmosis-frontend@laconic + - github.com/cerc-io/osmosis-frontend@laconic # TODO: Use release containers: # - cerc/act-runner # - cerc/act-runner-task-executor From 1ef0b316c6e564e96e9d9dbf922b19ea5d9ba4d1 Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Wed, 13 Dec 2023 14:58:26 +0530 Subject: [PATCH 46/75] Expose metrics endpoints for sushiswap and merkl sushiswap watchers (#685) --- .../compose/docker-compose-watcher-merkl-sushiswap-v3.yml | 4 ++-- .../data/compose/docker-compose-watcher-sushiswap-v3.yml | 4 ++-- .../watcher-merkl-sushiswap-v3/watcher-config-template.toml | 2 +- .../config/watcher-sushiswap-v3/watcher-config-template.toml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml b/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml index 0a83af89..aae7bb47 100644 --- a/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml +++ b/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml @@ -35,7 +35,7 @@ services: - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml - ../config/watcher-merkl-sushiswap-v3/start-job-runner.sh:/app/start-job-runner.sh ports: - - "9000" + - "9002:9000" healthcheck: test: ["CMD", "nc", "-v", "localhost", "9000"] interval: 20s @@ -62,7 +62,7 @@ services: - ../config/watcher-merkl-sushiswap-v3/start-server.sh:/app/start-server.sh ports: - "127.0.0.1:3007:3008" - - "9001" + - "9003:9001" healthcheck: test: ["CMD", "nc", "-v", "localhost", "3008"] interval: 20s diff --git a/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml b/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml index f7b75ca5..6c39320c 100644 --- a/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml +++ b/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml @@ -35,7 +35,7 @@ services: - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml - ../config/watcher-sushiswap-v3/start-job-runner.sh:/app/start-job-runner.sh ports: - - "9000" + - "9000:9000" healthcheck: test: ["CMD", "nc", "-v", "localhost", "9000"] interval: 20s @@ -62,7 +62,7 @@ services: - ../config/watcher-sushiswap-v3/start-server.sh:/app/start-server.sh ports: - "127.0.0.1:3008:3008" - - "9001" + - "9001:9001" healthcheck: test: ["CMD", "nc", "-v", "localhost", "3008"] interval: 20s diff --git a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml index f5355a4b..8d3b3f4d 100644 --- a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml +++ b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml @@ -41,7 +41,7 @@ timeTravelMaxAge = 86400 # 1 day [metrics] - host = "127.0.0.1" + host = "0.0.0.0" port = 9000 [metrics.gql] port = 9001 diff --git a/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml index 7cfabedd..e45cf0ea 100644 --- a/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml +++ b/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml @@ -41,7 +41,7 @@ timeTravelMaxAge = 86400 # 1 day [metrics] - host = "127.0.0.1" + host = "0.0.0.0" port = 9000 [metrics.gql] port = 9001 From 88f66a36263a238b0993d7bbd5aac1244c7673c5 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 13 Dec 2023 21:02:34 -0600 Subject: [PATCH 47/75] Add `deployment update` and `deploy-webapp-from-registry` commands. (#676) --- .../demo-records/demo-record-10.yml | 14 + .../demo-records/demo-record-11.yml | 12 + .../demo-records/demo-record-12.yml | 17 + .../demo-records/demo-record-7.yml | 11 + .../demo-records/demo-record-8.yml | 18 ++ .../demo-records/demo-record-9.yml | 17 + .../deploy/compose/deploy_docker.py | 6 + stack_orchestrator/deploy/deploy.py | 8 + stack_orchestrator/deploy/deployer.py | 4 + stack_orchestrator/deploy/deployment.py | 9 +- stack_orchestrator/deploy/k8s/cluster_info.py | 1 + stack_orchestrator/deploy/k8s/deploy_k8s.py | 29 ++ .../deploy/webapp/deploy_webapp.py | 47 +-- .../webapp/deploy_webapp_from_registry.py | 267 +++++++++++++++ stack_orchestrator/deploy/webapp/util.py | 303 ++++++++++++++++++ stack_orchestrator/main.py | 3 +- 16 files changed, 743 insertions(+), 23 deletions(-) create mode 100644 stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-10.yml create mode 100644 stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-11.yml create mode 100644 stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-12.yml create mode 100644 stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-7.yml create mode 100644 stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-8.yml create mode 100644 stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-9.yml create mode 100644 stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py create mode 100644 stack_orchestrator/deploy/webapp/util.py diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-10.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-10.yml new file mode 100644 index 00000000..a467903e --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-10.yml @@ -0,0 +1,14 @@ +record: + type: ApplicationDeploymentRecord + version: 1.2.3 + name: name + description: description + application: application + url: url + dns: dns + request: request + meta: + foo: bar + tags: + - a + - b diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-11.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-11.yml new file mode 100644 index 00000000..3afbd64d --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-11.yml @@ -0,0 +1,12 @@ +record: + type: GeneralRecord + version: 1.2.3 + name: name + description: description + category: category + value: value + meta: + foo: bar + tags: + - a + - b diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-12.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-12.yml new file mode 100644 index 00000000..4c5f024c --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-12.yml @@ -0,0 +1,17 @@ +record: + type: ApplicationArtifact + version: 1.2.3 + name: name + description: description + application: appidgoeshere + content_type: content_type + os: os + cpu: cpu + uri: + - uri://a + - uri://b + meta: + foo: bar + tags: + - a + - b diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-7.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-7.yml new file mode 100644 index 00000000..1bf4ad46 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-7.yml @@ -0,0 +1,11 @@ +record: + type: DnsRecord + version: 0.0.1 + name: "foo" + resource_type: "A" + value: "bar" + meta: + foo: bar + tags: + - a + - b diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-8.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-8.yml new file mode 100644 index 00000000..95bd195a --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-8.yml @@ -0,0 +1,18 @@ +record: + type: ApplicationRecord + version: 0.0.1 + name: my-demo-app + description: "Description of my app" + homepage: http://my.demo.app + license: license + author: author + repository: + - "https://my.demo.repo" + repository_ref: "v0.1.0" + app_version: "0.1.0" + app_type: "webapp" + meta: + foo: bar + tags: + - a + - b diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-9.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-9.yml new file mode 100644 index 00000000..415e5c74 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-9.yml @@ -0,0 +1,17 @@ +record: + type: ApplicationDeploymentRequest + version: 1.2.3 + application: application + dns: dns + config: + env: + ENV_VAR_A: A + ENV_VAR_B: B + crn: + - crn://foo.bar + - crn://bar.baz + meta: + foo: bar + tags: + - a + - b diff --git a/stack_orchestrator/deploy/compose/deploy_docker.py b/stack_orchestrator/deploy/compose/deploy_docker.py index d34d1e6f..b2622820 100644 --- a/stack_orchestrator/deploy/compose/deploy_docker.py +++ b/stack_orchestrator/deploy/compose/deploy_docker.py @@ -40,6 +40,12 @@ class DockerDeployer(Deployer): except DockerException as e: raise DeployerException(e) + def update(self): + try: + return self.docker.compose.restart() + except DockerException as e: + raise DeployerException(e) + def status(self): try: for p in self.docker.compose.ps(): diff --git a/stack_orchestrator/deploy/deploy.py b/stack_orchestrator/deploy/deploy.py index da96a500..18d27a21 100644 --- a/stack_orchestrator/deploy/deploy.py +++ b/stack_orchestrator/deploy/deploy.py @@ -120,6 +120,14 @@ def status_operation(ctx): ctx.obj.deployer.status() +def update_operation(ctx): + global_context = ctx.parent.parent.obj + if not global_context.dry_run: + if global_context.verbose: + print("Running compose update") + ctx.obj.deployer.update() + + def ps_operation(ctx): global_context = ctx.parent.parent.obj if not global_context.dry_run: diff --git a/stack_orchestrator/deploy/deployer.py b/stack_orchestrator/deploy/deployer.py index 2806044b..2df784a2 100644 --- a/stack_orchestrator/deploy/deployer.py +++ b/stack_orchestrator/deploy/deployer.py @@ -27,6 +27,10 @@ class Deployer(ABC): def down(self, timeout, volumes): pass + @abstractmethod + def update(self): + pass + @abstractmethod def ps(self): pass diff --git a/stack_orchestrator/deploy/deployment.py b/stack_orchestrator/deploy/deployment.py index 366a83f6..f364121f 100644 --- a/stack_orchestrator/deploy/deployment.py +++ b/stack_orchestrator/deploy/deployment.py @@ -19,7 +19,7 @@ import sys from stack_orchestrator import constants from stack_orchestrator.deploy.images import push_images_operation from stack_orchestrator.deploy.deploy import up_operation, down_operation, ps_operation, port_operation, status_operation -from stack_orchestrator.deploy.deploy import exec_operation, logs_operation, create_deploy_context +from stack_orchestrator.deploy.deploy import exec_operation, logs_operation, create_deploy_context, update_operation from stack_orchestrator.deploy.deploy_types import DeployCommandContext from stack_orchestrator.deploy.deployment_context import DeploymentContext @@ -149,3 +149,10 @@ def logs(ctx, tail, follow, extra_args): def status(ctx): ctx.obj = make_deploy_context(ctx) status_operation(ctx) + + +@command.command() +@click.pass_context +def update(ctx): + ctx.obj = make_deploy_context(ctx) + update_operation(ctx) diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 0aa74189..24fe15a0 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -189,6 +189,7 @@ class ClusterInfo: container = client.V1Container( name=container_name, image=image_to_use, + image_pull_policy="Always", env=envs_from_environment_variables_map(self.environment_variables.map), ports=[client.V1ContainerPort(container_port=port)], volume_mounts=volume_mounts, diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index 95131966..bf82ebdf 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -13,6 +13,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from datetime import datetime, timezone + from pathlib import Path from kubernetes import client, config @@ -304,6 +306,33 @@ class K8sDeployer(Deployer): log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test") return log_stream_from_string(log_data) + def update(self): + self.connect_api() + ref_deployment = self.cluster_info.get_deployment() + + deployment = self.apps_api.read_namespaced_deployment( + name=ref_deployment.metadata.name, + namespace=self.k8s_namespace + ) + + new_env = ref_deployment.spec.template.spec.containers[0].env + for container in deployment.spec.template.spec.containers: + old_env = container.env + if old_env != new_env: + container.env = new_env + + deployment.spec.template.metadata.annotations = { + "kubectl.kubernetes.io/restartedAt": datetime.utcnow() + .replace(tzinfo=timezone.utc) + .isoformat() + } + + self.apps_api.patch_namespaced_deployment( + name=ref_deployment.metadata.name, + namespace=self.k8s_namespace, + body=deployment + ) + def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, ports=[], detach=False): # We need to figure out how to do this -- check why we're being called first pass diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp.py b/stack_orchestrator/deploy/webapp/deploy_webapp.py index 391162c9..aca2df35 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp.py @@ -54,26 +54,7 @@ def _fixup_url_spec(spec_file_name: str, url: str): wfile.write(contents) -@click.group() -@click.pass_context -def command(ctx): - '''manage a webapp deployment''' - - # Check that --stack wasn't supplied - if ctx.parent.obj.stack: - error_exit("--stack can't be supplied with the deploy-webapp command") - - -@command.command() -@click.option("--kube-config", help="Provide a config file for a k8s deployment") -@click.option("--image-registry", help="Provide a container image registry url for this k8s cluster") -@click.option("--deployment-dir", help="Create deployment files in this directory", required=True) -@click.option("--image", help="image to deploy", required=True) -@click.option("--url", help="url to serve", required=True) -@click.option("--env-file", help="environment file for webapp") -@click.pass_context -def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_file): - '''create a deployment for the specified webapp container''' +def create_deployment(ctx, deployment_dir, image, url, kube_config, image_registry, env_file): # Do the equivalent of: # 1. laconic-so --stack webapp-template deploy --deploy-to k8s init --output webapp-spec.yml # --config (eqivalent of the contents of my-config.env) @@ -92,7 +73,7 @@ def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_fil # TODO: support env file deploy_command_context: DeployCommandContext = create_deploy_context( global_options2(ctx), None, stack, None, None, None, env_file, "k8s" - ) + ) init_operation( deploy_command_context, stack, @@ -116,3 +97,27 @@ def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_fil # Fix up the container tag inside the deployment compose file _fixup_container_tag(deployment_dir, image) os.remove(spec_file_name) + + +@click.group() +@click.pass_context +def command(ctx): + '''manage a webapp deployment''' + + # Check that --stack wasn't supplied + if ctx.parent.obj.stack: + error_exit("--stack can't be supplied with the deploy-webapp command") + + +@command.command() +@click.option("--kube-config", help="Provide a config file for a k8s deployment") +@click.option("--image-registry", help="Provide a container image registry url for this k8s cluster") +@click.option("--deployment-dir", help="Create deployment files in this directory", required=True) +@click.option("--image", help="image to deploy", required=True) +@click.option("--url", help="url to serve", required=True) +@click.option("--env-file", help="environment file for webapp") +@click.pass_context +def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_file): + '''create a deployment for the specified webapp container''' + + return create_deployment(ctx, deployment_dir, image, url, kube_config, image_registry, env_file) diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py new file mode 100644 index 00000000..5f9e712b --- /dev/null +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -0,0 +1,267 @@ +# Copyright ©2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import json +import os +import shlex +import shutil +import sys +import tempfile + +import click + +from stack_orchestrator.deploy.webapp import deploy_webapp +from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient, + build_container_image, push_container_image, + file_hash, deploy_to_k8s, publish_deployment, + hostname_for_deployment_request, generate_hostname_for_app) + + +def process_app_deployment_request( + ctx, + laconic: LaconicRegistryClient, + app_deployment_request, + deployment_record_namespace, + dns_record_namespace, + dns_suffix, + deployment_parent_dir, + kube_config, + image_registry +): + # 1. look up application + app = laconic.get_record(app_deployment_request.attributes.application, require=True) + + # 2. determine dns + requested_name = hostname_for_deployment_request(app_deployment_request, laconic) + + # HACK + if "." in requested_name: + raise Exception("Only unqualified hostnames allowed at this time.") + + fqdn = f"{requested_name}.{dns_suffix}" + container_tag = "%s:local" % app.attributes.name.replace("@", "") + + # 3. check ownership of existing dnsrecord vs this request + # TODO: Support foreign DNS + dns_crn = f"{dns_record_namespace}/{fqdn}" + dns_record = laconic.get_record(dns_crn) + if dns_record: + dns_record_owners = dns_record.owners + dns_request_owners = [] + if dns_record.request: + prev_request = laconic.get_record(dns_record.request, require=True) + dns_request_owners = prev_request.owners + + owner_match = None + + for owner in app_deployment_request.owners: + if owner in dns_request_owners or owner in dns_record_owners: + owner_match = owner + if owner_match: + print("Matched DnsRecord ownership to", owner) + else: + raise Exception("Unable to confirm ownership of DnsRecord %s for request %s" % + (dns_record.id, app_deployment_request.id)) + + # 4. get build and runtime config from request + env_filename = None + if app_deployment_request.attributes.config and "env" in app_deployment_request.attributes.config: + env_filename = tempfile.mktemp() + with open(env_filename, 'w') as file: + for k, v in app_deployment_request.attributes.config["env"].items(): + file.write("%s=%s\n" % (k, shlex.quote(str(v)))) + + # 5. determine new or existing deployment + # a. check for deployment crn + app_deployment_crn = f"{deployment_record_namespace}/{fqdn}" + if app_deployment_request.attributes.deployment: + app_deployment_crn = app_deployment_request.attributes.deployment + if not app_deployment_crn.startswith(deployment_record_namespace): + raise Exception("Deployment CRN %s is not in a supported namespace" % app_deployment_request.attributes.deployment) + + deployment_record = laconic.get_record(app_deployment_crn) + deployment_dir = os.path.join(deployment_parent_dir, fqdn) + deployment_config_file = os.path.join(deployment_dir, "config.env") + # b. check for deployment directory (create if necessary) + if not os.path.exists(deployment_dir): + if deployment_record: + raise ("Deployment record %s exists, but not deployment dir %s. Please remove name." % + (app_deployment_crn, deployment_dir)) + print("deploy_webapp", deployment_dir) + deploy_webapp.create_deployment(ctx, deployment_dir, container_tag, + f"https://{fqdn}", kube_config, image_registry, env_filename) + elif env_filename: + shutil.copyfile(env_filename, deployment_config_file) + + needs_k8s_deploy = False + # 6. build container (if needed) + if not deployment_record or deployment_record.attributes.application != app.id: + build_container_image(app, container_tag) + push_container_image(deployment_dir) + needs_k8s_deploy = True + + # 7. update config (if needed) + if not deployment_record or file_hash(deployment_config_file) != deployment_record.attributes.meta.config: + needs_k8s_deploy = True + + # 8. update k8s deployment + if needs_k8s_deploy: + print("Deploying to k8s") + deploy_to_k8s( + deployment_record, + deployment_dir, + ) + + publish_deployment( + laconic, + app, + deployment_record, + app_deployment_crn, + dns_record, + dns_crn, + deployment_dir, + app_deployment_request + ) + + +def load_known_requests(filename): + if filename and os.path.exists(filename): + return json.load(open(filename, "r")) + return {} + + +def dump_known_requests(filename, requests): + if not filename: + return + known_requests = load_known_requests(filename) + for r in requests: + known_requests[r.id] = r.createTime + json.dump(known_requests, open(filename, "w")) + + +@click.command() +@click.option("--kube-config", help="Provide a config file for a k8s deployment") +@click.option("--kube-config", help="Provide a config file for a k8s deployment") +@click.option("--laconic-config", help="Provide a config file for laconicd", required=True) +@click.option("--image-registry", help="Provide a container image registry url for this k8s cluster") +@click.option("--deployment-parent-dir", help="Create deployment directories beneath this directory", required=True) +@click.option("--request-id", help="The ApplicationDeploymentRequest to process") +@click.option("--discover", help="Discover and process all pending ApplicationDeploymentRequests", is_flag=True, default=False) +@click.option("--state-file", help="File to store state about previously seen requests.") +@click.option("--only-update-state", help="Only update the state file, don't process any requests anything.", is_flag=True) +@click.option("--dns-suffix", help="DNS domain to use eg, laconic.servesthe.world") +@click.option("--record-namespace-dns", help="eg, crn://laconic/dns") +@click.option("--record-namespace-deployments", help="eg, crn://laconic/deployments") +@click.pass_context +def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir, + request_id, discover, state_file, only_update_state, + dns_suffix, record_namespace_dns, record_namespace_deployments): + if request_id and discover: + print("Cannot specify both --request-id and --discover", file=sys.stderr) + sys.exit(2) + + if not request_id and not discover: + print("Must specify either --request-id or --discover", file=sys.stderr) + sys.exit(2) + + if only_update_state and not state_file: + print("--only-update-state requires --state-file", file=sys.stderr) + sys.exit(2) + + if not only_update_state: + if not record_namespace_dns or not record_namespace_deployments or not dns_suffix: + print("--dns-suffix, --record-namespace-dns, and --record-namespace-deployments are all required", file=sys.stderr) + sys.exit(2) + + laconic = LaconicRegistryClient(laconic_config) + + # Find deployment requests. + # single request + if request_id: + requests = [laconic.get_record(request_id, require=True)] + # all requests + elif discover: + requests = laconic.app_deployment_requests() + + if only_update_state: + dump_known_requests(state_file, requests) + return + + previous_requests = load_known_requests(state_file) + + # Collapse related requests. + requests.sort(key=lambda r: r.createTime) + requests.reverse() + requests_by_name = {} + for r in requests: + app = laconic.get_record(r.attributes.application) + if not app: + print("Skipping request %s, cannot locate app." % r.id) + continue + + requested_name = r.attributes.dns + if not requested_name: + requested_name = generate_hostname_for_app(app) + print("Generating name %s for request %s." % (requested_name, r.id)) + + if requested_name not in requests_by_name: + print( + "Found request %s to run application %s on %s." + % (r.id, r.attributes.application, requested_name) + ) + requests_by_name[requested_name] = r + else: + print( + "Ignoring request %s, it is superseded by %s." + % (r.id, requests_by_name[requested_name].id) + ) + + # Find deployments. + deployments = laconic.app_deployments() + deployments_by_request = {} + for d in deployments: + if d.attributes.request: + deployments_by_request[d.attributes.request] = d + + requests_to_execute = [] + for r in requests_by_name.values(): + if r.id not in deployments_by_request: + if r.id not in previous_requests: + print(f"Request {r.id} needs to processed.") + requests_to_execute.append(r) + else: + print( + f"Skipping unsatisfied request {r.id} because we have seen it before." + ) + else: + print(f"Found satisfied request {r.id} at {deployments_by_request[r.id].names[0]}") + + print("Found %d unsatisfied request(s) to process." % len(requests_to_execute)) + + for r in requests_to_execute: + try: + process_app_deployment_request( + ctx, + laconic, + r, + record_namespace_deployments, + record_namespace_dns, + dns_suffix, + deployment_parent_dir, + kube_config, + image_registry + ) + finally: + dump_known_requests(state_file, [r]) diff --git a/stack_orchestrator/deploy/webapp/util.py b/stack_orchestrator/deploy/webapp/util.py new file mode 100644 index 00000000..ebc14c3f --- /dev/null +++ b/stack_orchestrator/deploy/webapp/util.py @@ -0,0 +1,303 @@ +# Copyright © 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import hashlib +import json +import os +import random +import subprocess +import sys +import tempfile +import uuid + +import yaml + + +class AttrDict(dict): + def __init__(self, *args, **kwargs): + super(AttrDict, self).__init__(*args, **kwargs) + self.__dict__ = self + + def __getattribute__(self, attr): + __dict__ = super(AttrDict, self).__getattribute__("__dict__") + if attr in __dict__: + v = super(AttrDict, self).__getattribute__(attr) + if isinstance(v, dict): + return AttrDict(v) + return v + + +def cmd(*vargs): + try: + result = subprocess.run(vargs, capture_output=True) + result.check_returncode() + return result.stdout.decode() + except Exception as err: + print(result.stderr.decode()) + raise err + + +class LaconicRegistryClient: + def __init__(self, config_file): + self.config_file = config_file + self.cache = AttrDict( + { + "name_or_id": {}, + } + ) + + def list_records(self, criteria={}, all=False): + args = ["laconic", "-c", self.config_file, "cns", "record", "list"] + + if all: + args.append("--all") + + if criteria: + for k, v in criteria.items(): + args.append("--%s" % k) + args.append(str(v)) + + results = [AttrDict(r) for r in json.loads(cmd(*args))] + + # Most recent records first + results.sort(key=lambda r: r.createTime) + results.reverse() + + return results + + def is_crn(self, name_or_id: str): + if name_or_id: + return str(name_or_id).startswith("crn://") + return False + + def is_id(self, name_or_id: str): + return not self.is_crn(name_or_id) + + def _add_to_cache(self, records): + if not records: + return + + for p in records: + self.cache["name_or_id"][p.id] = p + if p.names: + for crn in p.names: + self.cache["name_or_id"][crn] = p + if p.attributes.type not in self.cache: + self.cache[p.attributes.type] = [] + self.cache[p.attributes.type].append(p) + + def resolve(self, name): + if not name: + return None + + if name in self.cache.name_or_id: + return self.cache.name_or_id[name] + + args = ["laconic", "-c", self.config_file, "cns", "name", "resolve", name] + + parsed = [AttrDict(r) for r in json.loads(cmd(*args))] + if parsed: + self._add_to_cache(parsed) + return parsed[0] + + return None + + def get_record(self, name_or_id, require=False): + if not name_or_id: + if require: + raise Exception("Cannot locate record:", name_or_id) + return None + + if name_or_id in self.cache.name_or_id: + return self.cache.name_or_id[name_or_id] + + if self.is_crn(name_or_id): + return self.resolve(name_or_id) + + args = [ + "laconic", + "-c", + self.config_file, + "cns", + "record", + "get", + "--id", + name_or_id, + ] + + parsed = [AttrDict(r) for r in json.loads(cmd(*args))] + if len(parsed): + self._add_to_cache(parsed) + return parsed[0] + + if require: + raise Exception("Cannot locate record:", name_or_id) + return None + + def app_deployment_requests(self): + return self.list_records({"type": "ApplicationDeploymentRequest"}, True) + + def app_deployments(self): + return self.list_records({"type": "ApplicationDeploymentRecord"}) + + def publish(self, record, names=[]): + tmpdir = tempfile.mkdtemp() + try: + record_fname = os.path.join(tmpdir, "record.yml") + record_file = open(record_fname, 'w') + yaml.dump(record, record_file) + record_file.close() + print(open(record_fname, 'r').read()) + + new_record_id = json.loads( + cmd("laconic", "-c", self.config_file, "cns", "record", "publish", "--filename", record_fname) + )["id"] + for name in names: + cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, new_record_id) + return new_record_id + finally: + cmd("rm", "-rf", tmpdir) + + +def file_hash(filename): + return hashlib.sha1(open(filename).read().encode()).hexdigest() + + +def build_container_image(app_record, tag, extra_build_args=[]): + tmpdir = tempfile.mkdtemp() + + try: + record_id = app_record["id"] + ref = app_record.attributes.repository_ref + repo = random.choice(app_record.attributes.repository) + clone_dir = os.path.join(tmpdir, record_id) + + print(f"Cloning repository {repo} to {clone_dir} ...") + if ref: + result = subprocess.run(["git", "clone", "--depth", "1", "--branch", ref, repo, clone_dir]) + result.check_returncode() + else: + result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir]) + result.check_returncode() + + print("Building webapp ...") + build_command = [sys.argv[0], "build-webapp", "--source-repo", clone_dir, "--tag", tag] + if extra_build_args: + build_command.append("--extra-build-args") + build_command.append(" ".join(extra_build_args)) + + result = subprocess.run(build_command) + result.check_returncode() + finally: + cmd("rm", "-rf", tmpdir) + + +def push_container_image(deployment_dir): + print("Pushing image ...") + result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"]) + result.check_returncode() + + +def deploy_to_k8s(deploy_record, deployment_dir): + if not deploy_record: + command = "up" + else: + command = "update" + + result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command]) + result.check_returncode() + + +def publish_deployment(laconic: LaconicRegistryClient, + app_record, + deploy_record, + deployment_crn, + dns_record, + dns_crn, + deployment_dir, + app_deployment_request=None): + if not deploy_record: + deploy_ver = "0.0.1" + else: + deploy_ver = "0.0.%d" % (int(deploy_record.attributes.version.split(".")[-1]) + 1) + + if not dns_record: + dns_ver = "0.0.1" + else: + dns_ver = "0.0.%d" % (int(dns_record.attributes.version.split(".")[-1]) + 1) + + spec = yaml.full_load(open(os.path.join(deployment_dir, "spec.yml"))) + fqdn = spec["network"]["http-proxy"][0]["host-name"] + + uniq = uuid.uuid4() + + new_dns_record = { + "record": { + "type": "DnsRecord", + "version": dns_ver, + "name": fqdn, + "resource_type": "A", + "meta": { + "so": uniq.hex + }, + } + } + if app_deployment_request: + new_dns_record["record"]["request"] = app_deployment_request.id + + dns_id = laconic.publish(new_dns_record, [dns_crn]) + + new_deployment_record = { + "record": { + "type": "ApplicationDeploymentRecord", + "version": deploy_ver, + "url": f"https://{fqdn}", + "name": app_record.attributes.name, + "application": app_record.id, + "dns": dns_id, + "meta": { + "config": file_hash(os.path.join(deployment_dir, "config.env")), + "so": uniq.hex + }, + } + } + if app_deployment_request: + new_deployment_record["record"]["request"] = app_deployment_request.id + + deployment_id = laconic.publish(new_deployment_record, [deployment_crn]) + return {"dns": dns_id, "deployment": deployment_id} + + +def hostname_for_deployment_request(app_deployment_request, laconic): + dns_name = app_deployment_request.attributes.dns + if not dns_name: + app = laconic.get_record(app_deployment_request.attributes.application, require=True) + dns_name = generate_hostname_for_app(app) + elif dns_name.startswith("crn://"): + record = laconic.get_record(dns_name, require=True) + dns_name = record.attributes.name + return dns_name + + +def generate_hostname_for_app(app): + last_part = app.attributes.name.split("/")[-1] + m = hashlib.sha256() + m.update(app.attributes.name.encode()) + m.update(b"|") + if isinstance(app.attributes.repository, list): + m.update(app.attributes.repository[0].encode()) + else: + m.update(app.attributes.repository.encode()) + return "%s-%s" % (last_part, m.hexdigest()[0:10]) diff --git a/stack_orchestrator/main.py b/stack_orchestrator/main.py index 26a011b0..fabc58ff 100644 --- a/stack_orchestrator/main.py +++ b/stack_orchestrator/main.py @@ -20,7 +20,7 @@ from stack_orchestrator.repos import setup_repositories from stack_orchestrator.build import build_containers from stack_orchestrator.build import build_npms from stack_orchestrator.build import build_webapp -from stack_orchestrator.deploy.webapp import run_webapp, deploy_webapp +from stack_orchestrator.deploy.webapp import run_webapp, deploy_webapp, deploy_webapp_from_registry from stack_orchestrator.deploy import deploy from stack_orchestrator import version from stack_orchestrator.deploy import deployment @@ -53,6 +53,7 @@ cli.add_command(build_npms.command, "build-npms") cli.add_command(build_webapp.command, "build-webapp") cli.add_command(run_webapp.command, "run-webapp") cli.add_command(deploy_webapp.command, "deploy-webapp") +cli.add_command(deploy_webapp_from_registry.command, "deploy-webapp-from-registry") cli.add_command(deploy.command, "deploy") # deploy is an alias for deploy-system cli.add_command(deploy.command, "deploy-system") cli.add_command(deployment.command, "deployment") From 1f4eb57069396a620dc9a1bcbc90a5441bc71833 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Wed, 13 Dec 2023 22:56:40 -0600 Subject: [PATCH 48/75] Add --dry-run option (#686) --- .../webapp/deploy_webapp_from_registry.py | 37 +- stack_orchestrator/deploy/webapp/util.py | 608 +++++++++--------- 2 files changed, 325 insertions(+), 320 deletions(-) diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py index 5f9e712b..ad439812 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -164,10 +164,11 @@ def dump_known_requests(filename, requests): @click.option("--dns-suffix", help="DNS domain to use eg, laconic.servesthe.world") @click.option("--record-namespace-dns", help="eg, crn://laconic/dns") @click.option("--record-namespace-deployments", help="eg, crn://laconic/deployments") +@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True) @click.pass_context def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir, request_id, discover, state_file, only_update_state, - dns_suffix, record_namespace_dns, record_namespace_deployments): + dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run): if request_id and discover: print("Cannot specify both --request-id and --discover", file=sys.stderr) sys.exit(2) @@ -196,7 +197,8 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_ requests = laconic.app_deployment_requests() if only_update_state: - dump_known_requests(state_file, requests) + if not dry_run: + dump_known_requests(state_file, requests) return previous_requests = load_known_requests(state_file) @@ -250,18 +252,19 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_ print("Found %d unsatisfied request(s) to process." % len(requests_to_execute)) - for r in requests_to_execute: - try: - process_app_deployment_request( - ctx, - laconic, - r, - record_namespace_deployments, - record_namespace_dns, - dns_suffix, - deployment_parent_dir, - kube_config, - image_registry - ) - finally: - dump_known_requests(state_file, [r]) + if not dry_run: + for r in requests_to_execute: + try: + process_app_deployment_request( + ctx, + laconic, + r, + record_namespace_deployments, + record_namespace_dns, + dns_suffix, + deployment_parent_dir, + kube_config, + image_registry + ) + finally: + dump_known_requests(state_file, [r]) diff --git a/stack_orchestrator/deploy/webapp/util.py b/stack_orchestrator/deploy/webapp/util.py index ebc14c3f..f95d3f3e 100644 --- a/stack_orchestrator/deploy/webapp/util.py +++ b/stack_orchestrator/deploy/webapp/util.py @@ -1,303 +1,305 @@ -# Copyright © 2023 Vulcanize - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. - -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import hashlib -import json -import os -import random -import subprocess -import sys -import tempfile -import uuid - -import yaml - - -class AttrDict(dict): - def __init__(self, *args, **kwargs): - super(AttrDict, self).__init__(*args, **kwargs) - self.__dict__ = self - - def __getattribute__(self, attr): - __dict__ = super(AttrDict, self).__getattribute__("__dict__") - if attr in __dict__: - v = super(AttrDict, self).__getattribute__(attr) - if isinstance(v, dict): - return AttrDict(v) - return v - - -def cmd(*vargs): - try: - result = subprocess.run(vargs, capture_output=True) - result.check_returncode() - return result.stdout.decode() - except Exception as err: - print(result.stderr.decode()) - raise err - - -class LaconicRegistryClient: - def __init__(self, config_file): - self.config_file = config_file - self.cache = AttrDict( - { - "name_or_id": {}, - } - ) - - def list_records(self, criteria={}, all=False): - args = ["laconic", "-c", self.config_file, "cns", "record", "list"] - - if all: - args.append("--all") - - if criteria: - for k, v in criteria.items(): - args.append("--%s" % k) - args.append(str(v)) - - results = [AttrDict(r) for r in json.loads(cmd(*args))] - - # Most recent records first - results.sort(key=lambda r: r.createTime) - results.reverse() - - return results - - def is_crn(self, name_or_id: str): - if name_or_id: - return str(name_or_id).startswith("crn://") - return False - - def is_id(self, name_or_id: str): - return not self.is_crn(name_or_id) - - def _add_to_cache(self, records): - if not records: - return - - for p in records: - self.cache["name_or_id"][p.id] = p - if p.names: - for crn in p.names: - self.cache["name_or_id"][crn] = p - if p.attributes.type not in self.cache: - self.cache[p.attributes.type] = [] - self.cache[p.attributes.type].append(p) - - def resolve(self, name): - if not name: - return None - - if name in self.cache.name_or_id: - return self.cache.name_or_id[name] - - args = ["laconic", "-c", self.config_file, "cns", "name", "resolve", name] - - parsed = [AttrDict(r) for r in json.loads(cmd(*args))] - if parsed: - self._add_to_cache(parsed) - return parsed[0] - - return None - - def get_record(self, name_or_id, require=False): - if not name_or_id: - if require: - raise Exception("Cannot locate record:", name_or_id) - return None - - if name_or_id in self.cache.name_or_id: - return self.cache.name_or_id[name_or_id] - - if self.is_crn(name_or_id): - return self.resolve(name_or_id) - - args = [ - "laconic", - "-c", - self.config_file, - "cns", - "record", - "get", - "--id", - name_or_id, - ] - - parsed = [AttrDict(r) for r in json.loads(cmd(*args))] - if len(parsed): - self._add_to_cache(parsed) - return parsed[0] - - if require: - raise Exception("Cannot locate record:", name_or_id) - return None - - def app_deployment_requests(self): - return self.list_records({"type": "ApplicationDeploymentRequest"}, True) - - def app_deployments(self): - return self.list_records({"type": "ApplicationDeploymentRecord"}) - - def publish(self, record, names=[]): - tmpdir = tempfile.mkdtemp() - try: - record_fname = os.path.join(tmpdir, "record.yml") - record_file = open(record_fname, 'w') - yaml.dump(record, record_file) - record_file.close() - print(open(record_fname, 'r').read()) - - new_record_id = json.loads( - cmd("laconic", "-c", self.config_file, "cns", "record", "publish", "--filename", record_fname) - )["id"] - for name in names: - cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, new_record_id) - return new_record_id - finally: - cmd("rm", "-rf", tmpdir) - - -def file_hash(filename): - return hashlib.sha1(open(filename).read().encode()).hexdigest() - - -def build_container_image(app_record, tag, extra_build_args=[]): - tmpdir = tempfile.mkdtemp() - - try: - record_id = app_record["id"] - ref = app_record.attributes.repository_ref - repo = random.choice(app_record.attributes.repository) - clone_dir = os.path.join(tmpdir, record_id) - - print(f"Cloning repository {repo} to {clone_dir} ...") - if ref: - result = subprocess.run(["git", "clone", "--depth", "1", "--branch", ref, repo, clone_dir]) - result.check_returncode() - else: - result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir]) - result.check_returncode() - - print("Building webapp ...") - build_command = [sys.argv[0], "build-webapp", "--source-repo", clone_dir, "--tag", tag] - if extra_build_args: - build_command.append("--extra-build-args") - build_command.append(" ".join(extra_build_args)) - - result = subprocess.run(build_command) - result.check_returncode() - finally: - cmd("rm", "-rf", tmpdir) - - -def push_container_image(deployment_dir): - print("Pushing image ...") - result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"]) - result.check_returncode() - - -def deploy_to_k8s(deploy_record, deployment_dir): - if not deploy_record: - command = "up" - else: - command = "update" - - result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command]) - result.check_returncode() - - -def publish_deployment(laconic: LaconicRegistryClient, - app_record, - deploy_record, - deployment_crn, - dns_record, - dns_crn, - deployment_dir, - app_deployment_request=None): - if not deploy_record: - deploy_ver = "0.0.1" - else: - deploy_ver = "0.0.%d" % (int(deploy_record.attributes.version.split(".")[-1]) + 1) - - if not dns_record: - dns_ver = "0.0.1" - else: - dns_ver = "0.0.%d" % (int(dns_record.attributes.version.split(".")[-1]) + 1) - - spec = yaml.full_load(open(os.path.join(deployment_dir, "spec.yml"))) - fqdn = spec["network"]["http-proxy"][0]["host-name"] - - uniq = uuid.uuid4() - - new_dns_record = { - "record": { - "type": "DnsRecord", - "version": dns_ver, - "name": fqdn, - "resource_type": "A", - "meta": { - "so": uniq.hex - }, - } - } - if app_deployment_request: - new_dns_record["record"]["request"] = app_deployment_request.id - - dns_id = laconic.publish(new_dns_record, [dns_crn]) - - new_deployment_record = { - "record": { - "type": "ApplicationDeploymentRecord", - "version": deploy_ver, - "url": f"https://{fqdn}", - "name": app_record.attributes.name, - "application": app_record.id, - "dns": dns_id, - "meta": { - "config": file_hash(os.path.join(deployment_dir, "config.env")), - "so": uniq.hex - }, - } - } - if app_deployment_request: - new_deployment_record["record"]["request"] = app_deployment_request.id - - deployment_id = laconic.publish(new_deployment_record, [deployment_crn]) - return {"dns": dns_id, "deployment": deployment_id} - - -def hostname_for_deployment_request(app_deployment_request, laconic): - dns_name = app_deployment_request.attributes.dns - if not dns_name: - app = laconic.get_record(app_deployment_request.attributes.application, require=True) - dns_name = generate_hostname_for_app(app) - elif dns_name.startswith("crn://"): - record = laconic.get_record(dns_name, require=True) - dns_name = record.attributes.name - return dns_name - - -def generate_hostname_for_app(app): - last_part = app.attributes.name.split("/")[-1] - m = hashlib.sha256() - m.update(app.attributes.name.encode()) - m.update(b"|") - if isinstance(app.attributes.repository, list): - m.update(app.attributes.repository[0].encode()) - else: - m.update(app.attributes.repository.encode()) - return "%s-%s" % (last_part, m.hexdigest()[0:10]) +# Copyright © 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import hashlib +import json +import os +import random +import subprocess +import sys +import tempfile +import uuid + +import yaml + + +class AttrDict(dict): + def __init__(self, *args, **kwargs): + super(AttrDict, self).__init__(*args, **kwargs) + self.__dict__ = self + + def __getattribute__(self, attr): + __dict__ = super(AttrDict, self).__getattribute__("__dict__") + if attr in __dict__: + v = super(AttrDict, self).__getattribute__(attr) + if isinstance(v, dict): + return AttrDict(v) + return v + + +def cmd(*vargs): + try: + result = subprocess.run(vargs, capture_output=True) + result.check_returncode() + return result.stdout.decode() + except Exception as err: + print(result.stderr.decode()) + raise err + + +class LaconicRegistryClient: + def __init__(self, config_file): + self.config_file = config_file + self.cache = AttrDict( + { + "name_or_id": {}, + } + ) + + def list_records(self, criteria={}, all=False): + args = ["laconic", "-c", self.config_file, "cns", "record", "list"] + + if all: + args.append("--all") + + if criteria: + for k, v in criteria.items(): + args.append("--%s" % k) + args.append(str(v)) + + results = [AttrDict(r) for r in json.loads(cmd(*args))] + + # Most recent records first + results.sort(key=lambda r: r.createTime) + results.reverse() + + return results + + def is_crn(self, name_or_id: str): + if name_or_id: + return str(name_or_id).startswith("crn://") + return False + + def is_id(self, name_or_id: str): + return not self.is_crn(name_or_id) + + def _add_to_cache(self, records): + if not records: + return + + for p in records: + self.cache["name_or_id"][p.id] = p + if p.names: + for crn in p.names: + self.cache["name_or_id"][crn] = p + if p.attributes.type not in self.cache: + self.cache[p.attributes.type] = [] + self.cache[p.attributes.type].append(p) + + def resolve(self, name): + if not name: + return None + + if name in self.cache.name_or_id: + return self.cache.name_or_id[name] + + args = ["laconic", "-c", self.config_file, "cns", "name", "resolve", name] + + parsed = [AttrDict(r) for r in json.loads(cmd(*args))] + if parsed: + self._add_to_cache(parsed) + return parsed[0] + + return None + + def get_record(self, name_or_id, require=False): + if not name_or_id: + if require: + raise Exception("Cannot locate record:", name_or_id) + return None + + if name_or_id in self.cache.name_or_id: + return self.cache.name_or_id[name_or_id] + + if self.is_crn(name_or_id): + return self.resolve(name_or_id) + + args = [ + "laconic", + "-c", + self.config_file, + "cns", + "record", + "get", + "--id", + name_or_id, + ] + + parsed = [AttrDict(r) for r in json.loads(cmd(*args))] + if len(parsed): + self._add_to_cache(parsed) + return parsed[0] + + if require: + raise Exception("Cannot locate record:", name_or_id) + return None + + def app_deployment_requests(self): + return self.list_records({"type": "ApplicationDeploymentRequest"}, True) + + def app_deployments(self): + return self.list_records({"type": "ApplicationDeploymentRecord"}) + + def publish(self, record, names=[]): + tmpdir = tempfile.mkdtemp() + try: + record_fname = os.path.join(tmpdir, "record.yml") + record_file = open(record_fname, 'w') + yaml.dump(record, record_file) + record_file.close() + print(open(record_fname, 'r').read()) + + new_record_id = json.loads( + cmd("laconic", "-c", self.config_file, "cns", "record", "publish", "--filename", record_fname) + )["id"] + for name in names: + cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, new_record_id) + return new_record_id + finally: + cmd("rm", "-rf", tmpdir) + + +def file_hash(filename): + return hashlib.sha1(open(filename).read().encode()).hexdigest() + + +def build_container_image(app_record, tag, extra_build_args=[]): + tmpdir = tempfile.mkdtemp() + + try: + record_id = app_record["id"] + ref = app_record.attributes.repository_ref + repo = random.choice(app_record.attributes.repository) + clone_dir = os.path.join(tmpdir, record_id) + + print(f"Cloning repository {repo} to {clone_dir} ...") + if ref: + # TODO: Determing branch or hash, and use depth 1 if we can. + result = subprocess.run(["git", "clone", repo, clone_dir]) + result.check_returncode() + subprocess.check_call(["git", "checkout", ref], cwd=clone_dir) + else: + result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir]) + result.check_returncode() + + print("Building webapp ...") + build_command = [sys.argv[0], "build-webapp", "--source-repo", clone_dir, "--tag", tag] + if extra_build_args: + build_command.append("--extra-build-args") + build_command.append(" ".join(extra_build_args)) + + result = subprocess.run(build_command) + result.check_returncode() + finally: + cmd("rm", "-rf", tmpdir) + + +def push_container_image(deployment_dir): + print("Pushing image ...") + result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"]) + result.check_returncode() + + +def deploy_to_k8s(deploy_record, deployment_dir): + if not deploy_record: + command = "up" + else: + command = "update" + + result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command]) + result.check_returncode() + + +def publish_deployment(laconic: LaconicRegistryClient, + app_record, + deploy_record, + deployment_crn, + dns_record, + dns_crn, + deployment_dir, + app_deployment_request=None): + if not deploy_record: + deploy_ver = "0.0.1" + else: + deploy_ver = "0.0.%d" % (int(deploy_record.attributes.version.split(".")[-1]) + 1) + + if not dns_record: + dns_ver = "0.0.1" + else: + dns_ver = "0.0.%d" % (int(dns_record.attributes.version.split(".")[-1]) + 1) + + spec = yaml.full_load(open(os.path.join(deployment_dir, "spec.yml"))) + fqdn = spec["network"]["http-proxy"][0]["host-name"] + + uniq = uuid.uuid4() + + new_dns_record = { + "record": { + "type": "DnsRecord", + "version": dns_ver, + "name": fqdn, + "resource_type": "A", + "meta": { + "so": uniq.hex + }, + } + } + if app_deployment_request: + new_dns_record["record"]["request"] = app_deployment_request.id + + dns_id = laconic.publish(new_dns_record, [dns_crn]) + + new_deployment_record = { + "record": { + "type": "ApplicationDeploymentRecord", + "version": deploy_ver, + "url": f"https://{fqdn}", + "name": app_record.attributes.name, + "application": app_record.id, + "dns": dns_id, + "meta": { + "config": file_hash(os.path.join(deployment_dir, "config.env")), + "so": uniq.hex + }, + } + } + if app_deployment_request: + new_deployment_record["record"]["request"] = app_deployment_request.id + + deployment_id = laconic.publish(new_deployment_record, [deployment_crn]) + return {"dns": dns_id, "deployment": deployment_id} + + +def hostname_for_deployment_request(app_deployment_request, laconic): + dns_name = app_deployment_request.attributes.dns + if not dns_name: + app = laconic.get_record(app_deployment_request.attributes.application, require=True) + dns_name = generate_hostname_for_app(app) + elif dns_name.startswith("crn://"): + record = laconic.get_record(dns_name, require=True) + dns_name = record.attributes.name + return dns_name + + +def generate_hostname_for_app(app): + last_part = app.attributes.name.split("/")[-1] + m = hashlib.sha256() + m.update(app.attributes.name.encode()) + m.update(b"|") + if isinstance(app.attributes.repository, list): + m.update(app.attributes.repository[0].encode()) + else: + m.update(app.attributes.repository.encode()) + return "%s-%s" % (last_part, m.hexdigest()[0:10]) From 9ba410b095bfa5c5814c183af6a258a4ef036b2d Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Thu, 14 Dec 2023 16:59:00 +0530 Subject: [PATCH 49/75] Add a stack for monitoring watchers with prometheus and grafana (#687) * Setup Prometheus and Grafana for monitoring stack * Add dashboard for azimuth watchers * Add a dashboard for sushiswap watcher * Persist prometheus server data * Additional metrics in watcher dashboards * Update dashboards and add for merkl sushiswap watcher * Add dashboards for remaining azimuth watchers * Separate out preconfigured watcher dashboards and add instructions * Keep the empty dashboards dir --- .../data/compose/docker-compose-grafana.yml | 21 + .../compose/docker-compose-prom-server.yml | 22 + .../monitoring/grafana/dashboards/.gitkeep | 0 .../provisioning/dashboards/dashboards.yml | 9 + .../provisioning/datasources/prometheus.yml | 16 + .../grafana/watcher-dashboard-template.json | 887 +++++++++++++ .../azimuth/azimuth-watcher-dashboard.json | 887 +++++++++++++ .../azimuth/censures-watcher-dashboard.json | 887 +++++++++++++ .../azimuth/claims-watcher-dashboard.json | 887 +++++++++++++ ...tional-star-release-watcher-dashboard.json | 887 +++++++++++++ ...ted-sending-watcher-watcher-dashboard.json | 887 +++++++++++++ .../azimuth/ecliptic-watcher-dashboard.json | 887 +++++++++++++ ...linear-star-release-watcher-dashboard.json | 887 +++++++++++++ .../azimuth/polls-watcher-dashboard.json | 887 +++++++++++++ .../merkl-sushiswap-watcher-dashboard.json | 1179 +++++++++++++++++ .../sushi/sushiswap-watcher-dashboard.json | 1179 +++++++++++++++++ .../monitoring/prometheus/prometheus.yml | 12 + .../data/stacks/monitoring/README.md | 94 ++ .../stacks/monitoring/monitoring-watchers.md | 124 ++ .../data/stacks/monitoring/stack.yml | 7 + 20 files changed, 10646 insertions(+) create mode 100644 stack_orchestrator/data/compose/docker-compose-grafana.yml create mode 100644 stack_orchestrator/data/compose/docker-compose-prom-server.yml create mode 100644 stack_orchestrator/data/config/monitoring/grafana/dashboards/.gitkeep create mode 100644 stack_orchestrator/data/config/monitoring/grafana/provisioning/dashboards/dashboards.yml create mode 100644 stack_orchestrator/data/config/monitoring/grafana/provisioning/datasources/prometheus.yml create mode 100644 stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json create mode 100644 stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json create mode 100644 stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json create mode 100644 stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json create mode 100644 stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json create mode 100644 stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json create mode 100644 stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json create mode 100644 stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json create mode 100644 stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json create mode 100644 stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json create mode 100644 stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json create mode 100644 stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml create mode 100644 stack_orchestrator/data/stacks/monitoring/README.md create mode 100644 stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md create mode 100644 stack_orchestrator/data/stacks/monitoring/stack.yml diff --git a/stack_orchestrator/data/compose/docker-compose-grafana.yml b/stack_orchestrator/data/compose/docker-compose-grafana.yml new file mode 100644 index 00000000..4aa8d1e1 --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-grafana.yml @@ -0,0 +1,21 @@ +version: "3.7" + +services: + grafana: + image: grafana/grafana + restart: always + volumes: + - ../config/monitoring/grafana/provisioning:/etc/grafana/provisioning + - ../config/monitoring/grafana/dashboards:/etc/grafana/dashboards + - grafana_storage:/var/lib/grafana + ports: + - "3000" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "3000"] + interval: 30s + timeout: 5s + retries: 10 + start_period: 3s + +volumes: + grafana_storage: diff --git a/stack_orchestrator/data/compose/docker-compose-prom-server.yml b/stack_orchestrator/data/compose/docker-compose-prom-server.yml new file mode 100644 index 00000000..26a61e88 --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-prom-server.yml @@ -0,0 +1,22 @@ +version: "3.7" + +services: + prometheus: + image: prom/prometheus + restart: always + volumes: + - ../config/monitoring/prometheus:/etc/prometheus + - prometheus_data:/prometheus + ports: + - "9090" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "9090"] + interval: 30s + timeout: 5s + retries: 10 + start_period: 3s + extra_hosts: + - "host.docker.internal:host-gateway" + +volumes: + prometheus_data: diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/.gitkeep b/stack_orchestrator/data/config/monitoring/grafana/dashboards/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/stack_orchestrator/data/config/monitoring/grafana/provisioning/dashboards/dashboards.yml b/stack_orchestrator/data/config/monitoring/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 00000000..4705688e --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +providers: + - name: dashboards + type: file + updateIntervalSeconds: 10 + options: + path: /etc/grafana/dashboards + foldersFromFilesStructure: true diff --git a/stack_orchestrator/data/config/monitoring/grafana/provisioning/datasources/prometheus.yml b/stack_orchestrator/data/config/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 00000000..b502d2e0 --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,16 @@ +apiVersion: 1 + +datasources: + - id: 1 + orgId: 1 + name: Prometheus + type: prometheus + typeName: Prometheus + typeLogoUrl: public/app/plugins/datasource/prometheus/img/prometheus_logo.svg + access: proxy + url: http://prometheus:9090 + isDefault: true + jsonData: + httpMethod: POST + version: 1 + editable: true diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json new file mode 100644 index 00000000..03144776 --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json @@ -0,0 +1,887 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", kind=\"latest_indexed\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Latest indexed block", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head block number of upstream endpoint", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "title": "Upstream chain head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head from a public endpoint (etherscan)", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "title": "External chain head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-green", + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-green", + "value": null + }, + { + "color": "red", + "value": 35 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "last_block_process_duration_seconds{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Last block process duration (s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Watcher sync status block numbers", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 3 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_processed_block_number{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "latest_processed", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", kind=\"latest_canonical\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", kind=\"intial_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "C" + } + ], + "title": "Sync status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_block_num_events_total{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number of events in the last block", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of entries in the block_progress and event table", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "dark-yellow", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "block_progress_total{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "block_progress", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "event_total{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "event", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "DB indexed count ", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 7 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "database_size_bytes{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", type=\"watcher\"} / 1024 / 1024", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "database_size_bytes{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", type=\"job-queue\"} / 1024 / 1024", + "hide": false, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "B" + } + ], + "title": "DB size (MB)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num block-processing jobs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num event-processing jobs", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 38, + "tags": [ + "REPLACE_WITH_WATCHER_CHAIN", + "REPLACE_WITH_WATCHER_JOB" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "REPLACE_WITH_WATCHER_NAME watcher", + "version": 3, + "weekStart": "" +} diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json new file mode 100644 index 00000000..6f924b12 --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json @@ -0,0 +1,887 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"azimuth\", kind=\"latest_indexed\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Latest indexed block", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head block number of upstream endpoint", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "title": "Upstream chain head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head from a public endpoint (etherscan)", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "title": "External chain head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-green", + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-green", + "value": null + }, + { + "color": "red", + "value": 35 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "last_block_process_duration_seconds{job=\"azimuth\", instance=\"azimuth\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Last block process duration (s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Watcher sync status block numbers", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 3 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_processed_block_number{job=\"azimuth\", instance=\"azimuth\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "latest_processed", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"azimuth\", kind=\"latest_canonical\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"azimuth\", kind=\"intial_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "C" + } + ], + "title": "Sync status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_block_num_events_total{job=\"azimuth\", instance=\"azimuth\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number of events in the last block", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of entries in the block_progress and event table", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "dark-yellow", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "block_progress_total{job=\"azimuth\", instance=\"azimuth\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "block_progress", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "event_total{job=\"azimuth\", instance=\"azimuth\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "event", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "DB indexed count ", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 7 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "database_size_bytes{job=\"azimuth\", instance=\"azimuth\", type=\"watcher\"} / 1024 / 1024", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "database_size_bytes{job=\"azimuth\", instance=\"azimuth\", type=\"job-queue\"} / 1024 / 1024", + "hide": false, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "B" + } + ], + "title": "DB size (MB)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"azimuth\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num block-processing jobs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"azimuth\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num event-processing jobs", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 38, + "tags": [ + "ethereum", + "azimuth" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "azimuth watcher", + "version": 3, + "weekStart": "" +} diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json new file mode 100644 index 00000000..cbeb07ae --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json @@ -0,0 +1,887 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"censures\", kind=\"latest_indexed\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Latest indexed block", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head block number of upstream endpoint", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "title": "Upstream chain head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head from a public endpoint (etherscan)", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "title": "External chain head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-green", + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-green", + "value": null + }, + { + "color": "red", + "value": 35 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "last_block_process_duration_seconds{job=\"azimuth\", instance=\"censures\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Last block process duration (s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Watcher sync status block numbers", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 3 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_processed_block_number{job=\"azimuth\", instance=\"censures\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "latest_processed", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"censures\", kind=\"latest_canonical\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"censures\", kind=\"intial_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "C" + } + ], + "title": "Sync status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_block_num_events_total{job=\"azimuth\", instance=\"censures\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number of events in the last block", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of entries in the block_progress and event table", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "dark-yellow", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "block_progress_total{job=\"azimuth\", instance=\"censures\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "block_progress", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "event_total{job=\"azimuth\", instance=\"censures\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "event", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "DB indexed count ", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 7 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "database_size_bytes{job=\"azimuth\", instance=\"censures\", type=\"watcher\"} / 1024 / 1024", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "database_size_bytes{job=\"azimuth\", instance=\"censures\", type=\"job-queue\"} / 1024 / 1024", + "hide": false, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "B" + } + ], + "title": "DB size (MB)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"censures\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num block-processing jobs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"censures\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num event-processing jobs", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 38, + "tags": [ + "ethereum", + "azimuth" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "censures watcher", + "version": 3, + "weekStart": "" +} diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json new file mode 100644 index 00000000..29f18fe1 --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json @@ -0,0 +1,887 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"claims\", kind=\"latest_indexed\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Latest indexed block", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head block number of upstream endpoint", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "title": "Upstream chain head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head from a public endpoint (etherscan)", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "title": "External chain head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-green", + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-green", + "value": null + }, + { + "color": "red", + "value": 35 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "last_block_process_duration_seconds{job=\"azimuth\", instance=\"claims\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Last block process duration (s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Watcher sync status block numbers", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 3 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_processed_block_number{job=\"azimuth\", instance=\"claims\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "latest_processed", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"claims\", kind=\"latest_canonical\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"claims\", kind=\"intial_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "C" + } + ], + "title": "Sync status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_block_num_events_total{job=\"azimuth\", instance=\"claims\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number of events in the last block", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of entries in the block_progress and event table", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "dark-yellow", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "block_progress_total{job=\"azimuth\", instance=\"claims\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "block_progress", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "event_total{job=\"azimuth\", instance=\"claims\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "event", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "DB indexed count ", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 7 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "database_size_bytes{job=\"azimuth\", instance=\"claims\", type=\"watcher\"} / 1024 / 1024", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "database_size_bytes{job=\"azimuth\", instance=\"claims\", type=\"job-queue\"} / 1024 / 1024", + "hide": false, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "B" + } + ], + "title": "DB size (MB)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"claims\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num block-processing jobs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"claims\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num event-processing jobs", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 38, + "tags": [ + "ethereum", + "azimuth" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "claims watcher", + "version": 3, + "weekStart": "" +} diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json new file mode 100644 index 00000000..28ec488d --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json @@ -0,0 +1,887 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"conditional_star_release\", kind=\"latest_indexed\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Latest indexed block", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head block number of upstream endpoint", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "title": "Upstream chain head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head from a public endpoint (etherscan)", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "title": "External chain head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-green", + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-green", + "value": null + }, + { + "color": "red", + "value": 35 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "last_block_process_duration_seconds{job=\"azimuth\", instance=\"conditional_star_release\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Last block process duration (s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Watcher sync status block numbers", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 3 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_processed_block_number{job=\"azimuth\", instance=\"conditional_star_release\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "latest_processed", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"conditional_star_release\", kind=\"latest_canonical\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"conditional_star_release\", kind=\"intial_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "C" + } + ], + "title": "Sync status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_block_num_events_total{job=\"azimuth\", instance=\"conditional_star_release\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number of events in the last block", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of entries in the block_progress and event table", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "dark-yellow", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "block_progress_total{job=\"azimuth\", instance=\"conditional_star_release\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "block_progress", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "event_total{job=\"azimuth\", instance=\"conditional_star_release\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "event", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "DB indexed count ", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 7 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "database_size_bytes{job=\"azimuth\", instance=\"conditional_star_release\", type=\"watcher\"} / 1024 / 1024", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "database_size_bytes{job=\"azimuth\", instance=\"conditional_star_release\", type=\"job-queue\"} / 1024 / 1024", + "hide": false, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "B" + } + ], + "title": "DB size (MB)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"conditional_star_release\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num block-processing jobs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"conditional_star_release\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num event-processing jobs", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 38, + "tags": [ + "ethereum", + "azimuth" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "conditional star release watcher", + "version": 3, + "weekStart": "" +} diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json new file mode 100644 index 00000000..b7411c45 --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json @@ -0,0 +1,887 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"delegated_sending_watcher\", kind=\"latest_indexed\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Latest indexed block", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head block number of upstream endpoint", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "title": "Upstream chain head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head from a public endpoint (etherscan)", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "title": "External chain head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-green", + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-green", + "value": null + }, + { + "color": "red", + "value": 35 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "last_block_process_duration_seconds{job=\"azimuth\", instance=\"delegated_sending_watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Last block process duration (s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Watcher sync status block numbers", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 3 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_processed_block_number{job=\"azimuth\", instance=\"delegated_sending_watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "latest_processed", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"delegated_sending_watcher\", kind=\"latest_canonical\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"delegated_sending_watcher\", kind=\"intial_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "C" + } + ], + "title": "Sync status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_block_num_events_total{job=\"azimuth\", instance=\"delegated_sending_watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number of events in the last block", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of entries in the block_progress and event table", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "dark-yellow", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "block_progress_total{job=\"azimuth\", instance=\"delegated_sending_watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "block_progress", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "event_total{job=\"azimuth\", instance=\"delegated_sending_watcher\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "event", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "DB indexed count ", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 7 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "database_size_bytes{job=\"azimuth\", instance=\"delegated_sending_watcher\", type=\"watcher\"} / 1024 / 1024", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "database_size_bytes{job=\"azimuth\", instance=\"delegated_sending_watcher\", type=\"job-queue\"} / 1024 / 1024", + "hide": false, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "B" + } + ], + "title": "DB size (MB)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"delegated_sending_watcher\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num block-processing jobs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"delegated_sending_watcher\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num event-processing jobs", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 38, + "tags": [ + "ethereum", + "azimuth" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "delegated sending watcher watcher", + "version": 3, + "weekStart": "" +} diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json new file mode 100644 index 00000000..3ee7d151 --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json @@ -0,0 +1,887 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"ecliptic\", kind=\"latest_indexed\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Latest indexed block", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head block number of upstream endpoint", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "title": "Upstream chain head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head from a public endpoint (etherscan)", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "title": "External chain head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-green", + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-green", + "value": null + }, + { + "color": "red", + "value": 35 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "last_block_process_duration_seconds{job=\"azimuth\", instance=\"ecliptic\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Last block process duration (s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Watcher sync status block numbers", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 3 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_processed_block_number{job=\"azimuth\", instance=\"ecliptic\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "latest_processed", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"ecliptic\", kind=\"latest_canonical\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"ecliptic\", kind=\"intial_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "C" + } + ], + "title": "Sync status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_block_num_events_total{job=\"azimuth\", instance=\"ecliptic\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number of events in the last block", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of entries in the block_progress and event table", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "dark-yellow", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "block_progress_total{job=\"azimuth\", instance=\"ecliptic\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "block_progress", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "event_total{job=\"azimuth\", instance=\"ecliptic\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "event", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "DB indexed count ", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 7 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "database_size_bytes{job=\"azimuth\", instance=\"ecliptic\", type=\"watcher\"} / 1024 / 1024", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "database_size_bytes{job=\"azimuth\", instance=\"ecliptic\", type=\"job-queue\"} / 1024 / 1024", + "hide": false, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "B" + } + ], + "title": "DB size (MB)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"ecliptic\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num block-processing jobs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"ecliptic\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num event-processing jobs", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 38, + "tags": [ + "ethereum", + "azimuth" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "ecliptic watcher", + "version": 3, + "weekStart": "" +} diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json new file mode 100644 index 00000000..be6ef9ec --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json @@ -0,0 +1,887 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"linear_star_release\", kind=\"latest_indexed\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Latest indexed block", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head block number of upstream endpoint", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "title": "Upstream chain head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head from a public endpoint (etherscan)", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "title": "External chain head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-green", + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-green", + "value": null + }, + { + "color": "red", + "value": 35 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "last_block_process_duration_seconds{job=\"azimuth\", instance=\"linear_star_release\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Last block process duration (s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Watcher sync status block numbers", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 3 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_processed_block_number{job=\"azimuth\", instance=\"linear_star_release\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "latest_processed", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"linear_star_release\", kind=\"latest_canonical\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"linear_star_release\", kind=\"intial_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "C" + } + ], + "title": "Sync status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_block_num_events_total{job=\"azimuth\", instance=\"linear_star_release\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number of events in the last block", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of entries in the block_progress and event table", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "dark-yellow", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "block_progress_total{job=\"azimuth\", instance=\"linear_star_release\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "block_progress", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "event_total{job=\"azimuth\", instance=\"linear_star_release\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "event", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "DB indexed count ", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 7 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "database_size_bytes{job=\"azimuth\", instance=\"linear_star_release\", type=\"watcher\"} / 1024 / 1024", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "database_size_bytes{job=\"azimuth\", instance=\"linear_star_release\", type=\"job-queue\"} / 1024 / 1024", + "hide": false, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "B" + } + ], + "title": "DB size (MB)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"linear_star_release\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num block-processing jobs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"linear_star_release\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num event-processing jobs", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 38, + "tags": [ + "ethereum", + "azimuth" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "linear star release watcher", + "version": 3, + "weekStart": "" +} diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json new file mode 100644 index 00000000..01910d9a --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json @@ -0,0 +1,887 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"polls\", kind=\"latest_indexed\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Latest indexed block", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head block number of upstream endpoint", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "title": "Upstream chain head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head from a public endpoint (etherscan)", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "title": "External chain head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-green", + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-green", + "value": null + }, + { + "color": "red", + "value": 35 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "last_block_process_duration_seconds{job=\"azimuth\", instance=\"polls\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Last block process duration (s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Watcher sync status block numbers", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 3 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_processed_block_number{job=\"azimuth\", instance=\"polls\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "latest_processed", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"polls\", kind=\"latest_canonical\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"polls\", kind=\"intial_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "C" + } + ], + "title": "Sync status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_block_num_events_total{job=\"azimuth\", instance=\"polls\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number of events in the last block", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of entries in the block_progress and event table", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "dark-yellow", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "block_progress_total{job=\"azimuth\", instance=\"polls\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "block_progress", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "event_total{job=\"azimuth\", instance=\"polls\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "event", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "DB indexed count ", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 7 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "database_size_bytes{job=\"azimuth\", instance=\"polls\", type=\"watcher\"} / 1024 / 1024", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "database_size_bytes{job=\"azimuth\", instance=\"polls\", type=\"job-queue\"} / 1024 / 1024", + "hide": false, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "B" + } + ], + "title": "DB size (MB)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"polls\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num block-processing jobs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"polls\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num event-processing jobs", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 38, + "tags": [ + "ethereum", + "azimuth" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "polls watcher", + "version": 3, + "weekStart": "" +} diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json new file mode 100644 index 00000000..f41b3298 --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json @@ -0,0 +1,1179 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"sushi\", instance=\"merkl_sushiswap\", kind=\"latest_indexed\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Latest indexed block", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head block number of upstream endpoint", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "title": "Upstream chain head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head from a public endpoint (filfox)", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "title": "External chain head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-green", + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-green", + "value": null + }, + { + "color": "red", + "value": 35 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "last_block_process_duration_seconds{job=\"sushi\", instance=\"merkl_sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Last block process duration (s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Watcher sync status block numbers", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 3 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_processed_block_number{job=\"sushi\", instance=\"merkl_sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "latest_processed", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"sushi\", instance=\"merkl_sushiswap\", kind=\"latest_canonical\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"sushi\", instance=\"merkl_sushiswap\", kind=\"intial_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "C" + } + ], + "title": "Sync status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_block_num_events_total{job=\"sushi\", instance=\"merkl_sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number of events in the last block", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of entries in the block_progress and event table", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "dark-yellow", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "block_progress_total{job=\"sushi\", instance=\"merkl_sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "block_progress", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "event_total{job=\"sushi\", instance=\"merkl_sushiswap\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "event", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "DB indexed count ", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 7 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "database_size_bytes{job=\"sushi\", instance=\"merkl_sushiswap\", type=\"watcher\"} / 1024 / 1024", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "database_size_bytes{job=\"sushi\", instance=\"merkl_sushiswap\", type=\"job-queue\"} / 1024 / 1024", + "hide": false, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "B" + } + ], + "title": "DB size (MB)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=\"sushi\", instance=\"merkl_sushiswap\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num block-processing jobs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=\"sushi\", instance=\"merkl_sushiswap\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num event-processing jobs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of subgraph entities loaded in event processing", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "event_processing_load_entity_total{job=\"sushi\", instance=\"merkl_sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num subgraph entities loaded", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of cache hits for subgraph entites in event processing ", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "event_processing_load_entity_cache_hit_total{job=\"sushi\", instance=\"merkl_sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num subgraph entities cache hits", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of subgraph entities currently present in pruned region of cache", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "cached_pruned_entities_total{job=\"sushi\", instance=\"merkl_sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num subgraph entities cached (pruned region)", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 38, + "tags": [ + "filecoin", + "sushi" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "merkl sushiswap watcher", + "version": 3, + "weekStart": "" +} diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json new file mode 100644 index 00000000..e602000f --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json @@ -0,0 +1,1179 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"sushi\", instance=\"sushiswap\", kind=\"latest_indexed\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Latest indexed block", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head block number of upstream endpoint", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "title": "Upstream chain head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head from a public endpoint (filfox)", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "title": "External chain head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-green", + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-green", + "value": null + }, + { + "color": "red", + "value": 35 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "last_block_process_duration_seconds{job=\"sushi\", instance=\"sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Last block process duration (s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Watcher sync status block numbers", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 3 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_processed_block_number{job=\"sushi\", instance=\"sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "latest_processed", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"sushi\", instance=\"sushiswap\", kind=\"latest_canonical\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "sync_status_block_number{job=\"sushi\", instance=\"sushiswap\", kind=\"intial_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "C" + } + ], + "title": "Sync status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_block_num_events_total{job=\"sushi\", instance=\"sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number of events in the last block", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of entries in the block_progress and event table", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "dark-yellow", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "block_progress_total{job=\"sushi\", instance=\"sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "block_progress", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "event_total{job=\"sushi\", instance=\"sushiswap\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "event", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "DB indexed count ", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 7 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "database_size_bytes{job=\"sushi\", instance=\"sushiswap\", type=\"watcher\"} / 1024 / 1024", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "database_size_bytes{job=\"sushi\", instance=\"sushiswap\", type=\"job-queue\"} / 1024 / 1024", + "hide": false, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "B" + } + ], + "title": "DB size (MB)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=\"sushi\", instance=\"sushiswap\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num block-processing jobs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=\"sushi\", instance=\"sushiswap\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num event-processing jobs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of subgraph entities loaded in event processing", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "event_processing_load_entity_total{job=\"sushi\", instance=\"sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num subgraph entities loaded", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of cache hits for subgraph entites in event processing ", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "event_processing_load_entity_cache_hit_total{job=\"sushi\", instance=\"sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num subgraph entities cache hits", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of subgraph entities currently present in pruned region of cache", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "cached_pruned_entities_total{job=\"sushi\", instance=\"sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num subgraph entities cached (pruned region)", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 38, + "tags": [ + "filecoin", + "sushi" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "sushiswap watcher", + "version": 3, + "weekStart": "" +} diff --git a/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml b/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml new file mode 100644 index 00000000..e8e7b8b8 --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml @@ -0,0 +1,12 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +rule_files: + # - "first.rules" + # - "second.rules" + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['localhost:9090'] diff --git a/stack_orchestrator/data/stacks/monitoring/README.md b/stack_orchestrator/data/stacks/monitoring/README.md new file mode 100644 index 00000000..6f1fc227 --- /dev/null +++ b/stack_orchestrator/data/stacks/monitoring/README.md @@ -0,0 +1,94 @@ +# monitoring + +* Instructions to setup and run a Prometheus server and Grafana dashboard +* See [monitoring-watchers.md](./monitoring-watchers.md) for an example usage of the stack with pre-configured dashboards for watchers + +## Create a deployment + +First, create a spec file for the deployment, which will map the stack's ports and volumes to the host: + +```bash +laconic-so --stack monitoring deploy init --output monitoring-spec.yml +``` + +### Ports + +Edit `network` in spec file to map container ports to same ports in host: + +``` +... +network: + ports: + prometheus: + - '9090:9090' + grafana: + - '3000:3000' +... +``` + +### Data volumes + +Container data volumes are bind-mounted to specified paths in the host filesystem. +The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory. The default mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`. + +--- + +Once you've made any needed changes to the spec file, create a deployment from it: + +```bash +laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --deployment-dir monitoring-deployment +``` + +## Configure + +### Prometheus Config + +Add desired scrape configs to prometheus config file (`monitoring-deployment/config/monitoring/prometheus/prometheus.yml`) in the deployment folder; for example: + + ```yml + ... + - job_name: + metrics_path: /metrics/path + scheme: http + static_configs: + - targets: [':'] + ``` + +Note: Use `host.docker.internal` as host to access ports on the host machine + +### Grafana Config + +Place the dashboard json files in grafana dashboards config directory (`monitoring-deployment/config/monitoring/grafana/dashboards`) in the deployment folder + +## Start the stack + +Start the deployment: + +```bash +laconic-so deployment --dir monitoring-deployment start +``` + +* List and check the health status of all the containers using `docker ps` and wait for them to be `healthy` + +* Grafana should now be visible at http://localhost:3000 with configured dashboards + +## Clean up + +To stop monitoring services running in the background, while preserving data: + +```bash +# Only stop the docker containers +laconic-so deployment --dir monitoring-deployment stop + +# Run 'start' to restart the deployment +``` + +To stop monitoring services and also delete data: + +```bash +# Stop the docker containers +laconic-so deployment --dir monitoring-deployment stop --delete-volumes + +# Remove deployment directory (deployment will have to be recreated for a re-run) +rm -rf monitoring-deployment +``` diff --git a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md new file mode 100644 index 00000000..f49bdcda --- /dev/null +++ b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md @@ -0,0 +1,124 @@ +# Monitoring Watchers + +Instructions to setup and run monitoring stack with pre-configured watcher dashboards + +## Create a deployment + +First, create a spec file for the deployment, which will map the stack's ports and volumes to the host: + +```bash +laconic-so --stack monitoring deploy init --output monitoring-watchers-spec.yml +``` + +### Ports + +Edit `network` in spec file to map container ports to same ports in host: + +``` +... +network: + ports: + prometheus: + - '9090:9090' + grafana: + - '3000:3000' +... +``` + +--- + +Once you've made any needed changes to the spec file, create a deployment from it: + +```bash +laconic-so --stack monitoring deploy create --spec-file monitoring-watchers-spec.yml --deployment-dir monitoring-watchers-deployment +``` + +## Configure + +### Prometheus Config + +Add the following scrape configs to prometheus config file (`monitoring-watchers-deployment/config/monitoring/prometheus/prometheus.yml`) in the deployment folder: + + ```yml + ... + - job_name: azimuth + metrics_path: /metrics + scheme: http + static_configs: + - targets: ['AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_PORT'] + labels: + instance: 'azimuth' + - targets: ['CENSURES_WATCHER_HOST:CENSURES_WATCHER_PORT'] + labels: + instance: 'censures' + - targets: ['CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_PORT'] + labels: + instance: 'claims' + - targets: ['CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_PORT'] + labels: + instance: 'conditional_star_release' + - targets: ['DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_PORT'] + labels: + instance: 'delegated_sending_watcher' + - targets: ['ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_PORT'] + labels: + instance: 'ecliptic' + - targets: ['LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_PORT'] + labels: + instance: 'linear_star_release' + - targets: ['POLLS_WATCHER_HOST:POLLS_WATCHER_PORT'] + labels: + instance: 'polls' + + - job_name: sushi + metrics_path: /metrics + scheme: http + static_configs: + - targets: ['SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_PORT'] + labels: + instance: 'sushiswap' + - targets: ['MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_PORT'] + labels: + instance: 'merkl_sushiswap' + ``` + +### Grafana Config + +In the deployment folder, copy over the pre-configured watcher dashboard JSON files to grafana dashboards config directory: + +```bash +cp -r monitoring-watchers-deployment/config/monitoring/grafana/watcher-dashboards/* monitoring-watchers-deployment/config/monitoring/grafana/dashboards/ +``` + +## Start the stack + +Start the deployment: + +```bash +laconic-so deployment --dir monitoring-watchers-deployment start +``` + +* List and check the health status of all the containers using `docker ps` and wait for them to be `healthy` + +* Grafana should now be visible at http://localhost:3000 with configured dashboards + +## Clean up + +To stop monitoring services running in the background, while preserving data: + +```bash +# Only stop the docker containers +laconic-so deployment --dir monitoring-watchers-deployment stop + +# Run 'start' to restart the deployment +``` + +To stop monitoring services and also delete data: + +```bash +# Stop the docker containers +laconic-so deployment --dir monitoring-watchers-deployment stop --delete-volumes + +# Remove deployment directory (deployment will have to be recreated for a re-run) +rm -rf monitoring-watchers-deployment +``` diff --git a/stack_orchestrator/data/stacks/monitoring/stack.yml b/stack_orchestrator/data/stacks/monitoring/stack.yml new file mode 100644 index 00000000..c3532024 --- /dev/null +++ b/stack_orchestrator/data/stacks/monitoring/stack.yml @@ -0,0 +1,7 @@ +version: "0.1" +name: monitoring +repos: +containers: +pods: + - prom-server + - grafana From b9b758bfdda091a4ea367e21fe1e06be9a18a1a0 Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Thu, 14 Dec 2023 17:28:10 +0530 Subject: [PATCH 50/75] Add a stack for running Osmosis frontend app on Urbit (#683) * Deploy osmosis on Urbit fake ship * Remove Urbit configuration from existing osmosis stack * Add a separate stack for Osmosis front end on Urbit * Run script for renaming build files with bash * Add environment variables required in urbit osmosis build * Fix BASEPATH in compose file * Remove ipfs-glob-host from network config in osmosis readme * Use laconic branch for osmosis frontend --------- Co-authored-by: Nabarun --- ...docker-compose-osmosis-front-end-urbit.yml | 20 +++ .../config/osmosis/build-app-for-urbit.sh | 22 +++ .../data/config/osmosis/build-app.sh | 2 +- .../config/osmosis/urbit-files/desk.docket-0 | 10 ++ .../config/osmosis/urbit-files/mar/ico.hoon | 12 ++ .../config/osmosis/urbit-files/mar/jpg.hoon | 12 ++ .../config/osmosis/urbit-files/mar/map.hoon | 18 ++ .../config/osmosis/urbit-files/mar/webp.hoon | 12 ++ .../cerc-osmosis-front-end-urbit/build.sh | 4 + .../cerc-urbit-globs-host/Dockerfile | 7 - .../cerc-urbit-globs-host/build.sh | 9 - .../data/stacks/osmosis-urbit-app/README.md | 158 ++++++++++++++++++ .../data/stacks/osmosis-urbit-app/stack.yml | 13 ++ .../data/stacks/osmosis/README.md | 10 +- .../data/stacks/osmosis/stack.yml | 2 +- .../data/stacks/uniswap-urbit-app/README.md | 11 +- .../data/stacks/uniswap-urbit-app/stack.yml | 1 - 17 files changed, 294 insertions(+), 29 deletions(-) create mode 100644 stack_orchestrator/data/compose/docker-compose-osmosis-front-end-urbit.yml create mode 100755 stack_orchestrator/data/config/osmosis/build-app-for-urbit.sh create mode 100644 stack_orchestrator/data/config/osmosis/urbit-files/desk.docket-0 create mode 100644 stack_orchestrator/data/config/osmosis/urbit-files/mar/ico.hoon create mode 100644 stack_orchestrator/data/config/osmosis/urbit-files/mar/jpg.hoon create mode 100644 stack_orchestrator/data/config/osmosis/urbit-files/mar/map.hoon create mode 100644 stack_orchestrator/data/config/osmosis/urbit-files/mar/webp.hoon create mode 100755 stack_orchestrator/data/container-build/cerc-osmosis-front-end-urbit/build.sh delete mode 100644 stack_orchestrator/data/container-build/cerc-urbit-globs-host/Dockerfile delete mode 100755 stack_orchestrator/data/container-build/cerc-urbit-globs-host/build.sh create mode 100644 stack_orchestrator/data/stacks/osmosis-urbit-app/README.md create mode 100644 stack_orchestrator/data/stacks/osmosis-urbit-app/stack.yml diff --git a/stack_orchestrator/data/compose/docker-compose-osmosis-front-end-urbit.yml b/stack_orchestrator/data/compose/docker-compose-osmosis-front-end-urbit.yml new file mode 100644 index 00000000..238231f4 --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-osmosis-front-end-urbit.yml @@ -0,0 +1,20 @@ +version: "3.2" + +services: + osmosis-front-end: + image: cerc/osmosis-front-end-urbit:local + restart: on-failure + environment: + - NEXT_PUBLIC_WEB_API_BASE_URL=${CERC_WEB_API_BASE_URL} + - NEXT_PUBLIC_BASEPATH=/apps/osmosis + - NEXT_PUBLIC_URBIT_DEPLOYMENT=true + working_dir: /app/packages/web + command: ["./build-app-for-urbit.sh"] + volumes: + - ../config/osmosis/build-app-for-urbit.sh:/app/packages/web/build-app-for-urbit.sh + - urbit_app_builds:/app-builds + - ../config/osmosis/urbit-files/mar:/app/packages/web/mar + - ../config/osmosis/urbit-files/desk.docket-0:/app/packages/web/desk.docket-0 + +volumes: + urbit_app_builds: diff --git a/stack_orchestrator/data/config/osmosis/build-app-for-urbit.sh b/stack_orchestrator/data/config/osmosis/build-app-for-urbit.sh new file mode 100755 index 00000000..1f7dba41 --- /dev/null +++ b/stack_orchestrator/data/config/osmosis/build-app-for-urbit.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +# Check and exit if a deployment already exists (on restarts) +if [ -d /app-builds/osmosis/build ]; then + echo "Build already exists, remove volume to rebuild" + exit 0 +fi + +yarn build:static +./build.sh + +# Move build to app-builds +mkdir -p /app-builds/osmosis +cp -r ./out /app-builds/osmosis/build + +cp -r mar /app-builds/osmosis/ +cp desk.docket-0 /app-builds/osmosis/ diff --git a/stack_orchestrator/data/config/osmosis/build-app.sh b/stack_orchestrator/data/config/osmosis/build-app.sh index 2f3cfd01..9d13bbc3 100755 --- a/stack_orchestrator/data/config/osmosis/build-app.sh +++ b/stack_orchestrator/data/config/osmosis/build-app.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -e if [ -n "$CERC_SCRIPT_DEBUG" ]; then diff --git a/stack_orchestrator/data/config/osmosis/urbit-files/desk.docket-0 b/stack_orchestrator/data/config/osmosis/urbit-files/desk.docket-0 new file mode 100644 index 00000000..969d6349 --- /dev/null +++ b/stack_orchestrator/data/config/osmosis/urbit-files/desk.docket-0 @@ -0,0 +1,10 @@ +:~ title+'Osmosis' + info+'Self-hosted osmosis frontend.' + color+0xcd.75df + image+'https://logowik.com/content/uploads/images/osmosis8548.logowik.com.webp' + base+'osmosis' + glob-http+['REPLACE_WITH_GLOB_URL' REPLACE_WITH_GLOB_HASH] + version+[0 0 1] + website+'https://osmosis.zone/' + license+'MIT' +== diff --git a/stack_orchestrator/data/config/osmosis/urbit-files/mar/ico.hoon b/stack_orchestrator/data/config/osmosis/urbit-files/mar/ico.hoon new file mode 100644 index 00000000..e862b9b9 --- /dev/null +++ b/stack_orchestrator/data/config/osmosis/urbit-files/mar/ico.hoon @@ -0,0 +1,12 @@ +|_ dat=@ +++ grow + |% + ++ mime [/image/x-icon (as-octs:mimes:html dat)] + -- +++ grab + |% + ++ mime |=([p=mite q=octs] q.q) + ++ noun @ + -- +++ grad %mime +-- diff --git a/stack_orchestrator/data/config/osmosis/urbit-files/mar/jpg.hoon b/stack_orchestrator/data/config/osmosis/urbit-files/mar/jpg.hoon new file mode 100644 index 00000000..81a77d46 --- /dev/null +++ b/stack_orchestrator/data/config/osmosis/urbit-files/mar/jpg.hoon @@ -0,0 +1,12 @@ +|_ dat=@ +++ grow + |% + ++ mime [/image/jpeg (as-octs:mimes:html dat)] + -- +++ grab + |% + ++ mime |=([p=mite q=octs] q.q) + ++ noun @ + -- +++ grad %mime +-- diff --git a/stack_orchestrator/data/config/osmosis/urbit-files/mar/map.hoon b/stack_orchestrator/data/config/osmosis/urbit-files/mar/map.hoon new file mode 100644 index 00000000..e5d61eee --- /dev/null +++ b/stack_orchestrator/data/config/osmosis/urbit-files/mar/map.hoon @@ -0,0 +1,18 @@ +:: +:::: /hoon/map/mar + :: Mark for js source maps +/? 310 +:: +=, eyre +|_ mud=@ +++ grow + |% + ++ mime [/application/octet-stream (as-octs:mimes:html (@t mud))] + -- +++ grab + |% :: convert from + ++ mime |=([p=mite q=octs] (@t q.q)) + ++ noun cord :: clam from %noun + -- +++ grad %mime +-- diff --git a/stack_orchestrator/data/config/osmosis/urbit-files/mar/webp.hoon b/stack_orchestrator/data/config/osmosis/urbit-files/mar/webp.hoon new file mode 100644 index 00000000..1909d0dd --- /dev/null +++ b/stack_orchestrator/data/config/osmosis/urbit-files/mar/webp.hoon @@ -0,0 +1,12 @@ +|_ dat=@ +++ grow + |% + ++ mime [/image/webp (as-octs:mimes:html dat)] + -- +++ grab + |% + ++ mime |=([p=mite q=octs] q.q) + ++ noun @ + -- +++ grad %mime +-- diff --git a/stack_orchestrator/data/container-build/cerc-osmosis-front-end-urbit/build.sh b/stack_orchestrator/data/container-build/cerc-osmosis-front-end-urbit/build.sh new file mode 100755 index 00000000..9a47ada9 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-osmosis-front-end-urbit/build.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Build the osmosis front end image +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh +docker build -t cerc/osmosis-front-end-urbit:local -f ${CERC_REPO_BASE_DIR}/osmosis-frontend/docker/Dockerfile.static ${build_command_args} ${CERC_REPO_BASE_DIR}/osmosis-frontend diff --git a/stack_orchestrator/data/container-build/cerc-urbit-globs-host/Dockerfile b/stack_orchestrator/data/container-build/cerc-urbit-globs-host/Dockerfile deleted file mode 100644 index 7a3ca9b7..00000000 --- a/stack_orchestrator/data/container-build/cerc-urbit-globs-host/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM python:3.13.0a2-alpine3.18 - -RUN apk --update --no-cache add alpine-sdk jq bash curl wget - -WORKDIR /app - -ENTRYPOINT [ "bash" ] diff --git a/stack_orchestrator/data/container-build/cerc-urbit-globs-host/build.sh b/stack_orchestrator/data/container-build/cerc-urbit-globs-host/build.sh deleted file mode 100755 index ebd396f1..00000000 --- a/stack_orchestrator/data/container-build/cerc-urbit-globs-host/build.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -# Build the urbit-globs-host image - -source ${CERC_CONTAINER_BASE_DIR}/build-base.sh - -# See: https://stackoverflow.com/a/246128/1701505 -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - -docker build -t cerc/urbit-globs-host:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${SCRIPT_DIR} diff --git a/stack_orchestrator/data/stacks/osmosis-urbit-app/README.md b/stack_orchestrator/data/stacks/osmosis-urbit-app/README.md new file mode 100644 index 00000000..59da7411 --- /dev/null +++ b/stack_orchestrator/data/stacks/osmosis-urbit-app/README.md @@ -0,0 +1,158 @@ +# Self-hosted Osmosis Front End On Urbit + +Instructions to build and deploy: +- Urbit +- An ipfs node +- The osmosis front end + +## Setup + +Clone required repositories: + +```bash +laconic-so --stack osmosis-urbit-app setup-repositories --pull + +# If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories and re-run the command +``` + +Build the container images: + +```bash +laconic-so --stack osmosis-urbit-app build-containers +``` + +## Create a deployment + +First, create a spec file for the deployment, which will map the stack's ports and volumes to the host: + +```bash +laconic-so --stack osmosis-urbit-app deploy init --output osmosis-urbit-app-spec.yml +``` + +### Ports + +Edit `network` in spec file to map container ports to same ports in host: + +```yml +... +network: + ports: + urbit-fake-ship: + - '8080:80' + proxy-server: + - '4000:4000' + ipfs: + - '8081:8080' + - '5001:5001' +``` + +Note: Skip the `ipfs` ports if need to use an externally running IPFS node + +### Data volumes + +Container data volumes are bind-mounted to specified paths in the host filesystem. +The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory. The default mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`. + +--- + +Once you've made any needed changes to the spec file, create a deployment from it: + +```bash +laconic-so --stack osmosis-urbit-app deploy create --spec-file osmosis-urbit-app-spec.yml --deployment-dir osmosis-urbit-app-deployment +``` + +## Set env variables + +Inside the deployment directory, open the file `config.env` and set the following env variables: + +(Note: Use the following config as is if running the stack locally) + + ```bash + # App to be installed (Do not change) + CERC_URBIT_APP=osmosis + + # Osmosis API base URL + # Set this to proxy server endpoint for osmosis app + # (Eg. http://localhost:4000 - in case stack is being run locally with proxy enabled) + # (Eg. https://abc.xyz.com - in case https://abc.xyz.com is pointed to the proxy endpoint) + CERC_WEB_API_BASE_URL=http://localhost:4000 + + # Optional + + # Whether to enable app installation on Urbit + # (just builds and uploads the glob file if disabled) (Default: true) + CERC_ENABLE_APP_INSTALL= + + # Whether to run the proxy server + # (Disable only if proxy not required to be run) (Default: true) + CERC_ENABLE_PROXY= + + # Proxy server configuration + # Used only if proxy is enabled + + # Upstream API URL + CERC_PROXY_UPSTREAM=https://app.osmosis.zone + + # IPFS configuration + + # IFPS endpoint to host the glob file on + # (Default: http://ipfs:5001 pointing to in-stack IPFS node) + CERC_IPFS_GLOB_HOST_ENDPOINT= + + # IFPS endpoint to fetch the glob file from + # (Default: http://ipfs:8080 pointing to in-stack IPFS node) + CERC_IPFS_SERVER_ENDPOINT= + ``` + +## Start the stack + +Start the deployment: + +```bash +laconic-so deployment --dir osmosis-urbit-app-deployment start +``` + +* List and check the health status of all the containers using `docker ps` and wait for them to be `healthy` + +* Run the following to get login password for Urbit web interface: + + ```bash + laconic-so deployment --dir osmosis-urbit-app-deployment exec urbit-fake-ship "curl -s --data '{\"source\":{\"dojo\":\"+code\"},\"sink\":{\"stdout\":null}}' http://localhost:12321" + + # Expected output: "\n"% + ``` + +* Open the Urbit web UI at http://localhost:8080 and use the `PASSWORD` from previous step to login + +* The osmosis app is not available when starting stack for the first time. Check `urbit-fake-ship` logs to see that app has installed: + + ```bash + laconic-so deployment --dir osmosis-urbit-app-deployment logs -f + + # Expected output: + # laconic-3ccf7ee79bdae874-urbit-fake-ship-1 | docket: fetching %http glob for %osmosis desk + # laconic-3ccf7ee79bdae874-urbit-fake-ship-1 | ">="">="osmosis app installed + ``` + +* The osmosis app will be now visible at http://localhost:8080 + +## Clean up + +To stop all osmosis-urbit-app services running in the background, while preserving data: + +```bash +# Only stop the docker containers +laconic-so deployment --dir osmosis-urbit-app-deployment stop + +# Run 'start' to restart the deployment +``` + +To stop all osmosis-urbit-app services and also delete data: + +```bash +# Stop the docker containers +laconic-so deployment --dir osmosis-urbit-app-deployment stop --delete-volumes + +# Remove deployment directory (deployment will have to be recreated for a re-run) +rm -r osmosis-urbit-app-deployment +``` diff --git a/stack_orchestrator/data/stacks/osmosis-urbit-app/stack.yml b/stack_orchestrator/data/stacks/osmosis-urbit-app/stack.yml new file mode 100644 index 00000000..2cf3e9fd --- /dev/null +++ b/stack_orchestrator/data/stacks/osmosis-urbit-app/stack.yml @@ -0,0 +1,13 @@ +version: "0.1" +name: osmosis-urbit-app +repos: + - github.com/cerc-io/osmosis-frontend@laconic + - github.com/cerc-io/watcher-ts@v0.2.78 +containers: + - cerc/osmosis-front-end-urbit + - cerc/watcher-ts +pods: + - osmosis-front-end-urbit + - proxy-server + - fixturenet-urbit + - kubo diff --git a/stack_orchestrator/data/stacks/osmosis/README.md b/stack_orchestrator/data/stacks/osmosis/README.md index ef5504ae..54cda3da 100644 --- a/stack_orchestrator/data/stacks/osmosis/README.md +++ b/stack_orchestrator/data/stacks/osmosis/README.md @@ -1,17 +1,17 @@ # self-hosted osmosis Instructions to build and deploy: -- self-hosted gitea -- an ipfs node -- the osmosis front end -- a laconicd chain +- Self-hosted gitea +- An ipfs node +- The osmosis front end +- A laconicd chain ## Setup Clone required repositories: ```bash -laconic-so --stack osmosis setup-repositories +laconic-so --stack osmosis setup-repositories --pull # If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories and re-run the command ``` diff --git a/stack_orchestrator/data/stacks/osmosis/stack.yml b/stack_orchestrator/data/stacks/osmosis/stack.yml index b0e51a1d..221fbb60 100644 --- a/stack_orchestrator/data/stacks/osmosis/stack.yml +++ b/stack_orchestrator/data/stacks/osmosis/stack.yml @@ -13,7 +13,7 @@ containers: - cerc/watcher-ts - cerc/osmosis-front-end pods: - - kubo + # - kubo - proxy-server - osmosis-front-end # TODO: Fix pod config for deployment command diff --git a/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md b/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md index 7f49c16f..cdbb7f6a 100644 --- a/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md +++ b/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md @@ -14,7 +14,7 @@ Clone required repositories: ```bash laconic-so --stack uniswap-urbit-app setup-repositories --pull -# If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories mentioned below and re-run the command +# If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories and re-run the command ``` Build the container images: @@ -35,7 +35,7 @@ laconic-so --stack uniswap-urbit-app deploy init --output uniswap-urbit-app-spec Edit `network` in spec file to map container ports to same ports in host: -``` +```yml ... network: ports: @@ -134,13 +134,14 @@ laconic-so deployment --dir uniswap-urbit-app-deployment start * Open the Urbit web UI at http://localhost:8080 and use the `PASSWORD` from previous step to login -* The uniswap app is not available when starting stack for the first time. Check `urbit-fake-ship` logs to see that app has installed - ``` +* The uniswap app is not available when starting stack for the first time. Check `urbit-fake-ship` logs to see that app has installed: + + ```bash laconic-so deployment --dir uniswap-urbit-app-deployment logs -f # Expected output: # laconic-3ccf7ee79bdae874-urbit-fake-ship-1 | docket: fetching %http glob for %uniswap desk - # laconic-3ccf7ee79bdae874-urbit-fake-ship-1 | ">="">="Uniswap app installed + # laconic-3ccf7ee79bdae874-urbit-fake-ship-1 | ">="">="uniswap app installed ``` * The uniswap app will be now visible at http://localhost:8080 diff --git a/stack_orchestrator/data/stacks/uniswap-urbit-app/stack.yml b/stack_orchestrator/data/stacks/uniswap-urbit-app/stack.yml index 31499406..1e5ad3f5 100644 --- a/stack_orchestrator/data/stacks/uniswap-urbit-app/stack.yml +++ b/stack_orchestrator/data/stacks/uniswap-urbit-app/stack.yml @@ -6,7 +6,6 @@ repos: containers: - cerc/uniswap-interface - cerc/watcher-ts - - cerc/urbit-globs-host pods: - uniswap-interface - proxy-server From 27a14737f89e999858be19b445f03f8435484646 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Thu, 14 Dec 2023 09:49:21 -0600 Subject: [PATCH 51/75] Make the container tag based on the deployment path. (#688) --- .../deploy/webapp/deploy_webapp_from_registry.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py index ad439812..02731992 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -1,5 +1,4 @@ # Copyright ©2023 Vulcanize - # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -13,6 +12,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import hashlib import json import os import shlex @@ -51,7 +51,6 @@ def process_app_deployment_request( raise Exception("Only unqualified hostnames allowed at this time.") fqdn = f"{requested_name}.{dns_suffix}" - container_tag = "%s:local" % app.attributes.name.replace("@", "") # 3. check ownership of existing dnsrecord vs this request # TODO: Support foreign DNS @@ -94,13 +93,14 @@ def process_app_deployment_request( deployment_record = laconic.get_record(app_deployment_crn) deployment_dir = os.path.join(deployment_parent_dir, fqdn) deployment_config_file = os.path.join(deployment_dir, "config.env") + deployment_container_tag = "laconic-webapp/%s:local" % hashlib.md5(deployment_dir.encode()).hexdigest() # b. check for deployment directory (create if necessary) if not os.path.exists(deployment_dir): if deployment_record: - raise ("Deployment record %s exists, but not deployment dir %s. Please remove name." % - (app_deployment_crn, deployment_dir)) + raise Exception("Deployment record %s exists, but not deployment dir %s. Please remove name." % + (app_deployment_crn, deployment_dir)) print("deploy_webapp", deployment_dir) - deploy_webapp.create_deployment(ctx, deployment_dir, container_tag, + deploy_webapp.create_deployment(ctx, deployment_dir, deployment_container_tag, f"https://{fqdn}", kube_config, image_registry, env_filename) elif env_filename: shutil.copyfile(env_filename, deployment_config_file) @@ -108,7 +108,7 @@ def process_app_deployment_request( needs_k8s_deploy = False # 6. build container (if needed) if not deployment_record or deployment_record.attributes.application != app.id: - build_container_image(app, container_tag) + build_container_image(app, deployment_container_tag) push_container_image(deployment_dir) needs_k8s_deploy = True @@ -262,7 +262,7 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_ record_namespace_deployments, record_namespace_dns, dns_suffix, - deployment_parent_dir, + os.path.abspath(deployment_parent_dir), kube_config, image_registry ) From 61f288450594a833b54bdb88d96fb8230829f70b Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Thu, 14 Dec 2023 17:46:03 -0600 Subject: [PATCH 52/75] Reduce base image size (first round of improvements) (#690) --- .../data/container-build/cerc-nextjs-base/Dockerfile | 4 ++-- .../container-build/cerc-nextjs-base/scripts/build-app.sh | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile index d3ff3f1b..8949c4e9 100644 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile @@ -1,6 +1,6 @@ # Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile # [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster -ARG VARIANT=20-bullseye +ARG VARIANT=20-bullseye-slim FROM node:${VARIANT} ARG USERNAME=node @@ -30,7 +30,7 @@ RUN \ # [Optional] Uncomment this section to install additional OS packages. RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get -y install --no-install-recommends jq gettext-base moreutils + && apt-get -y install --no-install-recommends jq gettext-base # [Optional] Uncomment if you want to install more global node modules # RUN su node -c "npm install -g " diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh index ef6244cf..759ba255 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh @@ -104,7 +104,8 @@ CUR_NEXT_VERSION="`jq -r '.dependencies.next' package.json`" if [ "$CERC_NEXT_VERSION" != "keep" ] && [ "$CUR_NEXT_VERSION" != "$CERC_NEXT_VERSION" ]; then echo "Changing 'next' version specifier from '$CUR_NEXT_VERSION' to '$CERC_NEXT_VERSION' (set with '--extra-build-args \"--build-arg CERC_NEXT_VERSION=$CERC_NEXT_VERSION\"')" - cat package.json | jq ".dependencies.next = \"$CERC_NEXT_VERSION\"" | sponge package.json + cat package.json | jq ".dependencies.next = \"$CERC_NEXT_VERSION\"" > package.json.$$ + mv package.json.$$ package.json fi $CERC_BUILD_TOOL install || exit 1 @@ -128,7 +129,8 @@ to use for the build with: ############################################################################### EOF - cat package.json | jq ".dependencies.next = \"^$CERC_MIN_NEXTVER\"" | sponge package.json + cat package.json | jq ".dependencies.next = \"^$CERC_MIN_NEXTVER\"" > package.json.$$ + mv package.json.$$ package.json $CERC_BUILD_TOOL install || exit 1 fi From 997496b8a52057ecfac1f31b77eb924565c8e2b3 Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Thu, 14 Dec 2023 19:47:30 -0600 Subject: [PATCH 53/75] Update script for new nextjs build output. (#691) --- .../cerc-nextjs-base/scripts/start-serving-app.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh index bf35bcdb..3692f2f6 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh @@ -42,7 +42,7 @@ if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then while [ $count -lt $CERC_MAX_GENERATE_TIME ] && [ "$generate_done" == "false" ]; do sleep 1 count=$((count + 1)) - grep 'rendered as static HTML' gen.out > /dev/null + grep 'rendered as static' gen.out > /dev/null if [ $? -eq 0 ]; then generate_done="true" fi From d8357df3458b5783e70381f0a968819b6a012bbd Mon Sep 17 00:00:00 2001 From: David Boreham Date: Fri, 15 Dec 2023 14:27:45 -0700 Subject: [PATCH 54/75] Add image pull secret to pods (#692) --- stack_orchestrator/deploy/k8s/cluster_info.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 24fe15a0..7718c777 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -200,9 +200,10 @@ class ClusterInfo: ) containers.append(container) volumes = volumes_for_pod_files(self.parsed_pod_yaml_map) + image_pull_secrets = [client.V1LocalObjectReference(name="laconic-registry")] template = client.V1PodTemplateSpec( metadata=client.V1ObjectMeta(labels={"app": self.app_name}), - spec=client.V1PodSpec(containers=containers, volumes=volumes), + spec=client.V1PodSpec(containers=containers, image_pull_secrets=image_pull_secrets, volumes=volumes), ) spec = client.V1DeploymentSpec( replicas=1, template=template, selector={ From def192edab1a6e80365dba9112cee2b92908ef84 Mon Sep 17 00:00:00 2001 From: Nabarun Gogoi Date: Mon, 18 Dec 2023 17:49:45 +0530 Subject: [PATCH 55/75] Update new environment values for Osmosis frontend app (#694) * Update new env values for Osmosis frontend app * Use .env.production instead of local --- .../data/compose/docker-compose-osmosis-front-end-urbit.yml | 1 + .../data/compose/docker-compose-osmosis-front-end.yml | 1 + stack_orchestrator/data/config/osmosis/.env.production | 3 +++ stack_orchestrator/data/config/osmosis/build-app-for-urbit.sh | 2 +- 4 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 stack_orchestrator/data/config/osmosis/.env.production diff --git a/stack_orchestrator/data/compose/docker-compose-osmosis-front-end-urbit.yml b/stack_orchestrator/data/compose/docker-compose-osmosis-front-end-urbit.yml index 238231f4..eebdfb01 100644 --- a/stack_orchestrator/data/compose/docker-compose-osmosis-front-end-urbit.yml +++ b/stack_orchestrator/data/compose/docker-compose-osmosis-front-end-urbit.yml @@ -12,6 +12,7 @@ services: command: ["./build-app-for-urbit.sh"] volumes: - ../config/osmosis/build-app-for-urbit.sh:/app/packages/web/build-app-for-urbit.sh + - ../config/osmosis/.env.production:/app/packages/web/.env.production - urbit_app_builds:/app-builds - ../config/osmosis/urbit-files/mar:/app/packages/web/mar - ../config/osmosis/urbit-files/desk.docket-0:/app/packages/web/desk.docket-0 diff --git a/stack_orchestrator/data/compose/docker-compose-osmosis-front-end.yml b/stack_orchestrator/data/compose/docker-compose-osmosis-front-end.yml index f4a25b54..0ada9367 100644 --- a/stack_orchestrator/data/compose/docker-compose-osmosis-front-end.yml +++ b/stack_orchestrator/data/compose/docker-compose-osmosis-front-end.yml @@ -10,6 +10,7 @@ services: command: ["./build-app.sh"] volumes: - ../config/osmosis/build-app.sh:/app/packages/web/build-app.sh + - ../config/osmosis/.env.production:/app/packages/web/.env.production - app_builds:/app-builds nginx: diff --git a/stack_orchestrator/data/config/osmosis/.env.production b/stack_orchestrator/data/config/osmosis/.env.production new file mode 100644 index 00000000..2fd24c8e --- /dev/null +++ b/stack_orchestrator/data/config/osmosis/.env.production @@ -0,0 +1,3 @@ +NEXT_PUBLIC_COINGECKO_URL=https://api.coingecko.com + +NEXT_PUBLIC_ENABLE_FEATURES=true diff --git a/stack_orchestrator/data/config/osmosis/build-app-for-urbit.sh b/stack_orchestrator/data/config/osmosis/build-app-for-urbit.sh index 1f7dba41..86e52178 100755 --- a/stack_orchestrator/data/config/osmosis/build-app-for-urbit.sh +++ b/stack_orchestrator/data/config/osmosis/build-app-for-urbit.sh @@ -12,7 +12,7 @@ if [ -d /app-builds/osmosis/build ]; then fi yarn build:static -./build.sh +./build-urbit.sh # Move build to app-builds mkdir -p /app-builds/osmosis From 42b92f7e238131683f462298b2128b4ed256fe9d Mon Sep 17 00:00:00 2001 From: Zach Date: Tue, 19 Dec 2023 09:50:14 -0500 Subject: [PATCH 56/75] use square logo for an urbit tile (#689) * use square logo for an urbit tile * bump version, improve info text --- .../data/config/osmosis/urbit-files/desk.docket-0 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stack_orchestrator/data/config/osmosis/urbit-files/desk.docket-0 b/stack_orchestrator/data/config/osmosis/urbit-files/desk.docket-0 index 969d6349..232fe70f 100644 --- a/stack_orchestrator/data/config/osmosis/urbit-files/desk.docket-0 +++ b/stack_orchestrator/data/config/osmosis/urbit-files/desk.docket-0 @@ -1,10 +1,10 @@ :~ title+'Osmosis' - info+'Self-hosted osmosis frontend.' + info+'Osmosis DEX Frontend, built and maintained by Laconic' color+0xcd.75df - image+'https://logowik.com/content/uploads/images/osmosis8548.logowik.com.webp' + image+'https://altcoinsbox.com/wp-content/uploads/2023/03/osmosis-logo.png' base+'osmosis' glob-http+['REPLACE_WITH_GLOB_URL' REPLACE_WITH_GLOB_HASH] - version+[0 0 1] + version+[0 0 2] website+'https://osmosis.zone/' license+'MIT' == From 4a1a46faccc23f0179d1280464ac203d66a312ba Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Thu, 21 Dec 2023 09:26:37 +0530 Subject: [PATCH 57/75] Update monitoring stack with additional dashboards and watcher metrics (#693) * Include retry jobs and update default refresh intervals * Add prometheus blackbox exporter and it's dashboard * Add NodeJS application dashboard * Allow UI updates * Update watcher dashboards for upstream and external chain heads * Update watcher dashboards with watcher config metrics * Upgrade sushiswap and azimuth watchers * Removed fixed title size values * Update instructions * Update instructions for env config * Update instructions with setup --- .../compose/docker-compose-prom-server.yml | 25 + .../data/config/monitoring/blackbox.yml | 7 + .../monitoring/grafana/dashboards/.gitkeep | 0 .../dashboards/nodejs-app-dashboard.json | 943 ++++++++++++++ .../prometheus-blackbox-exporter.json | 1097 +++++++++++++++++ .../provisioning/dashboards/dashboards.yml | 1 + .../grafana/watcher-dashboard-template.json | 923 +++++++++++++- .../azimuth/azimuth-watcher-dashboard.json | 923 +++++++++++++- .../azimuth/censures-watcher-dashboard.json | 923 +++++++++++++- .../azimuth/claims-watcher-dashboard.json | 923 +++++++++++++- ...tional-star-release-watcher-dashboard.json | 923 +++++++++++++- ...ted-sending-watcher-watcher-dashboard.json | 923 +++++++++++++- .../azimuth/ecliptic-watcher-dashboard.json | 923 +++++++++++++- ...linear-star-release-watcher-dashboard.json | 923 +++++++++++++- .../azimuth/polls-watcher-dashboard.json | 923 +++++++++++++- .../merkl-sushiswap-watcher-dashboard.json | 932 +++++++++++++- .../sushi/sushiswap-watcher-dashboard.json | 934 +++++++++++++- .../monitoring/prometheus/prometheus.yml | 32 +- .../watcher-config-template.toml | 2 - .../watcher-config-template.toml | 2 - .../data/stacks/azimuth/README.md | 16 + .../data/stacks/azimuth/stack.yml | 2 +- .../data/stacks/monitoring/README.md | 51 +- .../stacks/monitoring/monitoring-watchers.md | 19 +- .../data/stacks/monitoring/stack.yml | 2 + .../data/stacks/sushiswap-v3/stack.yml | 2 +- 26 files changed, 12143 insertions(+), 231 deletions(-) create mode 100644 stack_orchestrator/data/config/monitoring/blackbox.yml delete mode 100644 stack_orchestrator/data/config/monitoring/grafana/dashboards/.gitkeep create mode 100644 stack_orchestrator/data/config/monitoring/grafana/dashboards/nodejs-app-dashboard.json create mode 100644 stack_orchestrator/data/config/monitoring/grafana/dashboards/prometheus-blackbox-exporter.json diff --git a/stack_orchestrator/data/compose/docker-compose-prom-server.yml b/stack_orchestrator/data/compose/docker-compose-prom-server.yml index 26a61e88..27b8c18c 100644 --- a/stack_orchestrator/data/compose/docker-compose-prom-server.yml +++ b/stack_orchestrator/data/compose/docker-compose-prom-server.yml @@ -18,5 +18,30 @@ services: extra_hosts: - "host.docker.internal:host-gateway" + blackbox: + image: prom/blackbox-exporter:latest + restart: always + volumes: + - ../config/monitoring/blackbox.yml:/etc/blackbox_exporter/config.yml + ports: + - '9115' + extra_hosts: + - "host.docker.internal:host-gateway" + + chain-head-exporter: + image: cerc/watcher-ts:local + restart: always + working_dir: /app/packages/cli + environment: + ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT} + FIL_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT} + ETH_RPC_API_KEY: ${CERC_INFURA_KEY} + PORT: ${CERC_METRICS_PORT} + command: ["sh", "-c", "yarn export-metrics:chain-heads"] + ports: + - '5000' + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: prometheus_data: diff --git a/stack_orchestrator/data/config/monitoring/blackbox.yml b/stack_orchestrator/data/config/monitoring/blackbox.yml new file mode 100644 index 00000000..4a9d72eb --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/blackbox.yml @@ -0,0 +1,7 @@ +modules: + http_2xx: + prober: http + timeout: 5s + http: + valid_status_codes: [] #default to 2xx + method: GET diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/.gitkeep b/stack_orchestrator/data/config/monitoring/grafana/dashboards/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/nodejs-app-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/nodejs-app-dashboard.json new file mode 100644 index 00000000..5a95578a --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/nodejs-app-dashboard.json @@ -0,0 +1,943 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "node.js prometheus client basic metrics", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 11159, + "graphTooltip": 0, + "id": 15, + "links": [], + "liveNow": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 10, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(process_cpu_user_seconds_total{instance=~\"$instance\"}[2m]) * 100", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "User CPU - {{instance}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(process_cpu_system_seconds_total{instance=~\"$instance\"}[2m]) * 100", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Sys CPU - {{instance}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Process CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 9, + "x": 10, + "y": 0 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_eventloop_lag_seconds{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Event Loop Lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 19, + "y": 0 + }, + "id": 2, + "interval": "", + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum(nodejs_version_info{instance=~\"$instance\"}) by (version)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{version}}", + "refId": "A" + } + ], + "title": "Node.js Version", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#F2495C", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 19, + "y": 3 + }, + "id": 4, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum(changes(process_start_time_seconds{instance=~\"$instance\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Process Restart Times", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 16, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "process_resident_memory_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Process Memory - {{instance}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_heap_size_total_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Heap Total - {{instance}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_heap_size_used_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Heap Used - {{instance}}", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_external_memory_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "External Memory - {{instance}}", + "refId": "D" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Process Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_active_handles_total{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Active Handler - {{instance}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_active_requests_total{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Active Request - {{instance}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Active Handlers/Requests Total", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_heap_space_size_total_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Heap Total - {{instance}} - {{space}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Heap Total Detail", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 14 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_heap_space_size_used_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Heap Used - {{instance}} - {{space}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Heap Used Detail", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 14 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_heap_space_size_available_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Heap Used - {{instance}} - {{space}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Heap Available Detail", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": "10s", + "schemaVersion": 38, + "tags": [ + "nodejs" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(nodejs_version_info, instance)", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [], + "query": "label_values(nodejs_version_info, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "NodeJS Application Dashboard", + "uid": "PTSqcpJWk", + "version": 3, + "weekStart": "" +} diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/prometheus-blackbox-exporter.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/prometheus-blackbox-exporter.json new file mode 100644 index 00000000..638c4d5b --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/prometheus-blackbox-exporter.json @@ -0,0 +1,1097 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Prometheus Blackbox Exporter Overview", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 7587, + "graphTooltip": 0, + "id": 29, + "links": [], + "liveNow": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 138, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "probe_duration_seconds{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Global Probe Duration", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 15, + "panels": [], + "repeat": "target", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "$target status", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "text": "DOWN" + }, + "1": { + "text": "UP" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "#299c46", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 9 + }, + "id": 2, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "probe_success{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Status", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 10, + "x": 4, + "y": 9 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "probe_http_duration_seconds{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ phase }}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "HTTP Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 10, + "x": 14, + "y": 9 + }, + "hiddenSeries": false, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "probe_duration_seconds{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "seconds", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Probe Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "0": { + "text": "N/A" + }, + "1": { + "text": "YES" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 201 + }, + { + "color": "#d44a3a", + "value": 399 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 11 + }, + "id": 20, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "probe_http_status_code{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "HTTP Status Code", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 13 + }, + "id": 27, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "probe_http_version{instance=~\"$target\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "HTTP Version", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "text": "NO" + }, + "1": { + "text": "YES" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 + }, + { + "color": "#299c46", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 15 + }, + "id": 18, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "probe_http_ssl{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "SSL", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "0": { + "text": "NO" + }, + "1": { + "text": "YES" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 + }, + { + "color": "#299c46", + "value": 1209600 + } + ] + }, + "unit": "dtdurations" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 10, + "x": 4, + "y": 15 + }, + "id": 19, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "probe_ssl_earliest_cert_expiry{instance=~\"$target\"} - time()", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "SSL Expiry", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 5, + "x": 14, + "y": 15 + }, + "id": 23, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "avg(probe_duration_seconds{instance=~\"$target\"})", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Average Probe Duration", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 5, + "x": 19, + "y": 15 + }, + "id": 24, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "avg(probe_dns_lookup_time_seconds{instance=~\"$target\"})", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Average DNS Lookup", + "type": "stat" + } + ], + "refresh": "10s", + "schemaVersion": 38, + "tags": [ + "blackbox", + "prometheus" + ], + "templating": { + "list": [ + { + "auto": true, + "auto_count": 10, + "auto_min": "10s", + "current": { + "selected": false, + "text": "10s", + "value": "10s" + }, + "hide": 0, + "label": "Interval", + "name": "interval", + "options": [ + { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "5s", + "value": "5s" + }, + { + "selected": true, + "text": "10s", + "value": "10s" + }, + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "5s,10s,30s,1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "target", + "options": [], + "query": "label_values(probe_success, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Prometheus Blackbox Exporter", + "uid": "xtkCtBkiz", + "version": 2, + "weekStart": "" +} diff --git a/stack_orchestrator/data/config/monitoring/grafana/provisioning/dashboards/dashboards.yml b/stack_orchestrator/data/config/monitoring/grafana/provisioning/dashboards/dashboards.yml index 4705688e..9166cbff 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/provisioning/dashboards/dashboards.yml +++ b/stack_orchestrator/data/config/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -4,6 +4,7 @@ providers: - name: dashboards type: file updateIntervalSeconds: 10 + allowUiUpdates: true options: path: /etc/grafana/dashboards foldersFromFilesStructure: true diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json index 03144776..1e0f4e17 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json @@ -29,7 +29,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -53,7 +53,7 @@ }, "id": 1, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -95,11 +95,11 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head block number of upstream endpoint", + "description": "Chain head block number of the upstream endpoint", "fieldConfig": { "defaults": { "color": { - "fixedColor": "orange", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -123,7 +123,7 @@ }, "id": 11, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -138,6 +138,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_upstream_block_number{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "Upstream chain head", "type": "stat" }, @@ -146,11 +164,11 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head from a public endpoint (etherscan)", + "description": "Chain head from an external public endpoint", "fieldConfig": { "defaults": { "color": { - "fixedColor": "red", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -174,7 +192,7 @@ }, "id": 12, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -189,6 +207,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"REPLACE_WITH_WATCHER_CHAIN\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "External chain head", "type": "stat" }, @@ -294,6 +330,322 @@ "title": "Last block process duration (s)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "orange", + "index": 2, + "text": "LAZY" + }, + "1": { + "color": "semi-dark-green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"server\", field=\"is_active\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher kind", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 3 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "latest_upstream_block_number{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_upstream", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_upstream_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"REPLACE_WITH_WATCHER_CHAIN\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_external", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -320,10 +672,10 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 3 + "h": 3, + "w": 8, + "x": 4, + "y": 5 }, "id": 13, "options": { @@ -338,6 +690,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -393,6 +746,100 @@ "title": "Sync status", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-green", + "index": 1, + "text": "REAL TIME" + }, + "1": { + "color": "orange", + "index": 2, + "text": "HISTORICAL" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 5.5 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "is_syncing_historical_blocks{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher sync mode", + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -520,10 +967,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 0, - "y": 7 + "y": 8 }, "id": 7, "options": { @@ -538,6 +985,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -605,10 +1053,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 6, - "y": 7 + "y": 8 }, "id": 4, "options": { @@ -623,6 +1071,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -749,7 +1198,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "expr": "pgboss_jobs_total{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -851,7 +1300,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "expr": "pgboss_jobs_total{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -864,9 +1313,445 @@ ], "title": "Num event-processing jobs", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"upstream\", field=\"is_using_rpc_client\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"upstream\", field=\"is_fevm\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"server\", field=\"rpc_supports_block_hash\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"upstream\", field=\"filter_logs_by_topics\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + } + ], + "title": "Watcher config (upstream eth server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "watcher_config_info" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"server\", field=\"is_subgraph_watcher\"}", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"server\", field=\"max_events_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"server\", field=\"clear_entities_cache_interval\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"server\", field=\"max_simultaneous_requests\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"server\", field=\"max_request_queue_limit\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "F" + } + ], + "title": "Watcher config (server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "is_subgraph_watcher" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"jobqueue\", field=\"num_events_in_batch\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"jobqueue\", field=\"block_delay_seconds\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"jobqueue\", field=\"use_block_ranges\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + } + ], + "title": "Watcher config (job queue)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "use_block_ranges" + } + ], + "fields": {} + } + } + ], + "type": "stat" } ], - "refresh": "5s", + "refresh": "10s", "schemaVersion": 38, "tags": [ "REPLACE_WITH_WATCHER_CHAIN", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json index 6f924b12..3a67d7f4 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json @@ -29,7 +29,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -53,7 +53,7 @@ }, "id": 1, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -95,11 +95,11 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head block number of upstream endpoint", + "description": "Chain head block number of the upstream endpoint", "fieldConfig": { "defaults": { "color": { - "fixedColor": "orange", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -123,7 +123,7 @@ }, "id": 11, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -138,6 +138,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"azimuth\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "Upstream chain head", "type": "stat" }, @@ -146,11 +164,11 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head from a public endpoint (etherscan)", + "description": "Chain head from an external public endpoint", "fieldConfig": { "defaults": { "color": { - "fixedColor": "red", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -174,7 +192,7 @@ }, "id": 12, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -189,6 +207,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "External chain head", "type": "stat" }, @@ -294,6 +330,322 @@ "title": "Last block process duration (s)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "orange", + "index": 2, + "text": "LAZY" + }, + "1": { + "color": "semi-dark-green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"server\", field=\"is_active\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher kind", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 3 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"azimuth\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"azimuth\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_upstream", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_upstream_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"azimuth\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_external", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -320,10 +672,10 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 3 + "h": 3, + "w": 8, + "x": 4, + "y": 5 }, "id": 13, "options": { @@ -338,6 +690,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -393,6 +746,100 @@ "title": "Sync status", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-green", + "index": 1, + "text": "REAL TIME" + }, + "1": { + "color": "orange", + "index": 2, + "text": "HISTORICAL" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 5.5 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "is_syncing_historical_blocks{job=\"azimuth\", instance=\"azimuth\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher sync mode", + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -520,10 +967,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 0, - "y": 7 + "y": 8 }, "id": 7, "options": { @@ -538,6 +985,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -605,10 +1053,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 6, - "y": 7 + "y": 8 }, "id": 4, "options": { @@ -623,6 +1071,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -749,7 +1198,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"azimuth\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"azimuth\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -851,7 +1300,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"azimuth\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"azimuth\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -864,9 +1313,445 @@ ], "title": "Num event-processing jobs", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"upstream\", field=\"is_using_rpc_client\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"upstream\", field=\"is_fevm\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"server\", field=\"rpc_supports_block_hash\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"upstream\", field=\"filter_logs_by_topics\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + } + ], + "title": "Watcher config (upstream eth server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "watcher_config_info" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"server\", field=\"is_subgraph_watcher\"}", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"server\", field=\"max_events_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"server\", field=\"clear_entities_cache_interval\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"server\", field=\"max_simultaneous_requests\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"server\", field=\"max_request_queue_limit\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "F" + } + ], + "title": "Watcher config (server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "is_subgraph_watcher" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"jobqueue\", field=\"num_events_in_batch\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"jobqueue\", field=\"block_delay_seconds\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"jobqueue\", field=\"use_block_ranges\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + } + ], + "title": "Watcher config (job queue)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "use_block_ranges" + } + ], + "fields": {} + } + } + ], + "type": "stat" } ], - "refresh": "5s", + "refresh": "10s", "schemaVersion": 38, "tags": [ "ethereum", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json index cbeb07ae..04c93969 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json @@ -29,7 +29,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -53,7 +53,7 @@ }, "id": 1, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -95,11 +95,11 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head block number of upstream endpoint", + "description": "Chain head block number of the upstream endpoint", "fieldConfig": { "defaults": { "color": { - "fixedColor": "orange", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -123,7 +123,7 @@ }, "id": 11, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -138,6 +138,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"censures\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "Upstream chain head", "type": "stat" }, @@ -146,11 +164,11 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head from a public endpoint (etherscan)", + "description": "Chain head from an external public endpoint", "fieldConfig": { "defaults": { "color": { - "fixedColor": "red", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -174,7 +192,7 @@ }, "id": 12, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -189,6 +207,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "External chain head", "type": "stat" }, @@ -294,6 +330,322 @@ "title": "Last block process duration (s)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "orange", + "index": 2, + "text": "LAZY" + }, + "1": { + "color": "semi-dark-green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"server\", field=\"is_active\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher kind", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 3 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"censures\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"censures\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_upstream", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_upstream_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"censures\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_external", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -320,10 +672,10 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 3 + "h": 3, + "w": 8, + "x": 4, + "y": 5 }, "id": 13, "options": { @@ -338,6 +690,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -393,6 +746,100 @@ "title": "Sync status", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-green", + "index": 1, + "text": "REAL TIME" + }, + "1": { + "color": "orange", + "index": 2, + "text": "HISTORICAL" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 5.5 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "is_syncing_historical_blocks{job=\"azimuth\", instance=\"censures\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher sync mode", + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -520,10 +967,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 0, - "y": 7 + "y": 8 }, "id": 7, "options": { @@ -538,6 +985,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -605,10 +1053,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 6, - "y": 7 + "y": 8 }, "id": 4, "options": { @@ -623,6 +1071,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -749,7 +1198,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"censures\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"censures\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -851,7 +1300,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"censures\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"censures\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -864,9 +1313,445 @@ ], "title": "Num event-processing jobs", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"upstream\", field=\"is_using_rpc_client\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"upstream\", field=\"is_fevm\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"server\", field=\"rpc_supports_block_hash\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"upstream\", field=\"filter_logs_by_topics\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + } + ], + "title": "Watcher config (upstream eth server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "watcher_config_info" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"server\", field=\"is_subgraph_watcher\"}", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"server\", field=\"max_events_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"server\", field=\"clear_entities_cache_interval\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"server\", field=\"max_simultaneous_requests\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"server\", field=\"max_request_queue_limit\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "F" + } + ], + "title": "Watcher config (server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "is_subgraph_watcher" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"jobqueue\", field=\"num_events_in_batch\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"jobqueue\", field=\"block_delay_seconds\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"jobqueue\", field=\"use_block_ranges\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + } + ], + "title": "Watcher config (job queue)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "use_block_ranges" + } + ], + "fields": {} + } + } + ], + "type": "stat" } ], - "refresh": "5s", + "refresh": "10s", "schemaVersion": 38, "tags": [ "ethereum", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json index 29f18fe1..9c5c5510 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json @@ -29,7 +29,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -53,7 +53,7 @@ }, "id": 1, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -95,11 +95,11 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head block number of upstream endpoint", + "description": "Chain head block number of the upstream endpoint", "fieldConfig": { "defaults": { "color": { - "fixedColor": "orange", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -123,7 +123,7 @@ }, "id": 11, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -138,6 +138,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"claims\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "Upstream chain head", "type": "stat" }, @@ -146,11 +164,11 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head from a public endpoint (etherscan)", + "description": "Chain head from an external public endpoint", "fieldConfig": { "defaults": { "color": { - "fixedColor": "red", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -174,7 +192,7 @@ }, "id": 12, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -189,6 +207,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "External chain head", "type": "stat" }, @@ -294,6 +330,322 @@ "title": "Last block process duration (s)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "orange", + "index": 2, + "text": "LAZY" + }, + "1": { + "color": "semi-dark-green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"server\", field=\"is_active\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher kind", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 3 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"claims\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"claims\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_upstream", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_upstream_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"claims\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_external", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -320,10 +672,10 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 3 + "h": 3, + "w": 8, + "x": 4, + "y": 5 }, "id": 13, "options": { @@ -338,6 +690,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -393,6 +746,100 @@ "title": "Sync status", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-green", + "index": 1, + "text": "REAL TIME" + }, + "1": { + "color": "orange", + "index": 2, + "text": "HISTORICAL" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 5.5 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "is_syncing_historical_blocks{job=\"azimuth\", instance=\"claims\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher sync mode", + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -520,10 +967,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 0, - "y": 7 + "y": 8 }, "id": 7, "options": { @@ -538,6 +985,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -605,10 +1053,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 6, - "y": 7 + "y": 8 }, "id": 4, "options": { @@ -623,6 +1071,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -749,7 +1198,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"claims\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"claims\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -851,7 +1300,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"claims\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"claims\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -864,9 +1313,445 @@ ], "title": "Num event-processing jobs", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"upstream\", field=\"is_using_rpc_client\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"upstream\", field=\"is_fevm\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"server\", field=\"rpc_supports_block_hash\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"upstream\", field=\"filter_logs_by_topics\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + } + ], + "title": "Watcher config (upstream eth server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "watcher_config_info" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"server\", field=\"is_subgraph_watcher\"}", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"server\", field=\"max_events_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"server\", field=\"clear_entities_cache_interval\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"server\", field=\"max_simultaneous_requests\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"server\", field=\"max_request_queue_limit\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "F" + } + ], + "title": "Watcher config (server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "is_subgraph_watcher" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"jobqueue\", field=\"num_events_in_batch\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"jobqueue\", field=\"block_delay_seconds\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"jobqueue\", field=\"use_block_ranges\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + } + ], + "title": "Watcher config (job queue)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "use_block_ranges" + } + ], + "fields": {} + } + } + ], + "type": "stat" } ], - "refresh": "5s", + "refresh": "10s", "schemaVersion": 38, "tags": [ "ethereum", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json index 28ec488d..6f739555 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json @@ -29,7 +29,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -53,7 +53,7 @@ }, "id": 1, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -95,11 +95,11 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head block number of upstream endpoint", + "description": "Chain head block number of the upstream endpoint", "fieldConfig": { "defaults": { "color": { - "fixedColor": "orange", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -123,7 +123,7 @@ }, "id": 11, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -138,6 +138,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"conditional_star_release\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "Upstream chain head", "type": "stat" }, @@ -146,11 +164,11 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head from a public endpoint (etherscan)", + "description": "Chain head from an external public endpoint", "fieldConfig": { "defaults": { "color": { - "fixedColor": "red", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -174,7 +192,7 @@ }, "id": 12, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -189,6 +207,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "External chain head", "type": "stat" }, @@ -294,6 +330,322 @@ "title": "Last block process duration (s)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "orange", + "index": 2, + "text": "LAZY" + }, + "1": { + "color": "semi-dark-green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"server\", field=\"is_active\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher kind", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 3 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"conditional_star_release\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"conditional_star_release\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_upstream", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_upstream_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"conditional_star_release\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_external", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -320,10 +672,10 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 3 + "h": 3, + "w": 8, + "x": 4, + "y": 5 }, "id": 13, "options": { @@ -338,6 +690,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -393,6 +746,100 @@ "title": "Sync status", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-green", + "index": 1, + "text": "REAL TIME" + }, + "1": { + "color": "orange", + "index": 2, + "text": "HISTORICAL" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 5.5 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "is_syncing_historical_blocks{job=\"azimuth\", instance=\"conditional_star_release\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher sync mode", + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -520,10 +967,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 0, - "y": 7 + "y": 8 }, "id": 7, "options": { @@ -538,6 +985,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -605,10 +1053,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 6, - "y": 7 + "y": 8 }, "id": 4, "options": { @@ -623,6 +1071,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -749,7 +1198,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"conditional_star_release\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"conditional_star_release\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -851,7 +1300,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"conditional_star_release\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"conditional_star_release\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -864,9 +1313,445 @@ ], "title": "Num event-processing jobs", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"upstream\", field=\"is_using_rpc_client\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"upstream\", field=\"is_fevm\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"server\", field=\"rpc_supports_block_hash\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"upstream\", field=\"filter_logs_by_topics\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + } + ], + "title": "Watcher config (upstream eth server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "watcher_config_info" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"server\", field=\"is_subgraph_watcher\"}", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"server\", field=\"max_events_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"server\", field=\"clear_entities_cache_interval\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"server\", field=\"max_simultaneous_requests\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"server\", field=\"max_request_queue_limit\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "F" + } + ], + "title": "Watcher config (server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "is_subgraph_watcher" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"jobqueue\", field=\"num_events_in_batch\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"jobqueue\", field=\"block_delay_seconds\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"jobqueue\", field=\"use_block_ranges\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + } + ], + "title": "Watcher config (job queue)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "use_block_ranges" + } + ], + "fields": {} + } + } + ], + "type": "stat" } ], - "refresh": "5s", + "refresh": "10s", "schemaVersion": 38, "tags": [ "ethereum", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json index b7411c45..9c169174 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json @@ -29,7 +29,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -53,7 +53,7 @@ }, "id": 1, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -95,11 +95,11 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head block number of upstream endpoint", + "description": "Chain head block number of the upstream endpoint", "fieldConfig": { "defaults": { "color": { - "fixedColor": "orange", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -123,7 +123,7 @@ }, "id": 11, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -138,6 +138,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"delegated_sending_watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "Upstream chain head", "type": "stat" }, @@ -146,11 +164,11 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head from a public endpoint (etherscan)", + "description": "Chain head from an external public endpoint", "fieldConfig": { "defaults": { "color": { - "fixedColor": "red", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -174,7 +192,7 @@ }, "id": 12, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -189,6 +207,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "External chain head", "type": "stat" }, @@ -294,6 +330,322 @@ "title": "Last block process duration (s)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "orange", + "index": 2, + "text": "LAZY" + }, + "1": { + "color": "semi-dark-green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"server\", field=\"is_active\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher kind", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 3 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"delegated_sending_watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"delegated_sending_watcher\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_upstream", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_upstream_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"delegated_sending_watcher\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_external", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -320,10 +672,10 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 3 + "h": 3, + "w": 8, + "x": 4, + "y": 5 }, "id": 13, "options": { @@ -338,6 +690,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -393,6 +746,100 @@ "title": "Sync status", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-green", + "index": 1, + "text": "REAL TIME" + }, + "1": { + "color": "orange", + "index": 2, + "text": "HISTORICAL" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 5.5 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "is_syncing_historical_blocks{job=\"azimuth\", instance=\"delegated_sending_watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher sync mode", + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -520,10 +967,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 0, - "y": 7 + "y": 8 }, "id": 7, "options": { @@ -538,6 +985,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -605,10 +1053,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 6, - "y": 7 + "y": 8 }, "id": 4, "options": { @@ -623,6 +1071,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -749,7 +1198,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"delegated_sending_watcher\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"delegated_sending_watcher\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -851,7 +1300,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"delegated_sending_watcher\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"delegated_sending_watcher\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -864,9 +1313,445 @@ ], "title": "Num event-processing jobs", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"upstream\", field=\"is_using_rpc_client\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"upstream\", field=\"is_fevm\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"server\", field=\"rpc_supports_block_hash\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"upstream\", field=\"filter_logs_by_topics\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + } + ], + "title": "Watcher config (upstream eth server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "watcher_config_info" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"server\", field=\"is_subgraph_watcher\"}", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"server\", field=\"max_events_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"server\", field=\"clear_entities_cache_interval\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"server\", field=\"max_simultaneous_requests\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"server\", field=\"max_request_queue_limit\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "F" + } + ], + "title": "Watcher config (server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "is_subgraph_watcher" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"jobqueue\", field=\"num_events_in_batch\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"jobqueue\", field=\"block_delay_seconds\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"jobqueue\", field=\"use_block_ranges\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + } + ], + "title": "Watcher config (job queue)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "use_block_ranges" + } + ], + "fields": {} + } + } + ], + "type": "stat" } ], - "refresh": "5s", + "refresh": "10s", "schemaVersion": 38, "tags": [ "ethereum", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json index 3ee7d151..f46408f0 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json @@ -29,7 +29,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -53,7 +53,7 @@ }, "id": 1, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -95,11 +95,11 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head block number of upstream endpoint", + "description": "Chain head block number of the upstream endpoint", "fieldConfig": { "defaults": { "color": { - "fixedColor": "orange", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -123,7 +123,7 @@ }, "id": 11, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -138,6 +138,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"ecliptic\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "Upstream chain head", "type": "stat" }, @@ -146,11 +164,11 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head from a public endpoint (etherscan)", + "description": "Chain head from an external public endpoint", "fieldConfig": { "defaults": { "color": { - "fixedColor": "red", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -174,7 +192,7 @@ }, "id": 12, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -189,6 +207,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "External chain head", "type": "stat" }, @@ -294,6 +330,322 @@ "title": "Last block process duration (s)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "orange", + "index": 2, + "text": "LAZY" + }, + "1": { + "color": "semi-dark-green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"server\", field=\"is_active\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher kind", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 3 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"ecliptic\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"ecliptic\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_upstream", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_upstream_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"ecliptic\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_external", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -320,10 +672,10 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 3 + "h": 3, + "w": 8, + "x": 4, + "y": 5 }, "id": 13, "options": { @@ -338,6 +690,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -393,6 +746,100 @@ "title": "Sync status", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-green", + "index": 1, + "text": "REAL TIME" + }, + "1": { + "color": "orange", + "index": 2, + "text": "HISTORICAL" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 5.5 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "is_syncing_historical_blocks{job=\"azimuth\", instance=\"ecliptic\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher sync mode", + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -520,10 +967,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 0, - "y": 7 + "y": 8 }, "id": 7, "options": { @@ -538,6 +985,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -605,10 +1053,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 6, - "y": 7 + "y": 8 }, "id": 4, "options": { @@ -623,6 +1071,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -749,7 +1198,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"ecliptic\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"ecliptic\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -851,7 +1300,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"ecliptic\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"ecliptic\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -864,9 +1313,445 @@ ], "title": "Num event-processing jobs", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"upstream\", field=\"is_using_rpc_client\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"upstream\", field=\"is_fevm\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"server\", field=\"rpc_supports_block_hash\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"upstream\", field=\"filter_logs_by_topics\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + } + ], + "title": "Watcher config (upstream eth server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "watcher_config_info" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"server\", field=\"is_subgraph_watcher\"}", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"server\", field=\"max_events_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"server\", field=\"clear_entities_cache_interval\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"server\", field=\"max_simultaneous_requests\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"server\", field=\"max_request_queue_limit\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "F" + } + ], + "title": "Watcher config (server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "is_subgraph_watcher" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"jobqueue\", field=\"num_events_in_batch\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"jobqueue\", field=\"block_delay_seconds\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"jobqueue\", field=\"use_block_ranges\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + } + ], + "title": "Watcher config (job queue)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "use_block_ranges" + } + ], + "fields": {} + } + } + ], + "type": "stat" } ], - "refresh": "5s", + "refresh": "10s", "schemaVersion": 38, "tags": [ "ethereum", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json index be6ef9ec..e2187a43 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json @@ -29,7 +29,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -53,7 +53,7 @@ }, "id": 1, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -95,11 +95,11 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head block number of upstream endpoint", + "description": "Chain head block number of the upstream endpoint", "fieldConfig": { "defaults": { "color": { - "fixedColor": "orange", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -123,7 +123,7 @@ }, "id": 11, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -138,6 +138,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"linear_star_release\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "Upstream chain head", "type": "stat" }, @@ -146,11 +164,11 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head from a public endpoint (etherscan)", + "description": "Chain head from an external public endpoint", "fieldConfig": { "defaults": { "color": { - "fixedColor": "red", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -174,7 +192,7 @@ }, "id": 12, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -189,6 +207,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "External chain head", "type": "stat" }, @@ -294,6 +330,322 @@ "title": "Last block process duration (s)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "orange", + "index": 2, + "text": "LAZY" + }, + "1": { + "color": "semi-dark-green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"server\", field=\"is_active\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher kind", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 3 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"linear_star_release\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"linear_star_release\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_upstream", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_upstream_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"linear_star_release\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_external", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -320,10 +672,10 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 3 + "h": 3, + "w": 8, + "x": 4, + "y": 5 }, "id": 13, "options": { @@ -338,6 +690,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -393,6 +746,100 @@ "title": "Sync status", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-green", + "index": 1, + "text": "REAL TIME" + }, + "1": { + "color": "orange", + "index": 2, + "text": "HISTORICAL" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 5.5 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "is_syncing_historical_blocks{job=\"azimuth\", instance=\"linear_star_release\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher sync mode", + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -520,10 +967,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 0, - "y": 7 + "y": 8 }, "id": 7, "options": { @@ -538,6 +985,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -605,10 +1053,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 6, - "y": 7 + "y": 8 }, "id": 4, "options": { @@ -623,6 +1071,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -749,7 +1198,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"linear_star_release\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"linear_star_release\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -851,7 +1300,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"linear_star_release\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"linear_star_release\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -864,9 +1313,445 @@ ], "title": "Num event-processing jobs", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"upstream\", field=\"is_using_rpc_client\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"upstream\", field=\"is_fevm\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"server\", field=\"rpc_supports_block_hash\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"upstream\", field=\"filter_logs_by_topics\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + } + ], + "title": "Watcher config (upstream eth server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "watcher_config_info" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"server\", field=\"is_subgraph_watcher\"}", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"server\", field=\"max_events_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"server\", field=\"clear_entities_cache_interval\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"server\", field=\"max_simultaneous_requests\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"server\", field=\"max_request_queue_limit\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "F" + } + ], + "title": "Watcher config (server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "is_subgraph_watcher" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"jobqueue\", field=\"num_events_in_batch\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"jobqueue\", field=\"block_delay_seconds\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"jobqueue\", field=\"use_block_ranges\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + } + ], + "title": "Watcher config (job queue)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "use_block_ranges" + } + ], + "fields": {} + } + } + ], + "type": "stat" } ], - "refresh": "5s", + "refresh": "10s", "schemaVersion": 38, "tags": [ "ethereum", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json index 01910d9a..5e653ee3 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json @@ -29,7 +29,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -53,7 +53,7 @@ }, "id": 1, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -95,11 +95,11 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head block number of upstream endpoint", + "description": "Chain head block number of the upstream endpoint", "fieldConfig": { "defaults": { "color": { - "fixedColor": "orange", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -123,7 +123,7 @@ }, "id": 11, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -138,6 +138,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"polls\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "Upstream chain head", "type": "stat" }, @@ -146,11 +164,11 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head from a public endpoint (etherscan)", + "description": "Chain head from an external public endpoint", "fieldConfig": { "defaults": { "color": { - "fixedColor": "red", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -174,7 +192,7 @@ }, "id": 12, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -189,6 +207,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "External chain head", "type": "stat" }, @@ -294,6 +330,322 @@ "title": "Last block process duration (s)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "orange", + "index": 2, + "text": "LAZY" + }, + "1": { + "color": "semi-dark-green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"server\", field=\"is_active\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher kind", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 3 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"polls\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"polls\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_upstream", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_upstream_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"azimuth\", instance=\"polls\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_external", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -320,10 +672,10 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 3 + "h": 3, + "w": 8, + "x": 4, + "y": 5 }, "id": 13, "options": { @@ -338,6 +690,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -393,6 +746,100 @@ "title": "Sync status", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-green", + "index": 1, + "text": "REAL TIME" + }, + "1": { + "color": "orange", + "index": 2, + "text": "HISTORICAL" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 5.5 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "is_syncing_historical_blocks{job=\"azimuth\", instance=\"polls\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher sync mode", + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -520,10 +967,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 0, - "y": 7 + "y": 8 }, "id": 7, "options": { @@ -538,6 +985,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -605,10 +1053,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 6, - "y": 7 + "y": 8 }, "id": 4, "options": { @@ -623,6 +1071,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -749,7 +1198,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"polls\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"polls\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -851,7 +1300,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"polls\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"polls\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -864,9 +1313,445 @@ ], "title": "Num event-processing jobs", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"upstream\", field=\"is_using_rpc_client\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"upstream\", field=\"is_fevm\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"server\", field=\"rpc_supports_block_hash\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"upstream\", field=\"filter_logs_by_topics\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + } + ], + "title": "Watcher config (upstream eth server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "watcher_config_info" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"server\", field=\"is_subgraph_watcher\"}", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"server\", field=\"max_events_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"server\", field=\"clear_entities_cache_interval\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"server\", field=\"max_simultaneous_requests\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"server\", field=\"max_request_queue_limit\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "F" + } + ], + "title": "Watcher config (server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "is_subgraph_watcher" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"jobqueue\", field=\"num_events_in_batch\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"jobqueue\", field=\"block_delay_seconds\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"jobqueue\", field=\"use_block_ranges\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + } + ], + "title": "Watcher config (job queue)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "use_block_ranges" + } + ], + "fields": {} + } + } + ], + "type": "stat" } ], - "refresh": "5s", + "refresh": "10s", "schemaVersion": 38, "tags": [ "ethereum", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json index f41b3298..8e495464 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json @@ -29,7 +29,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -53,7 +53,7 @@ }, "id": 1, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -95,11 +95,11 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head block number of upstream endpoint", + "description": "Chain head block number of the upstream endpoint", "fieldConfig": { "defaults": { "color": { - "fixedColor": "orange", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -123,7 +123,7 @@ }, "id": 11, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -138,6 +138,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_upstream_block_number{job=\"sushi\", instance=\"merkl_sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "Upstream chain head", "type": "stat" }, @@ -146,11 +164,11 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head from a public endpoint (filfox)", + "description": "Chain head from an external public endpoint", "fieldConfig": { "defaults": { "color": { - "fixedColor": "red", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -174,7 +192,7 @@ }, "id": 12, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -189,6 +207,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"filecoin\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "External chain head", "type": "stat" }, @@ -294,6 +330,322 @@ "title": "Last block process duration (s)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "orange", + "index": 2, + "text": "LAZY" + }, + "1": { + "color": "semi-dark-green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"server\", field=\"is_active\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher kind", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 3 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "latest_upstream_block_number{job=\"sushi\", instance=\"merkl_sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"sushi\", instance=\"merkl_sushiswap\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_upstream", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_upstream_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"filecoin\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"sushi\", instance=\"merkl_sushiswap\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_external", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -320,10 +672,10 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 3 + "h": 3, + "w": 8, + "x": 4, + "y": 5 }, "id": 13, "options": { @@ -338,6 +690,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -393,6 +746,100 @@ "title": "Sync status", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-green", + "index": 1, + "text": "REAL TIME" + }, + "1": { + "color": "orange", + "index": 2, + "text": "HISTORICAL" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 5.5 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "is_syncing_historical_blocks{job=\"sushi\", instance=\"merkl_sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher sync mode", + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -520,10 +967,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 0, - "y": 7 + "y": 8 }, "id": 7, "options": { @@ -538,6 +985,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -605,10 +1053,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 6, - "y": 7 + "y": 8 }, "id": 4, "options": { @@ -623,6 +1071,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -749,7 +1198,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"sushi\", instance=\"merkl_sushiswap\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "expr": "pgboss_jobs_total{job=\"sushi\", instance=\"merkl_sushiswap\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -851,7 +1300,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"sushi\", instance=\"merkl_sushiswap\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "expr": "pgboss_jobs_total{job=\"sushi\", instance=\"merkl_sushiswap\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -914,8 +1363,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -1011,8 +1459,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -1108,8 +1555,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -1156,9 +1602,445 @@ ], "title": "Num subgraph entities cached (pruned region)", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 23 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"upstream\", field=\"is_using_rpc_client\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"upstream\", field=\"is_fevm\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"server\", field=\"rpc_supports_block_hash\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"upstream\", field=\"filter_logs_by_topics\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + } + ], + "title": "Watcher config (upstream eth server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "watcher_config_info" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 23 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"server\", field=\"is_subgraph_watcher\"}", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"server\", field=\"max_events_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"server\", field=\"clear_entities_cache_interval\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"server\", field=\"max_simultaneous_requests\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"server\", field=\"max_request_queue_limit\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "F" + } + ], + "title": "Watcher config (server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "is_subgraph_watcher" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 23 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"jobqueue\", field=\"num_events_in_batch\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"jobqueue\", field=\"block_delay_seconds\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"jobqueue\", field=\"use_block_ranges\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + } + ], + "title": "Watcher config (job queue)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "use_block_ranges" + } + ], + "fields": {} + } + } + ], + "type": "stat" } ], - "refresh": "5s", + "refresh": "20s", "schemaVersion": 38, "tags": [ "filecoin", diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json index e602000f..b611f04d 100644 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json +++ b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json @@ -29,7 +29,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -53,7 +53,7 @@ }, "id": 1, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -95,11 +95,11 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head block number of upstream endpoint", + "description": "Chain head block number of the upstream endpoint", "fieldConfig": { "defaults": { "color": { - "fixedColor": "orange", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -123,7 +123,7 @@ }, "id": 11, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -138,6 +138,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_upstream_block_number{job=\"sushi\", instance=\"sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "Upstream chain head", "type": "stat" }, @@ -146,11 +164,11 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Chain head from a public endpoint (filfox)", + "description": "Chain head from an external public endpoint", "fieldConfig": { "defaults": { "color": { - "fixedColor": "red", + "fixedColor": "blue", "mode": "shades" }, "mappings": [], @@ -174,7 +192,7 @@ }, "id": 12, "options": { - "colorMode": "value", + "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -189,6 +207,24 @@ "wideLayout": true }, "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"filecoin\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], "title": "External chain head", "type": "stat" }, @@ -294,6 +330,322 @@ "title": "Last block process duration (s)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "orange", + "index": 2, + "text": "LAZY" + }, + "1": { + "color": "semi-dark-green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"server\", field=\"is_active\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher kind", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 3 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "latest_upstream_block_number{job=\"sushi\", instance=\"sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"sushi\", instance=\"sushiswap\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_upstream", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_upstream_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"filecoin\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=\"sushi\", instance=\"sushiswap\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_external", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -320,10 +672,10 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 3 + "h": 3, + "w": 8, + "x": 4, + "y": 5 }, "id": 13, "options": { @@ -338,6 +690,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -393,6 +746,100 @@ "title": "Sync status", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-green", + "index": 1, + "text": "REAL TIME" + }, + "1": { + "color": "orange", + "index": 2, + "text": "HISTORICAL" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 5.5 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "is_syncing_historical_blocks{job=\"sushi\", instance=\"sushiswap\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher sync mode", + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -520,10 +967,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 0, - "y": 7 + "y": 8 }, "id": 7, "options": { @@ -538,6 +985,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -605,10 +1053,10 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 6, "x": 6, - "y": 7 + "y": 8 }, "id": 4, "options": { @@ -623,6 +1071,7 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto", "wideLayout": true }, @@ -749,7 +1198,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"sushi\", instance=\"sushiswap\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}", + "expr": "pgboss_jobs_total{job=\"sushi\", instance=\"sushiswap\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -851,7 +1300,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"sushi\", instance=\"sushiswap\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}", + "expr": "pgboss_jobs_total{job=\"sushi\", instance=\"sushiswap\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -914,8 +1363,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -1011,8 +1459,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -1108,8 +1555,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -1156,9 +1602,445 @@ ], "title": "Num subgraph entities cached (pruned region)", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 23 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"upstream\", field=\"is_using_rpc_client\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"upstream\", field=\"is_fevm\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"server\", field=\"rpc_supports_block_hash\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"upstream\", field=\"filter_logs_by_topics\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + } + ], + "title": "Watcher config (upstream eth server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "watcher_config_info" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 23 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"server\", field=\"is_subgraph_watcher\"}", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"server\", field=\"max_events_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"server\", field=\"clear_entities_cache_interval\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"server\", field=\"max_simultaneous_requests\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"server\", field=\"max_request_queue_limit\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "F" + } + ], + "title": "Watcher config (server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "is_subgraph_watcher" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 23 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"jobqueue\", field=\"num_events_in_batch\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"jobqueue\", field=\"block_delay_seconds\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"jobqueue\", field=\"use_block_ranges\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + } + ], + "title": "Watcher config (job queue)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "use_block_ranges" + } + ], + "fields": {} + } + } + ], + "type": "stat" } ], - "refresh": "5s", + "refresh": "20s", "schemaVersion": 38, "tags": [ "filecoin", @@ -1174,6 +2056,6 @@ "timepicker": {}, "timezone": "", "title": "sushiswap watcher", - "version": 3, + "version": 1, "weekStart": "" } diff --git a/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml b/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml index e8e7b8b8..40068287 100644 --- a/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml +++ b/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml @@ -1,5 +1,5 @@ global: - scrape_interval: 15s + scrape_interval: 10s evaluation_interval: 15s rule_files: @@ -10,3 +10,33 @@ scrape_configs: - job_name: prometheus static_configs: - targets: ['localhost:9090'] + + - job_name: 'blackbox' + scrape_interval: 10s + metrics_path: /probe + params: + module: [http_2xx] + static_configs: + # Add URLs to be monitored below + - targets: + # - https://github.com + relabel_configs: + - source_labels: [__address__] + regex: (.*)(:80)? + target_label: __param_target + - source_labels: [__param_target] + regex: (.*) + target_label: instance + replacement: ${1} + - source_labels: [] + regex: .* + target_label: __address__ + replacement: blackbox:9115 + + - job_name: chain_heads + scrape_interval: 10s + metrics_path: /metrics + scheme: http + static_configs: + - targets: ['chain-head-exporter:5000'] + diff --git a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml index 8d3b3f4d..57494ce3 100644 --- a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml +++ b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml @@ -84,8 +84,6 @@ subgraphEventsOrder = true # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime blockDelayInMilliSecs = 30000 - prefetchBlocksInMem = false - prefetchBlockCount = 10 # Boolean to switch between modes of processing events when starting the server. # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them. diff --git a/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml index e45cf0ea..24449d4f 100644 --- a/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml +++ b/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml @@ -84,8 +84,6 @@ subgraphEventsOrder = true # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime blockDelayInMilliSecs = 30000 - prefetchBlocksInMem = false - prefetchBlockCount = 10 # Boolean to switch between modes of processing events when starting the server. # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them. diff --git a/stack_orchestrator/data/stacks/azimuth/README.md b/stack_orchestrator/data/stacks/azimuth/README.md index f7d93f33..21200369 100644 --- a/stack_orchestrator/data/stacks/azimuth/README.md +++ b/stack_orchestrator/data/stacks/azimuth/README.md @@ -40,20 +40,36 @@ network: ports: watcher-db: - 0.0.0.0:15432:5432 + azimuth-watcher-job-runner: + - 0.0.0.0:9000:9000 azimuth-watcher-server: - 0.0.0.0:3001:3001 + censures-watcher-job-runner: + - 0.0.0.0:9002:9002 censures-watcher-server: - 0.0.0.0:3002:3002 + claims-watcher-job-runner: + - 0.0.0.0:9004:9004 claims-watcher-server: - 0.0.0.0:3003:3003 + conditional-star-release-watcher-job-runner: + - 0.0.0.0:9006:9006 conditional-star-release-watcher-server: - 0.0.0.0:3004:3004 + delegated-sending-watcher-job-runner: + - 0.0.0.0:9008:9008 delegated-sending-watcher-server: - 0.0.0.0:3005:3005 + ecliptic-watcher-job-runner: + - 0.0.0.0:9010:9010 ecliptic-watcher-server: - 0.0.0.0:3006:3006 + linear-star-release-watcher-job-runner: + - 0.0.0.0:9012:9012 linear-star-release-watcher-server: - 0.0.0.0:3007:3007 + polls-watcher-job-runner: + - 0.0.0.0:9014:9014 polls-watcher-server: - 0.0.0.0:3008:3008 gateway-server: diff --git a/stack_orchestrator/data/stacks/azimuth/stack.yml b/stack_orchestrator/data/stacks/azimuth/stack.yml index 7adbe663..374f2af0 100644 --- a/stack_orchestrator/data/stacks/azimuth/stack.yml +++ b/stack_orchestrator/data/stacks/azimuth/stack.yml @@ -1,7 +1,7 @@ version: "1.0" name: azimuth repos: - - github.com/cerc-io/azimuth-watcher-ts@v0.1.2 + - github.com/cerc-io/azimuth-watcher-ts@v0.1.3 containers: - cerc/watcher-azimuth pods: diff --git a/stack_orchestrator/data/stacks/monitoring/README.md b/stack_orchestrator/data/stacks/monitoring/README.md index 6f1fc227..803da548 100644 --- a/stack_orchestrator/data/stacks/monitoring/README.md +++ b/stack_orchestrator/data/stacks/monitoring/README.md @@ -1,8 +1,26 @@ # monitoring * Instructions to setup and run a Prometheus server and Grafana dashboard +* Comes with the following built-in exporters / dashboards: + * [Prometheus Blackbox Exporter](https://grafana.com/grafana/dashboards/7587-prometheus-blackbox-exporter/) - for tracking HTTP endpoints + * [NodeJS Application Dashboard](https://grafana.com/grafana/dashboards/11159-nodejs-application-dashboard/) - for default NodeJS metrics + * Chain Head Exporter - for tracking chain heads given external ETH RPC endpoints * See [monitoring-watchers.md](./monitoring-watchers.md) for an example usage of the stack with pre-configured dashboards for watchers +## Setup + +Clone required repositories: + +```bash +laconic-so --stack monitoring setup-repositories --git-ssh --pull +``` + +Build the container images: + +```bash +laconic-so --stack monitoring build-containers +``` + ## Create a deployment First, create a spec file for the deployment, which will map the stack's ports and volumes to the host: @@ -43,7 +61,7 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --de ### Prometheus Config -Add desired scrape configs to prometheus config file (`monitoring-deployment/config/monitoring/prometheus/prometheus.yml`) in the deployment folder; for example: +* Add desired scrape configs to prometheus config file (`monitoring-deployment/config/monitoring/prometheus/prometheus.yml`) in the deployment folder; for example: ```yml ... @@ -54,12 +72,43 @@ Add desired scrape configs to prometheus config file (`monitoring-deployment/con - targets: [':'] ``` +* Also update the `blackbox` job to add any endpoints to be monitored on the Blackbox dashboard: + + ```yml + ... + - job_name: 'blackbox' + ... + static_configs: + # Add URLs to be monitored below + - targets: + - + - + ``` + Note: Use `host.docker.internal` as host to access ports on the host machine ### Grafana Config Place the dashboard json files in grafana dashboards config directory (`monitoring-deployment/config/monitoring/grafana/dashboards`) in the deployment folder +### Env + +Set the following env variables in the deployment env config file (`monitoring-deployment/config.env`): + + ```bash + # External ETH RPC endpoint (ethereum) + # (Optional, default: https://mainnet.infura.io/v3) + CERC_ETH_RPC_ENDPOINT= + + # Infura key to be used + # (Optional, used with ETH_RPC_ENDPOINT if provided) + CERC_INFURA_KEY= + + # External ETH RPC endpoint (filecoin) + # (Optional, default: https://api.node.glif.io/rpc/v1) + CERC_FIL_RPC_ENDPOINT= + ``` + ## Start the stack Start the deployment: diff --git a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md index f49bdcda..4ef43b36 100644 --- a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md +++ b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md @@ -4,7 +4,7 @@ Instructions to setup and run monitoring stack with pre-configured watcher dashb ## Create a deployment -First, create a spec file for the deployment, which will map the stack's ports and volumes to the host: +After completing [setup](./README.md#setup), create a spec file for the deployment, which will map the stack's ports and volumes to the host: ```bash laconic-so --stack monitoring deploy init --output monitoring-watchers-spec.yml @@ -41,7 +41,14 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers ```yml ... + - job_name: 'blackbox' + ... + static_configs: + - targets: + - + ... - job_name: azimuth + scrape_interval: 10s metrics_path: /metrics scheme: http static_configs: @@ -71,6 +78,7 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers instance: 'polls' - job_name: sushi + scrape_interval: 20s metrics_path: /metrics scheme: http static_configs: @@ -90,6 +98,15 @@ In the deployment folder, copy over the pre-configured watcher dashboard JSON fi cp -r monitoring-watchers-deployment/config/monitoring/grafana/watcher-dashboards/* monitoring-watchers-deployment/config/monitoring/grafana/dashboards/ ``` +### Env + +Set the following env variables in the deployment env config file (`monitoring-watchers-deployment/config.env`): + + ```bash + # Infura key to be used + CERC_INFURA_KEY= + ``` + ## Start the stack Start the deployment: diff --git a/stack_orchestrator/data/stacks/monitoring/stack.yml b/stack_orchestrator/data/stacks/monitoring/stack.yml index c3532024..058ca971 100644 --- a/stack_orchestrator/data/stacks/monitoring/stack.yml +++ b/stack_orchestrator/data/stacks/monitoring/stack.yml @@ -1,7 +1,9 @@ version: "0.1" name: monitoring repos: + - github.com/cerc-io/watcher-ts@v0.2.79 containers: + - cerc/watcher-ts pods: - prom-server - grafana diff --git a/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml b/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml index 49c604bf..90766012 100644 --- a/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml +++ b/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml @@ -2,7 +2,7 @@ version: "1.0" name: sushiswap-v3 description: "SushiSwap v3 watcher stack" repos: - - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.4 + - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.5 containers: - cerc/watcher-sushiswap-v3 pods: From 38622fb33c4b6f0fd7bbac0098b8716a5f9567b7 Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Thu, 21 Dec 2023 13:41:36 +0530 Subject: [PATCH 58/75] [WIP] Use templating for watcher dashboard and add Postgres exporter (#695) * Add Postgres exporter and it's dashboard * Use templating for watcher dashboard * Add subgraph related panels to watcher dashboard * Remove individual watcher dashboards and update instructions --- .../compose/docker-compose-prom-server.yml | 10 + .../dashboards/postgres-dashboard.json | 2963 +++++++++++++++++ .../grafana/dashboards/watcher-dashboard.json | 2187 ++++++++++++ .../grafana/watcher-dashboard-template.json | 1772 ---------- .../azimuth/azimuth-watcher-dashboard.json | 1772 ---------- .../azimuth/censures-watcher-dashboard.json | 1772 ---------- .../azimuth/claims-watcher-dashboard.json | 1772 ---------- ...tional-star-release-watcher-dashboard.json | 1772 ---------- ...ted-sending-watcher-watcher-dashboard.json | 1772 ---------- .../azimuth/ecliptic-watcher-dashboard.json | 1772 ---------- ...linear-star-release-watcher-dashboard.json | 1772 ---------- .../azimuth/polls-watcher-dashboard.json | 1772 ---------- .../merkl-sushiswap-watcher-dashboard.json | 2061 ------------ .../sushi/sushiswap-watcher-dashboard.json | 2061 ------------ .../config/monitoring/postgres-exporter.yml | 8 + .../monitoring/prometheus/prometheus.yml | 18 + .../data/stacks/monitoring/README.md | 17 +- .../stacks/monitoring/monitoring-watchers.md | 22 +- 18 files changed, 5214 insertions(+), 20081 deletions(-) create mode 100644 stack_orchestrator/data/config/monitoring/grafana/dashboards/postgres-dashboard.json create mode 100644 stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json delete mode 100644 stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json delete mode 100644 stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json delete mode 100644 stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json delete mode 100644 stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json delete mode 100644 stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json delete mode 100644 stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json delete mode 100644 stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json delete mode 100644 stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json delete mode 100644 stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json delete mode 100644 stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json delete mode 100644 stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json create mode 100644 stack_orchestrator/data/config/monitoring/postgres-exporter.yml diff --git a/stack_orchestrator/data/compose/docker-compose-prom-server.yml b/stack_orchestrator/data/compose/docker-compose-prom-server.yml index 27b8c18c..b085e295 100644 --- a/stack_orchestrator/data/compose/docker-compose-prom-server.yml +++ b/stack_orchestrator/data/compose/docker-compose-prom-server.yml @@ -43,5 +43,15 @@ services: extra_hosts: - "host.docker.internal:host-gateway" + postgres-exporter: + image: quay.io/prometheuscommunity/postgres-exporter + restart: always + volumes: + - ../config/monitoring/postgres-exporter.yml:/postgres_exporter.yml + ports: + - '9187' + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: prometheus_data: diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/postgres-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/postgres-dashboard.json new file mode 100644 index 00000000..18e641e6 --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/postgres-dashboard.json @@ -0,0 +1,2963 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "This dashboard works with postgres_exporter for prometheus", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 9628, + "graphTooltip": 0, + "id": 15, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 34, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "General Counters", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 36, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_static{release=\"$release\", instance=\"$instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{short_version}}", + "refId": "A" + } + ], + "title": "Version", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "start time of the process", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "dateTimeFromNow" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 28, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_postmaster_start_time_seconds{release=\"$release\", instance=\"$instance\"} * 1000", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Start Time", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 10, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "SUM(pg_stat_database_tup_fetched{datname=~\"$datname\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 4 + } + ], + "title": "Current fetch data", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 11, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "SUM(pg_stat_database_tup_inserted{release=\"$release\", datname=~\"$datname\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 4 + } + ], + "title": "Current insert data", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 12, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "SUM(pg_stat_database_tup_updated{datname=~\"$datname\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 4 + } + ], + "title": "Current update data", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 38, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_settings_max_connections{release=\"$release\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Max Connections", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 32, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "Settings", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 4 + }, + "id": 40, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_settings_shared_buffers_bytes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Shared Buffers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 4 + }, + "id": 42, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_settings_effective_cache_size_bytes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Effective Cache", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 6, + "y": 4 + }, + "id": 44, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_settings_maintenance_work_mem_bytes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Maintenance Work Mem", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 9, + "y": 4 + }, + "id": 46, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_settings_work_mem_bytes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Work Mem", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 12, + "y": 4 + }, + "id": 48, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_settings_max_wal_size_bytes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Max WAL Size", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 15, + "y": 4 + }, + "id": 50, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_settings_random_page_cost{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Random Page Cost", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 18, + "y": 4 + }, + "id": 52, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_settings_seq_page_cost", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Seq Page Cost", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 20, + "y": 4 + }, + "id": 54, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_settings_max_worker_processes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Max Worker Processes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 22, + "y": 4 + }, + "id": 56, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_settings_max_parallel_workers{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Max Parallel Workers", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 30, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "Database Stats", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 1, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 3, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_stat_activity_count{datname=~\"$datname\", instance=~\"$instance\", state=\"active\"} !=0", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{datname}}, s: {{state}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Active sessions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 8 + }, + "hiddenSeries": false, + "id": 60, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(pg_stat_database_xact_commit{instance=\"$instance\", datname=~\"$datname\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{datname}} commits", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(pg_stat_database_xact_rollback{instance=\"$instance\", datname=~\"$datname\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{datname}} rollbacks", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Transactions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 8 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_stat_database_tup_updated{datname=~\"$datname\", instance=~\"$instance\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Update data", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 15 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_stat_database_tup_fetched{datname=~\"$datname\", instance=~\"$instance\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Fetch data (SELECT)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 15 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_stat_database_tup_inserted{datname=~\"$datname\", instance=~\"$instance\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Insert data", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 0, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 15 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_locks_count{datname=~\"$datname\", instance=~\"$instance\", mode=~\"$mode\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}},{{mode}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Lock tables", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 22 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "total", + "sortDesc": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_stat_database_tup_returned{datname=~\"$datname\", instance=~\"$instance\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Return data", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 22 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_stat_activity_count{datname=~\"$datname\", instance=~\"$instance\", state=~\"idle|idle in transaction|idle in transaction (aborted)\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}}, s: {{state}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Idle sessions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 22 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_stat_database_tup_deleted{datname=~\"$datname\", instance=~\"$instance\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Delete data", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 2, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 29 + }, + "hiddenSeries": false, + "id": 62, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_stat_database_blks_hit{instance=\"$instance\", datname=~\"$datname\"} / (pg_stat_database_blks_read{instance=\"$instance\", datname=~\"$datname\"} + pg_stat_database_blks_hit{instance=\"$instance\", datname=~\"$datname\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ datname }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Cache Hit Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 4, + "format": "percentunit", + "label": "", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 29 + }, + "hiddenSeries": false, + "id": 64, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(pg_stat_bgwriter_buffers_backend{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "buffers_backend", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(pg_stat_bgwriter_buffers_alloc{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "buffers_alloc", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "backend_fsync", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(pg_stat_bgwriter_buffers_checkpoint{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "buffers_checkpoint", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(pg_stat_bgwriter_buffers_clean{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "buffers_clean", + "refId": "E" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Buffers (bgwriter)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 0, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 29 + }, + "hiddenSeries": false, + "id": 66, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(pg_stat_database_conflicts{instance=\"$instance\", datname=~\"$datname\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{datname}} conflicts", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(pg_stat_database_deadlocks{instance=\"$instance\", datname=~\"$datname\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{datname}} deadlocks", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Conflicts/Deadlocks", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total amount of data written to temporary files by queries in this database. All temporary files are counted, regardless of why the temporary file was created, and regardless of the log_temp_files setting.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 36 + }, + "hiddenSeries": false, + "id": 68, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(pg_stat_database_temp_bytes{instance=\"$instance\", datname=~\"$datname\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{datname}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Temp File (Bytes)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 16, + "x": 8, + "y": 36 + }, + "hiddenSeries": false, + "id": 70, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(pg_stat_bgwriter_checkpoint_write_time{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "write_time - Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk.", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(pg_stat_bgwriter_checkpoint_sync_time{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "sync_time - Total amount of time that has been spent in the portion of checkpoint processing where files are synchronized to disk.", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Checkpoint Stats", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": "10s", + "schemaVersion": 38, + "tags": [ + "postgres", + "db", + "stats" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "auto": true, + "auto_count": 200, + "auto_min": "1s", + "current": { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + "hide": 0, + "label": "Interval", + "name": "interval", + "options": [ + { + "selected": true, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "1s", + "value": "1s" + }, + { + "selected": false, + "text": "5s", + "value": "5s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + } + ], + "query": "1s,5s,1m,5m,1h,6h,1d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "query_result(pg_exporter_last_scrape_duration_seconds)", + "refresh": 2, + "regex": "/.*kubernetes_namespace=\"([^\"]+).*/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Release", + "multi": false, + "name": "release", + "options": [], + "query": "query_result(pg_exporter_last_scrape_duration_seconds{kubernetes_namespace=\"$namespace\"})", + "refresh": 2, + "regex": "/.*release=\"([^\"]+)/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": "query_result(pg_up{release=\"$release\"})", + "refresh": 1, + "regex": "/.*instance=\"([^\"]+).*/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Database", + "multi": true, + "name": "datname", + "options": [], + "query": "label_values(datname)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Lock table", + "multi": true, + "name": "mode", + "options": [], + "query": "label_values({mode=~\"accessexclusivelock|accesssharelock|exclusivelock|rowexclusivelock|rowsharelock|sharelock|sharerowexclusivelock|shareupdateexclusivelock\"}, mode)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "PostgreSQL Database", + "uid": "000000039", + "version": 1, + "weekStart": "" +} diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json new file mode 100644 index 00000000..fe5fd244 --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json @@ -0,0 +1,2187 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [ + { + "asDropdown": false, + "icon": "dashboard", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Blackbox", + "tooltip": "", + "type": "link", + "url": "/d/xtkCtBkiz/prometheus-blackbox-exporter" + }, + { + "asDropdown": false, + "icon": "dashboard", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "PG", + "tooltip": "", + "type": "link", + "url": "/d/000000039/postgresql-database" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 25, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_indexed\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Latest indexed block", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head block number of the upstream endpoint", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 11, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_upstream_block_number{job=~\"$job\", instance=~\"$watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Upstream head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head from an external public endpoint", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 12, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"$target_chain\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "External head ($target_chain)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-green", + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-green", + "value": null + }, + { + "color": "red", + "value": 35 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "last_block_process_duration_seconds{job=~\"$job\", instance=~\"$watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Last block process duration (s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "orange", + "index": 2, + "text": "LAZY" + }, + "1": { + "color": "semi-dark-green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"server\", field=\"is_active\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher kind", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "latest_upstream_block_number{job=~\"$job\", instance=~\"$watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_upstream", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_upstream_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 4 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"$target_chain\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_external", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Watcher sync status block numbers", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 4, + "y": 6 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_processed_block_number{job=~\"$job\", instance=~\"$watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "latest_processed", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_canonical\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"intial_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "C" + } + ], + "title": "Sync status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-green", + "index": 1, + "text": "REAL TIME" + }, + "1": { + "color": "orange", + "index": 2, + "text": "HISTORICAL" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 6.5 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "is_syncing_historical_blocks{job=~\"$job\", instance=~\"$watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher sync mode", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_block_num_events_total{job=~\"$job\", instance=~\"$watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number of events in the last block", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of entries in the block_progress and event table", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "dark-yellow", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 9 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "block_progress_total{job=~\"$job\", instance=~\"$watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "block_progress", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "event_total{job=~\"$job\", instance=~\"$watcher\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "event", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "DB indexed count ", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 9 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "database_size_bytes{job=~\"$job\", instance=~\"$watcher\", type=\"watcher\"} / 1024 / 1024", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "database_size_bytes{job=~\"$job\", instance=~\"$watcher\", type=\"job-queue\"} / 1024 / 1024", + "hide": false, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "B" + } + ], + "title": "DB size (MB)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=~\"$job\", instance=~\"$watcher\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num block-processing jobs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=~\"$job\", instance=~\"$watcher\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num event-processing jobs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 18 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"upstream\", field=\"is_using_rpc_client\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"upstream\", field=\"is_fevm\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"server\", field=\"rpc_supports_block_hash\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"upstream\", field=\"filter_logs_by_topics\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + } + ], + "title": "Watcher config (upstream eth server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "watcher_config_info" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 18 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"server\", field=\"is_subgraph_watcher\"}", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"server\", field=\"max_events_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"server\", field=\"clear_entities_cache_interval\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"server\", field=\"max_simultaneous_requests\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"server\", field=\"max_request_queue_limit\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "F" + } + ], + "title": "Watcher config (server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "is_subgraph_watcher" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 18 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"jobqueue\", field=\"num_events_in_batch\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"jobqueue\", field=\"block_delay_seconds\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"jobqueue\", field=\"use_block_ranges\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + } + ], + "title": "Watcher config (job queue)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "use_block_ranges" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of subgraph entities loaded in event processing", + "hide": true, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 23 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "event_processing_load_entity_total{job=~\"$job\", instance=~\"$watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num subgraph entities loaded", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of cache hits for subgraph entites in event processing ", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 23 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "event_processing_load_entity_cache_hit_total{job=~\"$job\", instance=~\"$watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num subgraph entities cache hits", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of subgraph entities currently present in pruned region of cache", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 23 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "cached_pruned_entities_total{job=~\"$job\", instance=~\"$watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num subgraph entities cached (pruned region)", + "type": "timeseries" + } + ], + "repeat": "watcher", + "repeatDirection": "h", + "title": "$watcher", + "type": "row" + } + ], + "refresh": "10s", + "schemaVersion": 38, + "tags": [ + "watcher" + ], + "templating": { + "list": [ + { + "current": { + "isNone": true, + "selected": false, + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(watcher_config_info,job)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "job", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(watcher_config_info,job)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "isNone": true, + "selected": false, + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values({job=\"$job\"},instance)", + "hide": 0, + "includeAll": true, + "label": "", + "multi": false, + "name": "watcher", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({job=\"$job\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values({instance=~\"$watcher\"},chain)", + "hide": 2, + "includeAll": false, + "multi": false, + "name": "target_chain", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({instance=~\"$watcher\"},chain)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Watchers", + "version": 1, + "weekStart": "" +} diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json deleted file mode 100644 index 1e0f4e17..00000000 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json +++ /dev/null @@ -1,1772 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", kind=\"latest_indexed\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Latest indexed block", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Chain head block number of the upstream endpoint", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 0 - }, - "id": 11, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_upstream_block_number{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Upstream chain head", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Chain head from an external public endpoint", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 0 - }, - "id": 12, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_block_number{job=\"chain_heads\", chain=\"REPLACE_WITH_WATCHER_CHAIN\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "External chain head", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "semi-dark-green", - "mode": "thresholds" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - }, - { - "color": "red", - "value": 35 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "last_block_process_duration_seconds{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Last block process duration (s)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "orange", - "index": 2, - "text": "LAZY" - }, - "1": { - "color": "semi-dark-green", - "index": 1, - "text": "ACTIVE" - } - }, - "type": "value" - }, - { - "options": { - "match": "null+nan", - "result": { - "color": "semi-dark-red", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2.5, - "w": 4, - "x": 0, - "y": 3 - }, - "id": 20, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"server\", field=\"is_active\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Watcher kind", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 8 - }, - { - "color": "red", - "value": 16 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 4, - "y": 3 - }, - "id": 22, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "valueSize": 25 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "latest_upstream_block_number{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sync_status_block_number{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", kind=\"latest_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "B" - } - ], - "title": "diff_upstream", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "", - "binary": { - "left": "latest_upstream_block_number", - "operator": "-", - "right": "sync_status_block_number" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 8 - }, - { - "color": "red", - "value": 16 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 8, - "y": 3 - }, - "id": 24, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "valueSize": 25 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_block_number{job=\"chain_heads\", chain=\"REPLACE_WITH_WATCHER_CHAIN\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sync_status_block_number{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", kind=\"latest_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "B" - } - ], - "title": "diff_external", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "", - "binary": { - "left": "latest_block_number", - "operator": "-", - "right": "sync_status_block_number" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Watcher sync status block numbers", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 4, - "y": 5 - }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "last_processed_block_number{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "latest_processed", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", kind=\"latest_canonical\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{kind}}", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", kind=\"intial_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{kind}}", - "range": true, - "refId": "C" - } - ], - "title": "Sync status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "semi-dark-green", - "index": 1, - "text": "REAL TIME" - }, - "1": { - "color": "orange", - "index": 2, - "text": "HISTORICAL" - } - }, - "type": "value" - }, - { - "options": { - "match": "null+nan", - "result": { - "color": "semi-dark-red", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2.5, - "w": 4, - "x": 0, - "y": 5.5 - }, - "id": 16, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "is_syncing_historical_blocks{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Watcher sync mode", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "orange", - "mode": "shades" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 6 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "last_block_num_events_total{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Number of events in the last block", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of entries in the block_progress and event table", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "dark-yellow", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "orange", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 8 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "block_progress_total{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "block_progress", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "event_total{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "event", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "DB indexed count ", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "purple", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 6, - "y": 8 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "database_size_bytes{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", type=\"watcher\"} / 1024 / 1024", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{type}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "database_size_bytes{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", type=\"job-queue\"} / 1024 / 1024", - "hide": false, - "instant": false, - "legendFormat": "{{type}}", - "range": true, - "refId": "B" - } - ], - "title": "DB size (MB)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of pgboss jobs in the db", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 11 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num block-processing jobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of pgboss jobs in the db", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 11 - }, - "id": 15, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num event-processing jobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 17 - }, - "id": 18, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"upstream\", field=\"is_using_rpc_client\"}", - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"upstream\", field=\"is_fevm\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"server\", field=\"rpc_supports_block_hash\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"upstream\", field=\"filter_logs_by_topics\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - } - ], - "title": "Watcher config (upstream eth server)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "watcher_config_info" - } - ], - "fields": {} - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 17 - }, - "id": 17, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "value_and_name", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"server\", field=\"is_subgraph_watcher\"}", - "format": "time_series", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"server\", field=\"max_events_block_range\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"server\", field=\"clear_entities_cache_interval\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"server\", field=\"max_simultaneous_requests\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"server\", field=\"max_request_queue_limit\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "F" - } - ], - "title": "Watcher config (server)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "is_subgraph_watcher" - } - ], - "fields": {} - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 17 - }, - "id": 19, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"jobqueue\", field=\"num_events_in_batch\"}", - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"jobqueue\", field=\"block_delay_seconds\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"jobqueue\", field=\"use_block_ranges\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - } - ], - "title": "Watcher config (job queue)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "use_block_ranges" - } - ], - "fields": {} - } - } - ], - "type": "stat" - } - ], - "refresh": "10s", - "schemaVersion": 38, - "tags": [ - "REPLACE_WITH_WATCHER_CHAIN", - "REPLACE_WITH_WATCHER_JOB" - ], - "templating": { - "list": [] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "REPLACE_WITH_WATCHER_NAME watcher", - "version": 3, - "weekStart": "" -} diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json deleted file mode 100644 index 3a67d7f4..00000000 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json +++ /dev/null @@ -1,1772 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"azimuth\", kind=\"latest_indexed\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Latest indexed block", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Chain head block number of the upstream endpoint", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 0 - }, - "id": 11, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"azimuth\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Upstream chain head", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Chain head from an external public endpoint", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 0 - }, - "id": 12, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "External chain head", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "semi-dark-green", - "mode": "thresholds" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - }, - { - "color": "red", - "value": 35 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "last_block_process_duration_seconds{job=\"azimuth\", instance=\"azimuth\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Last block process duration (s)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "orange", - "index": 2, - "text": "LAZY" - }, - "1": { - "color": "semi-dark-green", - "index": 1, - "text": "ACTIVE" - } - }, - "type": "value" - }, - { - "options": { - "match": "null+nan", - "result": { - "color": "semi-dark-red", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2.5, - "w": 4, - "x": 0, - "y": 3 - }, - "id": 20, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"server\", field=\"is_active\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Watcher kind", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 8 - }, - { - "color": "red", - "value": 16 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 4, - "y": 3 - }, - "id": 22, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "valueSize": 25 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"azimuth\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"azimuth\", kind=\"latest_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "B" - } - ], - "title": "diff_upstream", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "", - "binary": { - "left": "latest_upstream_block_number", - "operator": "-", - "right": "sync_status_block_number" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 8 - }, - { - "color": "red", - "value": 16 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 8, - "y": 3 - }, - "id": 24, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "valueSize": 25 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"azimuth\", kind=\"latest_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "B" - } - ], - "title": "diff_external", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "", - "binary": { - "left": "latest_block_number", - "operator": "-", - "right": "sync_status_block_number" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Watcher sync status block numbers", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 4, - "y": 5 - }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "last_processed_block_number{job=\"azimuth\", instance=\"azimuth\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "latest_processed", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"azimuth\", kind=\"latest_canonical\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{kind}}", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"azimuth\", kind=\"intial_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{kind}}", - "range": true, - "refId": "C" - } - ], - "title": "Sync status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "semi-dark-green", - "index": 1, - "text": "REAL TIME" - }, - "1": { - "color": "orange", - "index": 2, - "text": "HISTORICAL" - } - }, - "type": "value" - }, - { - "options": { - "match": "null+nan", - "result": { - "color": "semi-dark-red", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2.5, - "w": 4, - "x": 0, - "y": 5.5 - }, - "id": 16, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "is_syncing_historical_blocks{job=\"azimuth\", instance=\"azimuth\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Watcher sync mode", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "orange", - "mode": "shades" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 6 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "last_block_num_events_total{job=\"azimuth\", instance=\"azimuth\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Number of events in the last block", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of entries in the block_progress and event table", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "dark-yellow", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "orange", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 8 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "block_progress_total{job=\"azimuth\", instance=\"azimuth\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "block_progress", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "event_total{job=\"azimuth\", instance=\"azimuth\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "event", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "DB indexed count ", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "purple", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 6, - "y": 8 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "database_size_bytes{job=\"azimuth\", instance=\"azimuth\", type=\"watcher\"} / 1024 / 1024", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{type}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "database_size_bytes{job=\"azimuth\", instance=\"azimuth\", type=\"job-queue\"} / 1024 / 1024", - "hide": false, - "instant": false, - "legendFormat": "{{type}}", - "range": true, - "refId": "B" - } - ], - "title": "DB size (MB)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of pgboss jobs in the db", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 11 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"azimuth\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num block-processing jobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of pgboss jobs in the db", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 11 - }, - "id": 15, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"azimuth\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num event-processing jobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 17 - }, - "id": 18, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"upstream\", field=\"is_using_rpc_client\"}", - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"upstream\", field=\"is_fevm\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"server\", field=\"rpc_supports_block_hash\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"upstream\", field=\"filter_logs_by_topics\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - } - ], - "title": "Watcher config (upstream eth server)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "watcher_config_info" - } - ], - "fields": {} - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 17 - }, - "id": 17, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "value_and_name", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"server\", field=\"is_subgraph_watcher\"}", - "format": "time_series", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"server\", field=\"max_events_block_range\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"server\", field=\"clear_entities_cache_interval\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"server\", field=\"max_simultaneous_requests\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"server\", field=\"max_request_queue_limit\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "F" - } - ], - "title": "Watcher config (server)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "is_subgraph_watcher" - } - ], - "fields": {} - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 17 - }, - "id": 19, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"jobqueue\", field=\"num_events_in_batch\"}", - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"jobqueue\", field=\"block_delay_seconds\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"jobqueue\", field=\"use_block_ranges\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"azimuth\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - } - ], - "title": "Watcher config (job queue)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "use_block_ranges" - } - ], - "fields": {} - } - } - ], - "type": "stat" - } - ], - "refresh": "10s", - "schemaVersion": 38, - "tags": [ - "ethereum", - "azimuth" - ], - "templating": { - "list": [] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "azimuth watcher", - "version": 3, - "weekStart": "" -} diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json deleted file mode 100644 index 04c93969..00000000 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json +++ /dev/null @@ -1,1772 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"censures\", kind=\"latest_indexed\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Latest indexed block", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Chain head block number of the upstream endpoint", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 0 - }, - "id": 11, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"censures\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Upstream chain head", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Chain head from an external public endpoint", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 0 - }, - "id": 12, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "External chain head", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "semi-dark-green", - "mode": "thresholds" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - }, - { - "color": "red", - "value": 35 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "last_block_process_duration_seconds{job=\"azimuth\", instance=\"censures\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Last block process duration (s)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "orange", - "index": 2, - "text": "LAZY" - }, - "1": { - "color": "semi-dark-green", - "index": 1, - "text": "ACTIVE" - } - }, - "type": "value" - }, - { - "options": { - "match": "null+nan", - "result": { - "color": "semi-dark-red", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2.5, - "w": 4, - "x": 0, - "y": 3 - }, - "id": 20, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"server\", field=\"is_active\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Watcher kind", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 8 - }, - { - "color": "red", - "value": 16 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 4, - "y": 3 - }, - "id": 22, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "valueSize": 25 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"censures\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"censures\", kind=\"latest_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "B" - } - ], - "title": "diff_upstream", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "", - "binary": { - "left": "latest_upstream_block_number", - "operator": "-", - "right": "sync_status_block_number" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 8 - }, - { - "color": "red", - "value": 16 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 8, - "y": 3 - }, - "id": 24, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "valueSize": 25 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"censures\", kind=\"latest_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "B" - } - ], - "title": "diff_external", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "", - "binary": { - "left": "latest_block_number", - "operator": "-", - "right": "sync_status_block_number" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Watcher sync status block numbers", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 4, - "y": 5 - }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "last_processed_block_number{job=\"azimuth\", instance=\"censures\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "latest_processed", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"censures\", kind=\"latest_canonical\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{kind}}", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"censures\", kind=\"intial_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{kind}}", - "range": true, - "refId": "C" - } - ], - "title": "Sync status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "semi-dark-green", - "index": 1, - "text": "REAL TIME" - }, - "1": { - "color": "orange", - "index": 2, - "text": "HISTORICAL" - } - }, - "type": "value" - }, - { - "options": { - "match": "null+nan", - "result": { - "color": "semi-dark-red", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2.5, - "w": 4, - "x": 0, - "y": 5.5 - }, - "id": 16, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "is_syncing_historical_blocks{job=\"azimuth\", instance=\"censures\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Watcher sync mode", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "orange", - "mode": "shades" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 6 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "last_block_num_events_total{job=\"azimuth\", instance=\"censures\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Number of events in the last block", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of entries in the block_progress and event table", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "dark-yellow", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "orange", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 8 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "block_progress_total{job=\"azimuth\", instance=\"censures\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "block_progress", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "event_total{job=\"azimuth\", instance=\"censures\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "event", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "DB indexed count ", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "purple", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 6, - "y": 8 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "database_size_bytes{job=\"azimuth\", instance=\"censures\", type=\"watcher\"} / 1024 / 1024", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{type}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "database_size_bytes{job=\"azimuth\", instance=\"censures\", type=\"job-queue\"} / 1024 / 1024", - "hide": false, - "instant": false, - "legendFormat": "{{type}}", - "range": true, - "refId": "B" - } - ], - "title": "DB size (MB)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of pgboss jobs in the db", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 11 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"censures\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num block-processing jobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of pgboss jobs in the db", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 11 - }, - "id": 15, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"censures\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num event-processing jobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 17 - }, - "id": 18, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"upstream\", field=\"is_using_rpc_client\"}", - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"upstream\", field=\"is_fevm\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"server\", field=\"rpc_supports_block_hash\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"upstream\", field=\"filter_logs_by_topics\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - } - ], - "title": "Watcher config (upstream eth server)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "watcher_config_info" - } - ], - "fields": {} - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 17 - }, - "id": 17, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "value_and_name", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"server\", field=\"is_subgraph_watcher\"}", - "format": "time_series", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"server\", field=\"max_events_block_range\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"server\", field=\"clear_entities_cache_interval\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"server\", field=\"max_simultaneous_requests\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"server\", field=\"max_request_queue_limit\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "F" - } - ], - "title": "Watcher config (server)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "is_subgraph_watcher" - } - ], - "fields": {} - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 17 - }, - "id": 19, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"jobqueue\", field=\"num_events_in_batch\"}", - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"jobqueue\", field=\"block_delay_seconds\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"jobqueue\", field=\"use_block_ranges\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"censures\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - } - ], - "title": "Watcher config (job queue)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "use_block_ranges" - } - ], - "fields": {} - } - } - ], - "type": "stat" - } - ], - "refresh": "10s", - "schemaVersion": 38, - "tags": [ - "ethereum", - "azimuth" - ], - "templating": { - "list": [] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "censures watcher", - "version": 3, - "weekStart": "" -} diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json deleted file mode 100644 index 9c5c5510..00000000 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json +++ /dev/null @@ -1,1772 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"claims\", kind=\"latest_indexed\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Latest indexed block", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Chain head block number of the upstream endpoint", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 0 - }, - "id": 11, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"claims\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Upstream chain head", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Chain head from an external public endpoint", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 0 - }, - "id": 12, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "External chain head", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "semi-dark-green", - "mode": "thresholds" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - }, - { - "color": "red", - "value": 35 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "last_block_process_duration_seconds{job=\"azimuth\", instance=\"claims\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Last block process duration (s)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "orange", - "index": 2, - "text": "LAZY" - }, - "1": { - "color": "semi-dark-green", - "index": 1, - "text": "ACTIVE" - } - }, - "type": "value" - }, - { - "options": { - "match": "null+nan", - "result": { - "color": "semi-dark-red", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2.5, - "w": 4, - "x": 0, - "y": 3 - }, - "id": 20, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"server\", field=\"is_active\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Watcher kind", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 8 - }, - { - "color": "red", - "value": 16 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 4, - "y": 3 - }, - "id": 22, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "valueSize": 25 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"claims\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"claims\", kind=\"latest_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "B" - } - ], - "title": "diff_upstream", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "", - "binary": { - "left": "latest_upstream_block_number", - "operator": "-", - "right": "sync_status_block_number" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 8 - }, - { - "color": "red", - "value": 16 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 8, - "y": 3 - }, - "id": 24, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "valueSize": 25 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"claims\", kind=\"latest_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "B" - } - ], - "title": "diff_external", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "", - "binary": { - "left": "latest_block_number", - "operator": "-", - "right": "sync_status_block_number" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Watcher sync status block numbers", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 4, - "y": 5 - }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "last_processed_block_number{job=\"azimuth\", instance=\"claims\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "latest_processed", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"claims\", kind=\"latest_canonical\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{kind}}", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"claims\", kind=\"intial_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{kind}}", - "range": true, - "refId": "C" - } - ], - "title": "Sync status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "semi-dark-green", - "index": 1, - "text": "REAL TIME" - }, - "1": { - "color": "orange", - "index": 2, - "text": "HISTORICAL" - } - }, - "type": "value" - }, - { - "options": { - "match": "null+nan", - "result": { - "color": "semi-dark-red", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2.5, - "w": 4, - "x": 0, - "y": 5.5 - }, - "id": 16, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "is_syncing_historical_blocks{job=\"azimuth\", instance=\"claims\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Watcher sync mode", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "orange", - "mode": "shades" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 6 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "last_block_num_events_total{job=\"azimuth\", instance=\"claims\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Number of events in the last block", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of entries in the block_progress and event table", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "dark-yellow", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "orange", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 8 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "block_progress_total{job=\"azimuth\", instance=\"claims\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "block_progress", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "event_total{job=\"azimuth\", instance=\"claims\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "event", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "DB indexed count ", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "purple", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 6, - "y": 8 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "database_size_bytes{job=\"azimuth\", instance=\"claims\", type=\"watcher\"} / 1024 / 1024", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{type}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "database_size_bytes{job=\"azimuth\", instance=\"claims\", type=\"job-queue\"} / 1024 / 1024", - "hide": false, - "instant": false, - "legendFormat": "{{type}}", - "range": true, - "refId": "B" - } - ], - "title": "DB size (MB)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of pgboss jobs in the db", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 11 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"claims\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num block-processing jobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of pgboss jobs in the db", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 11 - }, - "id": 15, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"claims\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num event-processing jobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 17 - }, - "id": 18, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"upstream\", field=\"is_using_rpc_client\"}", - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"upstream\", field=\"is_fevm\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"server\", field=\"rpc_supports_block_hash\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"upstream\", field=\"filter_logs_by_topics\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - } - ], - "title": "Watcher config (upstream eth server)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "watcher_config_info" - } - ], - "fields": {} - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 17 - }, - "id": 17, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "value_and_name", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"server\", field=\"is_subgraph_watcher\"}", - "format": "time_series", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"server\", field=\"max_events_block_range\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"server\", field=\"clear_entities_cache_interval\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"server\", field=\"max_simultaneous_requests\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"server\", field=\"max_request_queue_limit\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "F" - } - ], - "title": "Watcher config (server)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "is_subgraph_watcher" - } - ], - "fields": {} - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 17 - }, - "id": 19, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"jobqueue\", field=\"num_events_in_batch\"}", - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"jobqueue\", field=\"block_delay_seconds\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"jobqueue\", field=\"use_block_ranges\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"claims\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - } - ], - "title": "Watcher config (job queue)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "use_block_ranges" - } - ], - "fields": {} - } - } - ], - "type": "stat" - } - ], - "refresh": "10s", - "schemaVersion": 38, - "tags": [ - "ethereum", - "azimuth" - ], - "templating": { - "list": [] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "claims watcher", - "version": 3, - "weekStart": "" -} diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json deleted file mode 100644 index 6f739555..00000000 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json +++ /dev/null @@ -1,1772 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"conditional_star_release\", kind=\"latest_indexed\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Latest indexed block", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Chain head block number of the upstream endpoint", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 0 - }, - "id": 11, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"conditional_star_release\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Upstream chain head", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Chain head from an external public endpoint", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 0 - }, - "id": 12, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "External chain head", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "semi-dark-green", - "mode": "thresholds" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - }, - { - "color": "red", - "value": 35 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "last_block_process_duration_seconds{job=\"azimuth\", instance=\"conditional_star_release\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Last block process duration (s)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "orange", - "index": 2, - "text": "LAZY" - }, - "1": { - "color": "semi-dark-green", - "index": 1, - "text": "ACTIVE" - } - }, - "type": "value" - }, - { - "options": { - "match": "null+nan", - "result": { - "color": "semi-dark-red", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2.5, - "w": 4, - "x": 0, - "y": 3 - }, - "id": 20, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"server\", field=\"is_active\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Watcher kind", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 8 - }, - { - "color": "red", - "value": 16 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 4, - "y": 3 - }, - "id": 22, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "valueSize": 25 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"conditional_star_release\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"conditional_star_release\", kind=\"latest_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "B" - } - ], - "title": "diff_upstream", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "", - "binary": { - "left": "latest_upstream_block_number", - "operator": "-", - "right": "sync_status_block_number" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 8 - }, - { - "color": "red", - "value": 16 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 8, - "y": 3 - }, - "id": 24, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "valueSize": 25 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"conditional_star_release\", kind=\"latest_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "B" - } - ], - "title": "diff_external", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "", - "binary": { - "left": "latest_block_number", - "operator": "-", - "right": "sync_status_block_number" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Watcher sync status block numbers", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 4, - "y": 5 - }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "last_processed_block_number{job=\"azimuth\", instance=\"conditional_star_release\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "latest_processed", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"conditional_star_release\", kind=\"latest_canonical\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{kind}}", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"conditional_star_release\", kind=\"intial_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{kind}}", - "range": true, - "refId": "C" - } - ], - "title": "Sync status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "semi-dark-green", - "index": 1, - "text": "REAL TIME" - }, - "1": { - "color": "orange", - "index": 2, - "text": "HISTORICAL" - } - }, - "type": "value" - }, - { - "options": { - "match": "null+nan", - "result": { - "color": "semi-dark-red", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2.5, - "w": 4, - "x": 0, - "y": 5.5 - }, - "id": 16, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "is_syncing_historical_blocks{job=\"azimuth\", instance=\"conditional_star_release\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Watcher sync mode", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "orange", - "mode": "shades" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 6 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "last_block_num_events_total{job=\"azimuth\", instance=\"conditional_star_release\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Number of events in the last block", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of entries in the block_progress and event table", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "dark-yellow", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "orange", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 8 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "block_progress_total{job=\"azimuth\", instance=\"conditional_star_release\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "block_progress", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "event_total{job=\"azimuth\", instance=\"conditional_star_release\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "event", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "DB indexed count ", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "purple", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 6, - "y": 8 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "database_size_bytes{job=\"azimuth\", instance=\"conditional_star_release\", type=\"watcher\"} / 1024 / 1024", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{type}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "database_size_bytes{job=\"azimuth\", instance=\"conditional_star_release\", type=\"job-queue\"} / 1024 / 1024", - "hide": false, - "instant": false, - "legendFormat": "{{type}}", - "range": true, - "refId": "B" - } - ], - "title": "DB size (MB)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of pgboss jobs in the db", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 11 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"conditional_star_release\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num block-processing jobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of pgboss jobs in the db", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 11 - }, - "id": 15, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"conditional_star_release\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num event-processing jobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 17 - }, - "id": 18, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"upstream\", field=\"is_using_rpc_client\"}", - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"upstream\", field=\"is_fevm\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"server\", field=\"rpc_supports_block_hash\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"upstream\", field=\"filter_logs_by_topics\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - } - ], - "title": "Watcher config (upstream eth server)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "watcher_config_info" - } - ], - "fields": {} - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 17 - }, - "id": 17, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "value_and_name", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"server\", field=\"is_subgraph_watcher\"}", - "format": "time_series", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"server\", field=\"max_events_block_range\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"server\", field=\"clear_entities_cache_interval\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"server\", field=\"max_simultaneous_requests\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"server\", field=\"max_request_queue_limit\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "F" - } - ], - "title": "Watcher config (server)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "is_subgraph_watcher" - } - ], - "fields": {} - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 17 - }, - "id": 19, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"jobqueue\", field=\"num_events_in_batch\"}", - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"jobqueue\", field=\"block_delay_seconds\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"jobqueue\", field=\"use_block_ranges\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"conditional_star_release\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - } - ], - "title": "Watcher config (job queue)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "use_block_ranges" - } - ], - "fields": {} - } - } - ], - "type": "stat" - } - ], - "refresh": "10s", - "schemaVersion": 38, - "tags": [ - "ethereum", - "azimuth" - ], - "templating": { - "list": [] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "conditional star release watcher", - "version": 3, - "weekStart": "" -} diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json deleted file mode 100644 index 9c169174..00000000 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json +++ /dev/null @@ -1,1772 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"delegated_sending_watcher\", kind=\"latest_indexed\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Latest indexed block", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Chain head block number of the upstream endpoint", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 0 - }, - "id": 11, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"delegated_sending_watcher\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Upstream chain head", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Chain head from an external public endpoint", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 0 - }, - "id": 12, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "External chain head", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "semi-dark-green", - "mode": "thresholds" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - }, - { - "color": "red", - "value": 35 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "last_block_process_duration_seconds{job=\"azimuth\", instance=\"delegated_sending_watcher\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Last block process duration (s)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "orange", - "index": 2, - "text": "LAZY" - }, - "1": { - "color": "semi-dark-green", - "index": 1, - "text": "ACTIVE" - } - }, - "type": "value" - }, - { - "options": { - "match": "null+nan", - "result": { - "color": "semi-dark-red", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2.5, - "w": 4, - "x": 0, - "y": 3 - }, - "id": 20, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"server\", field=\"is_active\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Watcher kind", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 8 - }, - { - "color": "red", - "value": 16 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 4, - "y": 3 - }, - "id": 22, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "valueSize": 25 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"delegated_sending_watcher\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"delegated_sending_watcher\", kind=\"latest_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "B" - } - ], - "title": "diff_upstream", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "", - "binary": { - "left": "latest_upstream_block_number", - "operator": "-", - "right": "sync_status_block_number" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 8 - }, - { - "color": "red", - "value": 16 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 8, - "y": 3 - }, - "id": 24, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "valueSize": 25 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"delegated_sending_watcher\", kind=\"latest_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "B" - } - ], - "title": "diff_external", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "", - "binary": { - "left": "latest_block_number", - "operator": "-", - "right": "sync_status_block_number" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Watcher sync status block numbers", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 4, - "y": 5 - }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "last_processed_block_number{job=\"azimuth\", instance=\"delegated_sending_watcher\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "latest_processed", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"delegated_sending_watcher\", kind=\"latest_canonical\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{kind}}", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"delegated_sending_watcher\", kind=\"intial_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{kind}}", - "range": true, - "refId": "C" - } - ], - "title": "Sync status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "semi-dark-green", - "index": 1, - "text": "REAL TIME" - }, - "1": { - "color": "orange", - "index": 2, - "text": "HISTORICAL" - } - }, - "type": "value" - }, - { - "options": { - "match": "null+nan", - "result": { - "color": "semi-dark-red", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2.5, - "w": 4, - "x": 0, - "y": 5.5 - }, - "id": 16, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "is_syncing_historical_blocks{job=\"azimuth\", instance=\"delegated_sending_watcher\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Watcher sync mode", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "orange", - "mode": "shades" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 6 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "last_block_num_events_total{job=\"azimuth\", instance=\"delegated_sending_watcher\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Number of events in the last block", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of entries in the block_progress and event table", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "dark-yellow", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "orange", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 8 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "block_progress_total{job=\"azimuth\", instance=\"delegated_sending_watcher\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "block_progress", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "event_total{job=\"azimuth\", instance=\"delegated_sending_watcher\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "event", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "DB indexed count ", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "purple", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 6, - "y": 8 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "database_size_bytes{job=\"azimuth\", instance=\"delegated_sending_watcher\", type=\"watcher\"} / 1024 / 1024", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{type}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "database_size_bytes{job=\"azimuth\", instance=\"delegated_sending_watcher\", type=\"job-queue\"} / 1024 / 1024", - "hide": false, - "instant": false, - "legendFormat": "{{type}}", - "range": true, - "refId": "B" - } - ], - "title": "DB size (MB)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of pgboss jobs in the db", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 11 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"delegated_sending_watcher\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num block-processing jobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of pgboss jobs in the db", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 11 - }, - "id": 15, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"delegated_sending_watcher\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num event-processing jobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 17 - }, - "id": 18, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"upstream\", field=\"is_using_rpc_client\"}", - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"upstream\", field=\"is_fevm\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"server\", field=\"rpc_supports_block_hash\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"upstream\", field=\"filter_logs_by_topics\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - } - ], - "title": "Watcher config (upstream eth server)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "watcher_config_info" - } - ], - "fields": {} - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 17 - }, - "id": 17, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "value_and_name", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"server\", field=\"is_subgraph_watcher\"}", - "format": "time_series", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"server\", field=\"max_events_block_range\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"server\", field=\"clear_entities_cache_interval\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"server\", field=\"max_simultaneous_requests\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"server\", field=\"max_request_queue_limit\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "F" - } - ], - "title": "Watcher config (server)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "is_subgraph_watcher" - } - ], - "fields": {} - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 17 - }, - "id": 19, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"jobqueue\", field=\"num_events_in_batch\"}", - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"jobqueue\", field=\"block_delay_seconds\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"jobqueue\", field=\"use_block_ranges\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"delegated_sending_watcher\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - } - ], - "title": "Watcher config (job queue)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "use_block_ranges" - } - ], - "fields": {} - } - } - ], - "type": "stat" - } - ], - "refresh": "10s", - "schemaVersion": 38, - "tags": [ - "ethereum", - "azimuth" - ], - "templating": { - "list": [] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "delegated sending watcher watcher", - "version": 3, - "weekStart": "" -} diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json deleted file mode 100644 index f46408f0..00000000 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json +++ /dev/null @@ -1,1772 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"ecliptic\", kind=\"latest_indexed\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Latest indexed block", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Chain head block number of the upstream endpoint", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 0 - }, - "id": 11, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"ecliptic\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Upstream chain head", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Chain head from an external public endpoint", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 0 - }, - "id": 12, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "External chain head", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "semi-dark-green", - "mode": "thresholds" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - }, - { - "color": "red", - "value": 35 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "last_block_process_duration_seconds{job=\"azimuth\", instance=\"ecliptic\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Last block process duration (s)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "orange", - "index": 2, - "text": "LAZY" - }, - "1": { - "color": "semi-dark-green", - "index": 1, - "text": "ACTIVE" - } - }, - "type": "value" - }, - { - "options": { - "match": "null+nan", - "result": { - "color": "semi-dark-red", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2.5, - "w": 4, - "x": 0, - "y": 3 - }, - "id": 20, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"server\", field=\"is_active\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Watcher kind", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 8 - }, - { - "color": "red", - "value": 16 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 4, - "y": 3 - }, - "id": 22, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "valueSize": 25 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"ecliptic\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"ecliptic\", kind=\"latest_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "B" - } - ], - "title": "diff_upstream", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "", - "binary": { - "left": "latest_upstream_block_number", - "operator": "-", - "right": "sync_status_block_number" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 8 - }, - { - "color": "red", - "value": 16 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 8, - "y": 3 - }, - "id": 24, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "valueSize": 25 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"ecliptic\", kind=\"latest_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "B" - } - ], - "title": "diff_external", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "", - "binary": { - "left": "latest_block_number", - "operator": "-", - "right": "sync_status_block_number" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Watcher sync status block numbers", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 4, - "y": 5 - }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "last_processed_block_number{job=\"azimuth\", instance=\"ecliptic\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "latest_processed", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"ecliptic\", kind=\"latest_canonical\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{kind}}", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"ecliptic\", kind=\"intial_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{kind}}", - "range": true, - "refId": "C" - } - ], - "title": "Sync status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "semi-dark-green", - "index": 1, - "text": "REAL TIME" - }, - "1": { - "color": "orange", - "index": 2, - "text": "HISTORICAL" - } - }, - "type": "value" - }, - { - "options": { - "match": "null+nan", - "result": { - "color": "semi-dark-red", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2.5, - "w": 4, - "x": 0, - "y": 5.5 - }, - "id": 16, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "is_syncing_historical_blocks{job=\"azimuth\", instance=\"ecliptic\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Watcher sync mode", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "orange", - "mode": "shades" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 6 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "last_block_num_events_total{job=\"azimuth\", instance=\"ecliptic\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Number of events in the last block", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of entries in the block_progress and event table", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "dark-yellow", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "orange", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 8 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "block_progress_total{job=\"azimuth\", instance=\"ecliptic\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "block_progress", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "event_total{job=\"azimuth\", instance=\"ecliptic\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "event", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "DB indexed count ", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "purple", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 6, - "y": 8 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "database_size_bytes{job=\"azimuth\", instance=\"ecliptic\", type=\"watcher\"} / 1024 / 1024", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{type}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "database_size_bytes{job=\"azimuth\", instance=\"ecliptic\", type=\"job-queue\"} / 1024 / 1024", - "hide": false, - "instant": false, - "legendFormat": "{{type}}", - "range": true, - "refId": "B" - } - ], - "title": "DB size (MB)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of pgboss jobs in the db", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 11 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"ecliptic\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num block-processing jobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of pgboss jobs in the db", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 11 - }, - "id": 15, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"ecliptic\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num event-processing jobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 17 - }, - "id": 18, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"upstream\", field=\"is_using_rpc_client\"}", - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"upstream\", field=\"is_fevm\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"server\", field=\"rpc_supports_block_hash\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"upstream\", field=\"filter_logs_by_topics\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - } - ], - "title": "Watcher config (upstream eth server)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "watcher_config_info" - } - ], - "fields": {} - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 17 - }, - "id": 17, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "value_and_name", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"server\", field=\"is_subgraph_watcher\"}", - "format": "time_series", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"server\", field=\"max_events_block_range\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"server\", field=\"clear_entities_cache_interval\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"server\", field=\"max_simultaneous_requests\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"server\", field=\"max_request_queue_limit\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "F" - } - ], - "title": "Watcher config (server)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "is_subgraph_watcher" - } - ], - "fields": {} - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 17 - }, - "id": 19, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"jobqueue\", field=\"num_events_in_batch\"}", - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"jobqueue\", field=\"block_delay_seconds\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"jobqueue\", field=\"use_block_ranges\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"ecliptic\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - } - ], - "title": "Watcher config (job queue)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "use_block_ranges" - } - ], - "fields": {} - } - } - ], - "type": "stat" - } - ], - "refresh": "10s", - "schemaVersion": 38, - "tags": [ - "ethereum", - "azimuth" - ], - "templating": { - "list": [] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "ecliptic watcher", - "version": 3, - "weekStart": "" -} diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json deleted file mode 100644 index e2187a43..00000000 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json +++ /dev/null @@ -1,1772 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"linear_star_release\", kind=\"latest_indexed\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Latest indexed block", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Chain head block number of the upstream endpoint", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 0 - }, - "id": 11, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"linear_star_release\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Upstream chain head", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Chain head from an external public endpoint", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 0 - }, - "id": 12, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "External chain head", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "semi-dark-green", - "mode": "thresholds" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - }, - { - "color": "red", - "value": 35 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "last_block_process_duration_seconds{job=\"azimuth\", instance=\"linear_star_release\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Last block process duration (s)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "orange", - "index": 2, - "text": "LAZY" - }, - "1": { - "color": "semi-dark-green", - "index": 1, - "text": "ACTIVE" - } - }, - "type": "value" - }, - { - "options": { - "match": "null+nan", - "result": { - "color": "semi-dark-red", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2.5, - "w": 4, - "x": 0, - "y": 3 - }, - "id": 20, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"server\", field=\"is_active\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Watcher kind", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 8 - }, - { - "color": "red", - "value": 16 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 4, - "y": 3 - }, - "id": 22, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "valueSize": 25 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"linear_star_release\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"linear_star_release\", kind=\"latest_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "B" - } - ], - "title": "diff_upstream", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "", - "binary": { - "left": "latest_upstream_block_number", - "operator": "-", - "right": "sync_status_block_number" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 8 - }, - { - "color": "red", - "value": 16 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 8, - "y": 3 - }, - "id": 24, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "valueSize": 25 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"linear_star_release\", kind=\"latest_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "B" - } - ], - "title": "diff_external", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "", - "binary": { - "left": "latest_block_number", - "operator": "-", - "right": "sync_status_block_number" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Watcher sync status block numbers", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 4, - "y": 5 - }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "last_processed_block_number{job=\"azimuth\", instance=\"linear_star_release\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "latest_processed", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"linear_star_release\", kind=\"latest_canonical\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{kind}}", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"linear_star_release\", kind=\"intial_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{kind}}", - "range": true, - "refId": "C" - } - ], - "title": "Sync status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "semi-dark-green", - "index": 1, - "text": "REAL TIME" - }, - "1": { - "color": "orange", - "index": 2, - "text": "HISTORICAL" - } - }, - "type": "value" - }, - { - "options": { - "match": "null+nan", - "result": { - "color": "semi-dark-red", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2.5, - "w": 4, - "x": 0, - "y": 5.5 - }, - "id": 16, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "is_syncing_historical_blocks{job=\"azimuth\", instance=\"linear_star_release\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Watcher sync mode", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "orange", - "mode": "shades" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 6 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "last_block_num_events_total{job=\"azimuth\", instance=\"linear_star_release\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Number of events in the last block", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of entries in the block_progress and event table", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "dark-yellow", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "orange", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 8 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "block_progress_total{job=\"azimuth\", instance=\"linear_star_release\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "block_progress", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "event_total{job=\"azimuth\", instance=\"linear_star_release\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "event", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "DB indexed count ", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "purple", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 6, - "y": 8 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "database_size_bytes{job=\"azimuth\", instance=\"linear_star_release\", type=\"watcher\"} / 1024 / 1024", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{type}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "database_size_bytes{job=\"azimuth\", instance=\"linear_star_release\", type=\"job-queue\"} / 1024 / 1024", - "hide": false, - "instant": false, - "legendFormat": "{{type}}", - "range": true, - "refId": "B" - } - ], - "title": "DB size (MB)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of pgboss jobs in the db", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 11 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"linear_star_release\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num block-processing jobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of pgboss jobs in the db", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 11 - }, - "id": 15, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"linear_star_release\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num event-processing jobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 17 - }, - "id": 18, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"upstream\", field=\"is_using_rpc_client\"}", - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"upstream\", field=\"is_fevm\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"server\", field=\"rpc_supports_block_hash\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"upstream\", field=\"filter_logs_by_topics\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - } - ], - "title": "Watcher config (upstream eth server)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "watcher_config_info" - } - ], - "fields": {} - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 17 - }, - "id": 17, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "value_and_name", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"server\", field=\"is_subgraph_watcher\"}", - "format": "time_series", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"server\", field=\"max_events_block_range\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"server\", field=\"clear_entities_cache_interval\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"server\", field=\"max_simultaneous_requests\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"server\", field=\"max_request_queue_limit\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "F" - } - ], - "title": "Watcher config (server)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "is_subgraph_watcher" - } - ], - "fields": {} - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 17 - }, - "id": 19, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"jobqueue\", field=\"num_events_in_batch\"}", - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"jobqueue\", field=\"block_delay_seconds\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"jobqueue\", field=\"use_block_ranges\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"linear_star_release\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - } - ], - "title": "Watcher config (job queue)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "use_block_ranges" - } - ], - "fields": {} - } - } - ], - "type": "stat" - } - ], - "refresh": "10s", - "schemaVersion": 38, - "tags": [ - "ethereum", - "azimuth" - ], - "templating": { - "list": [] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "linear star release watcher", - "version": 3, - "weekStart": "" -} diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json deleted file mode 100644 index 5e653ee3..00000000 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json +++ /dev/null @@ -1,1772 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"polls\", kind=\"latest_indexed\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Latest indexed block", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Chain head block number of the upstream endpoint", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 0 - }, - "id": 11, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"polls\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Upstream chain head", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Chain head from an external public endpoint", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 0 - }, - "id": 12, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "External chain head", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "semi-dark-green", - "mode": "thresholds" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - }, - { - "color": "red", - "value": 35 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "last_block_process_duration_seconds{job=\"azimuth\", instance=\"polls\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Last block process duration (s)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "orange", - "index": 2, - "text": "LAZY" - }, - "1": { - "color": "semi-dark-green", - "index": 1, - "text": "ACTIVE" - } - }, - "type": "value" - }, - { - "options": { - "match": "null+nan", - "result": { - "color": "semi-dark-red", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2.5, - "w": 4, - "x": 0, - "y": 3 - }, - "id": 20, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"server\", field=\"is_active\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Watcher kind", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 8 - }, - { - "color": "red", - "value": 16 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 4, - "y": 3 - }, - "id": 22, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "valueSize": 25 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "latest_upstream_block_number{job=\"azimuth\", instance=\"polls\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"polls\", kind=\"latest_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "B" - } - ], - "title": "diff_upstream", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "", - "binary": { - "left": "latest_upstream_block_number", - "operator": "-", - "right": "sync_status_block_number" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 8 - }, - { - "color": "red", - "value": 16 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 8, - "y": 3 - }, - "id": 24, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "valueSize": 25 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_block_number{job=\"chain_heads\", chain=\"ethereum\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"polls\", kind=\"latest_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "B" - } - ], - "title": "diff_external", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "", - "binary": { - "left": "latest_block_number", - "operator": "-", - "right": "sync_status_block_number" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Watcher sync status block numbers", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 4, - "y": 5 - }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "last_processed_block_number{job=\"azimuth\", instance=\"polls\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "latest_processed", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"polls\", kind=\"latest_canonical\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{kind}}", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"azimuth\", instance=\"polls\", kind=\"intial_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{kind}}", - "range": true, - "refId": "C" - } - ], - "title": "Sync status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "semi-dark-green", - "index": 1, - "text": "REAL TIME" - }, - "1": { - "color": "orange", - "index": 2, - "text": "HISTORICAL" - } - }, - "type": "value" - }, - { - "options": { - "match": "null+nan", - "result": { - "color": "semi-dark-red", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2.5, - "w": 4, - "x": 0, - "y": 5.5 - }, - "id": 16, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "is_syncing_historical_blocks{job=\"azimuth\", instance=\"polls\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Watcher sync mode", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "orange", - "mode": "shades" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 6 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "last_block_num_events_total{job=\"azimuth\", instance=\"polls\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Number of events in the last block", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of entries in the block_progress and event table", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "dark-yellow", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "orange", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 8 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "block_progress_total{job=\"azimuth\", instance=\"polls\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "block_progress", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "event_total{job=\"azimuth\", instance=\"polls\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "event", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "DB indexed count ", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "purple", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 6, - "y": 8 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "database_size_bytes{job=\"azimuth\", instance=\"polls\", type=\"watcher\"} / 1024 / 1024", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{type}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "database_size_bytes{job=\"azimuth\", instance=\"polls\", type=\"job-queue\"} / 1024 / 1024", - "hide": false, - "instant": false, - "legendFormat": "{{type}}", - "range": true, - "refId": "B" - } - ], - "title": "DB size (MB)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of pgboss jobs in the db", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 11 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"polls\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num block-processing jobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of pgboss jobs in the db", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 11 - }, - "id": 15, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"polls\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num event-processing jobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 17 - }, - "id": 18, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"upstream\", field=\"is_using_rpc_client\"}", - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"upstream\", field=\"is_fevm\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"server\", field=\"rpc_supports_block_hash\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"upstream\", field=\"filter_logs_by_topics\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - } - ], - "title": "Watcher config (upstream eth server)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "watcher_config_info" - } - ], - "fields": {} - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 17 - }, - "id": 17, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "value_and_name", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"server\", field=\"is_subgraph_watcher\"}", - "format": "time_series", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"server\", field=\"max_events_block_range\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"server\", field=\"clear_entities_cache_interval\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"server\", field=\"max_simultaneous_requests\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"server\", field=\"max_request_queue_limit\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "F" - } - ], - "title": "Watcher config (server)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "is_subgraph_watcher" - } - ], - "fields": {} - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 17 - }, - "id": 19, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"jobqueue\", field=\"num_events_in_batch\"}", - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"jobqueue\", field=\"block_delay_seconds\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"jobqueue\", field=\"use_block_ranges\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"azimuth\", instance=\"polls\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - } - ], - "title": "Watcher config (job queue)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "use_block_ranges" - } - ], - "fields": {} - } - } - ], - "type": "stat" - } - ], - "refresh": "10s", - "schemaVersion": 38, - "tags": [ - "ethereum", - "azimuth" - ], - "templating": { - "list": [] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "polls watcher", - "version": 3, - "weekStart": "" -} diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json deleted file mode 100644 index 8e495464..00000000 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/merkl-sushiswap-watcher-dashboard.json +++ /dev/null @@ -1,2061 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"sushi\", instance=\"merkl_sushiswap\", kind=\"latest_indexed\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Latest indexed block", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Chain head block number of the upstream endpoint", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 0 - }, - "id": 11, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_upstream_block_number{job=\"sushi\", instance=\"merkl_sushiswap\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Upstream chain head", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Chain head from an external public endpoint", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 0 - }, - "id": 12, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_block_number{job=\"chain_heads\", chain=\"filecoin\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "External chain head", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "semi-dark-green", - "mode": "thresholds" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - }, - { - "color": "red", - "value": 35 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "last_block_process_duration_seconds{job=\"sushi\", instance=\"merkl_sushiswap\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Last block process duration (s)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "orange", - "index": 2, - "text": "LAZY" - }, - "1": { - "color": "semi-dark-green", - "index": 1, - "text": "ACTIVE" - } - }, - "type": "value" - }, - { - "options": { - "match": "null+nan", - "result": { - "color": "semi-dark-red", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2.5, - "w": 4, - "x": 0, - "y": 3 - }, - "id": 20, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"server\", field=\"is_active\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Watcher kind", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 8 - }, - { - "color": "red", - "value": 16 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 4, - "y": 3 - }, - "id": 22, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "valueSize": 25 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "latest_upstream_block_number{job=\"sushi\", instance=\"merkl_sushiswap\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sync_status_block_number{job=\"sushi\", instance=\"merkl_sushiswap\", kind=\"latest_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "B" - } - ], - "title": "diff_upstream", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "", - "binary": { - "left": "latest_upstream_block_number", - "operator": "-", - "right": "sync_status_block_number" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 8 - }, - { - "color": "red", - "value": 16 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 8, - "y": 3 - }, - "id": 24, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "valueSize": 25 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_block_number{job=\"chain_heads\", chain=\"filecoin\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sync_status_block_number{job=\"sushi\", instance=\"merkl_sushiswap\", kind=\"latest_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "B" - } - ], - "title": "diff_external", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "", - "binary": { - "left": "latest_block_number", - "operator": "-", - "right": "sync_status_block_number" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Watcher sync status block numbers", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 4, - "y": 5 - }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "last_processed_block_number{job=\"sushi\", instance=\"merkl_sushiswap\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "latest_processed", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"sushi\", instance=\"merkl_sushiswap\", kind=\"latest_canonical\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{kind}}", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"sushi\", instance=\"merkl_sushiswap\", kind=\"intial_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{kind}}", - "range": true, - "refId": "C" - } - ], - "title": "Sync status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "semi-dark-green", - "index": 1, - "text": "REAL TIME" - }, - "1": { - "color": "orange", - "index": 2, - "text": "HISTORICAL" - } - }, - "type": "value" - }, - { - "options": { - "match": "null+nan", - "result": { - "color": "semi-dark-red", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2.5, - "w": 4, - "x": 0, - "y": 5.5 - }, - "id": 16, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "is_syncing_historical_blocks{job=\"sushi\", instance=\"merkl_sushiswap\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Watcher sync mode", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "orange", - "mode": "shades" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 6 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "last_block_num_events_total{job=\"sushi\", instance=\"merkl_sushiswap\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Number of events in the last block", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of entries in the block_progress and event table", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "dark-yellow", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "orange", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 8 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "block_progress_total{job=\"sushi\", instance=\"merkl_sushiswap\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "block_progress", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "event_total{job=\"sushi\", instance=\"merkl_sushiswap\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "event", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "DB indexed count ", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "purple", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 6, - "y": 8 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "database_size_bytes{job=\"sushi\", instance=\"merkl_sushiswap\", type=\"watcher\"} / 1024 / 1024", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{type}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "database_size_bytes{job=\"sushi\", instance=\"merkl_sushiswap\", type=\"job-queue\"} / 1024 / 1024", - "hide": false, - "instant": false, - "legendFormat": "{{type}}", - "range": true, - "refId": "B" - } - ], - "title": "DB size (MB)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of pgboss jobs in the db", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 11 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"sushi\", instance=\"merkl_sushiswap\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num block-processing jobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of pgboss jobs in the db", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 11 - }, - "id": 15, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"sushi\", instance=\"merkl_sushiswap\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num event-processing jobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of subgraph entities loaded in event processing", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "purple", - "mode": "shades" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 17 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "event_processing_load_entity_total{job=\"sushi\", instance=\"merkl_sushiswap\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num subgraph entities loaded", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of cache hits for subgraph entites in event processing ", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "purple", - "mode": "shades" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 17 - }, - "id": 9, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "event_processing_load_entity_cache_hit_total{job=\"sushi\", instance=\"merkl_sushiswap\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num subgraph entities cache hits", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of subgraph entities currently present in pruned region of cache", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "purple", - "mode": "shades" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 17 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "cached_pruned_entities_total{job=\"sushi\", instance=\"merkl_sushiswap\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num subgraph entities cached (pruned region)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 23 - }, - "id": 18, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"upstream\", field=\"is_using_rpc_client\"}", - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"upstream\", field=\"is_fevm\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"server\", field=\"rpc_supports_block_hash\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"upstream\", field=\"filter_logs_by_topics\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - } - ], - "title": "Watcher config (upstream eth server)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "watcher_config_info" - } - ], - "fields": {} - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 23 - }, - "id": 17, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "value_and_name", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"server\", field=\"is_subgraph_watcher\"}", - "format": "time_series", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"server\", field=\"max_events_block_range\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"server\", field=\"clear_entities_cache_interval\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"server\", field=\"max_simultaneous_requests\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"server\", field=\"max_request_queue_limit\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "F" - } - ], - "title": "Watcher config (server)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "is_subgraph_watcher" - } - ], - "fields": {} - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 23 - }, - "id": 19, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"jobqueue\", field=\"num_events_in_batch\"}", - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"jobqueue\", field=\"block_delay_seconds\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"jobqueue\", field=\"use_block_ranges\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"merkl_sushiswap\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - } - ], - "title": "Watcher config (job queue)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "use_block_ranges" - } - ], - "fields": {} - } - } - ], - "type": "stat" - } - ], - "refresh": "20s", - "schemaVersion": 38, - "tags": [ - "filecoin", - "sushi" - ], - "templating": { - "list": [] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "merkl sushiswap watcher", - "version": 3, - "weekStart": "" -} diff --git a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json deleted file mode 100644 index b611f04d..00000000 --- a/stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/sushi/sushiswap-watcher-dashboard.json +++ /dev/null @@ -1,2061 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"sushi\", instance=\"sushiswap\", kind=\"latest_indexed\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Latest indexed block", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Chain head block number of the upstream endpoint", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 0 - }, - "id": 11, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_upstream_block_number{job=\"sushi\", instance=\"sushiswap\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Upstream chain head", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Chain head from an external public endpoint", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "shades" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 0 - }, - "id": 12, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_block_number{job=\"chain_heads\", chain=\"filecoin\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "External chain head", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "semi-dark-green", - "mode": "thresholds" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green", - "value": null - }, - { - "color": "red", - "value": 35 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "last_block_process_duration_seconds{job=\"sushi\", instance=\"sushiswap\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Last block process duration (s)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "orange", - "index": 2, - "text": "LAZY" - }, - "1": { - "color": "semi-dark-green", - "index": 1, - "text": "ACTIVE" - } - }, - "type": "value" - }, - { - "options": { - "match": "null+nan", - "result": { - "color": "semi-dark-red", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2.5, - "w": 4, - "x": 0, - "y": 3 - }, - "id": 20, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"server\", field=\"is_active\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Watcher kind", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 8 - }, - { - "color": "red", - "value": 16 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 4, - "y": 3 - }, - "id": 22, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "valueSize": 25 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "latest_upstream_block_number{job=\"sushi\", instance=\"sushiswap\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sync_status_block_number{job=\"sushi\", instance=\"sushiswap\", kind=\"latest_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "B" - } - ], - "title": "diff_upstream", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "", - "binary": { - "left": "latest_upstream_block_number", - "operator": "-", - "right": "sync_status_block_number" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 8 - }, - { - "color": "red", - "value": 16 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 4, - "x": 8, - "y": 3 - }, - "id": 24, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "valueSize": 25 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "latest_block_number{job=\"chain_heads\", chain=\"filecoin\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sync_status_block_number{job=\"sushi\", instance=\"sushiswap\", kind=\"latest_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "B" - } - ], - "title": "diff_external", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "", - "binary": { - "left": "latest_block_number", - "operator": "-", - "right": "sync_status_block_number" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Watcher sync status block numbers", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 4, - "y": 5 - }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "last_processed_block_number{job=\"sushi\", instance=\"sushiswap\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "latest_processed", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"sushi\", instance=\"sushiswap\", kind=\"latest_canonical\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{kind}}", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "expr": "sync_status_block_number{job=\"sushi\", instance=\"sushiswap\", kind=\"intial_indexed\"}", - "hide": false, - "instant": false, - "legendFormat": "{{kind}}", - "range": true, - "refId": "C" - } - ], - "title": "Sync status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "semi-dark-green", - "index": 1, - "text": "REAL TIME" - }, - "1": { - "color": "orange", - "index": 2, - "text": "HISTORICAL" - } - }, - "type": "value" - }, - { - "options": { - "match": "null+nan", - "result": { - "color": "semi-dark-red", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2.5, - "w": 4, - "x": 0, - "y": 5.5 - }, - "id": 16, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "is_syncing_historical_blocks{job=\"sushi\", instance=\"sushiswap\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Watcher sync mode", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "orange", - "mode": "shades" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 6 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "last_block_num_events_total{job=\"sushi\", instance=\"sushiswap\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Number of events in the last block", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of entries in the block_progress and event table", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "dark-yellow", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "orange", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 8 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "block_progress_total{job=\"sushi\", instance=\"sushiswap\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "block_progress", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "event_total{job=\"sushi\", instance=\"sushiswap\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "event", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "DB indexed count ", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "purple", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 6, - "y": 8 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "database_size_bytes{job=\"sushi\", instance=\"sushiswap\", type=\"watcher\"} / 1024 / 1024", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{type}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "database_size_bytes{job=\"sushi\", instance=\"sushiswap\", type=\"job-queue\"} / 1024 / 1024", - "hide": false, - "instant": false, - "legendFormat": "{{type}}", - "range": true, - "refId": "B" - } - ], - "title": "DB size (MB)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of pgboss jobs in the db", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 11 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"sushi\", instance=\"sushiswap\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num block-processing jobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of pgboss jobs in the db", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 11 - }, - "id": 15, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "pgboss_jobs_total{job=\"sushi\", instance=\"sushiswap\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num event-processing jobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of subgraph entities loaded in event processing", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "purple", - "mode": "shades" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 17 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "event_processing_load_entity_total{job=\"sushi\", instance=\"sushiswap\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num subgraph entities loaded", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of cache hits for subgraph entites in event processing ", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "purple", - "mode": "shades" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 17 - }, - "id": 9, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "event_processing_load_entity_cache_hit_total{job=\"sushi\", instance=\"sushiswap\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num subgraph entities cache hits", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Total number of subgraph entities currently present in pruned region of cache", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "purple", - "mode": "shades" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 17 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "cached_pruned_entities_total{job=\"sushi\", instance=\"sushiswap\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Num subgraph entities cached (pruned region)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 23 - }, - "id": 18, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"upstream\", field=\"is_using_rpc_client\"}", - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"upstream\", field=\"is_fevm\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"server\", field=\"rpc_supports_block_hash\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"upstream\", field=\"filter_logs_by_topics\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - } - ], - "title": "Watcher config (upstream eth server)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "watcher_config_info" - } - ], - "fields": {} - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 23 - }, - "id": 17, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "value_and_name", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"server\", field=\"is_subgraph_watcher\"}", - "format": "time_series", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"server\", field=\"max_events_block_range\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"server\", field=\"clear_entities_cache_interval\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"server\", field=\"max_simultaneous_requests\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"server\", field=\"max_request_queue_limit\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "F" - } - ], - "title": "Watcher config (server)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "is_subgraph_watcher" - } - ], - "fields": {} - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "color": "#757575", - "index": 0, - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 23 - }, - "id": 19, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^(?!Time$).*/", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"jobqueue\", field=\"num_events_in_batch\"}", - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"jobqueue\", field=\"block_delay_seconds\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"jobqueue\", field=\"use_block_ranges\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "watcher_config_info{job=\"sushi\", instance=\"sushiswap\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", - "hide": false, - "instant": false, - "legendFormat": "{{field}}", - "range": true, - "refId": "E" - } - ], - "title": "Watcher config (job queue)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "boolean", - "targetField": "use_block_ranges" - } - ], - "fields": {} - } - } - ], - "type": "stat" - } - ], - "refresh": "20s", - "schemaVersion": 38, - "tags": [ - "filecoin", - "sushi" - ], - "templating": { - "list": [] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "sushiswap watcher", - "version": 1, - "weekStart": "" -} diff --git a/stack_orchestrator/data/config/monitoring/postgres-exporter.yml b/stack_orchestrator/data/config/monitoring/postgres-exporter.yml new file mode 100644 index 00000000..72b8ffa1 --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/postgres-exporter.yml @@ -0,0 +1,8 @@ +auth_modules: + foo: + type: userpass + userpass: + username: username + password: password + options: + sslmode: disable diff --git a/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml b/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml index 40068287..b5f03034 100644 --- a/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml +++ b/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml @@ -40,3 +40,21 @@ scrape_configs: static_configs: - targets: ['chain-head-exporter:5000'] + - job_name: 'postgres' + scrape_interval: 30s + scrape_timeout: 30s + static_configs: + # Add DB targets below + # - targets: [example-server:5432] + # labels: + # instance: 'example-label' + metrics_path: /probe + params: + auth_module: [foo] + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: postgres-exporter:9187 diff --git a/stack_orchestrator/data/stacks/monitoring/README.md b/stack_orchestrator/data/stacks/monitoring/README.md index 803da548..feced015 100644 --- a/stack_orchestrator/data/stacks/monitoring/README.md +++ b/stack_orchestrator/data/stacks/monitoring/README.md @@ -72,7 +72,7 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --de - targets: [':'] ``` -* Also update the `blackbox` job to add any endpoints to be monitored on the Blackbox dashboard: +* Blackbox: update the `blackbox` job to add any endpoints to be monitored on the Blackbox dashboard: ```yml ... @@ -85,6 +85,21 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --de - ``` +* Postgres: + * Update the `postgres` job to add Postgres db targets to be monitored: + + ```yml + ... + - job_name: 'postgres' + ... + static_configs: + # Add DB targets below + - targets: [example-server:5432] + labels: + instance: 'example-db' + ``` + * Add database credentials to be used in `auth_modules` in the postgres-exporter config file (`monitoring-deployment/config/monitoring/postgres-exporter.yml`) + Note: Use `host.docker.internal` as host to access ports on the host machine ### Grafana Config diff --git a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md index 4ef43b36..676ca463 100644 --- a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md +++ b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md @@ -35,8 +35,6 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-watchers-spec ## Configure -### Prometheus Config - Add the following scrape configs to prometheus config file (`monitoring-watchers-deployment/config/monitoring/prometheus/prometheus.yml`) in the deployment folder: ```yml @@ -55,27 +53,35 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers - targets: ['AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_PORT'] labels: instance: 'azimuth' + chain: 'ethereum' - targets: ['CENSURES_WATCHER_HOST:CENSURES_WATCHER_PORT'] labels: instance: 'censures' + chain: 'ethereum' - targets: ['CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_PORT'] labels: instance: 'claims' + chain: 'ethereum' - targets: ['CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_PORT'] labels: instance: 'conditional_star_release' + chain: 'ethereum' - targets: ['DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_PORT'] labels: - instance: 'delegated_sending_watcher' + instance: 'delegated_sending' + chain: 'ethereum' - targets: ['ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_PORT'] labels: instance: 'ecliptic' + chain: 'ethereum' - targets: ['LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_PORT'] labels: instance: 'linear_star_release' + chain: 'ethereum' - targets: ['POLLS_WATCHER_HOST:POLLS_WATCHER_PORT'] labels: instance: 'polls' + chain: 'ethereum' - job_name: sushi scrape_interval: 20s @@ -85,18 +91,14 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers - targets: ['SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_PORT'] labels: instance: 'sushiswap' + chain: 'filecoin' - targets: ['MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_PORT'] labels: instance: 'merkl_sushiswap' + chain: 'filecoin' ``` -### Grafana Config - -In the deployment folder, copy over the pre-configured watcher dashboard JSON files to grafana dashboards config directory: - -```bash -cp -r monitoring-watchers-deployment/config/monitoring/grafana/watcher-dashboards/* monitoring-watchers-deployment/config/monitoring/grafana/dashboards/ -``` +Add scrape config as done above for any additional watcher to add it to the watcher dashboard. ### Env From a57b0cdd267b9b538f9024e50c28a7a814a41ce6 Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Thu, 21 Dec 2023 15:15:03 +0530 Subject: [PATCH 59/75] Add a stack for prom node exporter and its dashboard in monitoring stack (#696) * Add a stack for Prometheus node exporter * Add node exporter dashboard to monitoring stack --- .../compose/docker-compose-node-exporter.yml | 16 + .../dashboards/node-exporter-full.json | 23806 ++++++++++++++++ .../monitoring/prometheus/prometheus.yml | 7 + .../data/stacks/monitoring/README.md | 26 +- .../stacks/monitoring/monitoring-watchers.md | 4 +- .../data/stacks/node-exporter/README.md | 17 + .../data/stacks/node-exporter/stack.yml | 6 + 7 files changed, 23876 insertions(+), 6 deletions(-) create mode 100644 stack_orchestrator/data/compose/docker-compose-node-exporter.yml create mode 100644 stack_orchestrator/data/config/monitoring/grafana/dashboards/node-exporter-full.json create mode 100644 stack_orchestrator/data/stacks/node-exporter/README.md create mode 100644 stack_orchestrator/data/stacks/node-exporter/stack.yml diff --git a/stack_orchestrator/data/compose/docker-compose-node-exporter.yml b/stack_orchestrator/data/compose/docker-compose-node-exporter.yml new file mode 100644 index 00000000..8b49c269 --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-node-exporter.yml @@ -0,0 +1,16 @@ +version: '3.8' + +services: + node-exporter: + image: prom/node-exporter:latest + restart: unless-stopped + command: + - '--path.rootfs=/host' + - '--collector.systemd' + - '--collector.processes' + network_mode: host + pid: host + ports: + - 9100 + volumes: + - '/:/host:ro,rslave' diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/node-exporter-full.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/node-exporter-full.json new file mode 100644 index 00000000..106ca42c --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/node-exporter-full.json @@ -0,0 +1,23806 @@ +{ + "annotations": { + "list": [ + { + "$$hashKey": "object:1058", + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 1860, + "graphTooltip": 1, + "id": 5, + "links": [ + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "GitHub", + "type": "link", + "url": "https://github.com/rfmoz/grafana-dashboards" + }, + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "Grafana", + "type": "link", + "url": "https://grafana.com/grafana/dashboards/1860" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 261, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Quick CPU / Mem / Disk", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Busy state of all CPU cores together", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 85 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 20, + "links": [], + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))) * 100", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "CPU Busy", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Busy state of all CPU cores together (5 min average)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 85 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 155, + "links": [], + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg_over_time(node_load5{instance=\"$node\",job=\"$job\"}[$__rate_interval]) * 100 / on(instance) group_left sum by (instance)(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "Sys Load (5m avg)", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Busy state of all CPU cores together (15 min average)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 85 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 19, + "links": [], + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg_over_time(node_load15{instance=\"$node\",job=\"$job\"}[$__rate_interval]) * 100 / on(instance) group_left sum by (instance)(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]))", + "hide": false, + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "Sys Load (15m avg)", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Non available RAM memory", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 1 + }, + "hideTimeOverride": false, + "id": 16, + "links": [], + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "((avg_over_time(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - avg_over_time(node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])) / (avg_over_time(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) )) * 100", + "format": "time_series", + "hide": true, + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "100 - ((avg_over_time(node_memory_MemAvailable_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) * 100) / avg_over_time(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "B", + "step": 240 + } + ], + "title": "RAM Used", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Used Swap", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 10 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 25 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 21, + "links": [], + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "((avg_over_time(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - avg_over_time(node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])) / (avg_over_time(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) )) * 100", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "SWAP Used", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Used Root FS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 154, + "links": [], + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "100 - ((avg_over_time(node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}[$__rate_interval]) * 100) / avg_over_time(node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}[$__rate_interval]))", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "Root FS Used", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of CPU cores", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 18, + "y": 1 + }, + "id": 14, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU Cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "System uptime", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 20, + "y": 1 + }, + "hideTimeOverride": true, + "id": 15, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "node_time_seconds{instance=\"$node\",job=\"$job\"} - node_boot_time_seconds{instance=\"$node\",job=\"$job\"}", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total RootFS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 70 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 18, + "y": 3 + }, + "id": 23, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "RootFS Total", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total RAM", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 20, + "y": 3 + }, + "id": 75, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "RAM Total", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total SWAP", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 22, + "y": 3 + }, + "id": 18, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "SWAP Total", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 263, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Basic CPU / Mem / Net / Disk", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Basic CPU info", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Busy Iowait" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy Iowait" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy System" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy User" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy Other" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 77, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "width": 250 + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Busy System", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Busy User", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Busy Iowait", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=~\".*irq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Busy IRQs", + "range": true, + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!='idle',mode!='user',mode!='system',mode!='iowait',mode!='irq',mode!='softirq'}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Busy Other", + "range": true, + "refId": "E", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Idle", + "range": true, + "refId": "F", + "step": 240 + } + ], + "title": "CPU Basic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Basic memory usage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SWAP Used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap Used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM Cache + Buffer" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#DEDAF7", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "normal" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 78, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "RAM Total", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - (node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "RAM Used", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "RAM Cache + Buffer", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "RAM Free", + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SWAP Used", + "refId": "E", + "step": 240 + } + ], + "title": "Memory Basic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Basic network info per interface", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Recv_bytes_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_drop_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_errs_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CCA300", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_bytes_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_drop_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_errs_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CCA300", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_drop_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#967302", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_errs_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_bytes_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_drop_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#967302", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_errs_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 74, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "recv {{device}}", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "trans {{device}} ", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Basic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Disk space used of all filesystems mounted", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 152, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "100 - ((node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} * 100) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}}", + "refId": "A", + "step": 240 + } + ], + "title": "Disk Space Used Basic", + "type": "timeseries" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 265, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "percentage", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 70, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Idle - Waiting for something to happen" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Iowait - Waiting for I/O to complete" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Irq - Servicing interrupts" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Nice - Niced processes executing in user mode" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Softirq - Servicing softirqs" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Steal - Time spent in other operating systems when running in a virtualized environment" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCE2DE", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "System - Processes executing in kernel mode" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "User - Normal processes executing in user mode" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#5195CE", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 3, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 250 + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "System - Processes executing in kernel mode", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "User - Normal processes executing in user mode", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nice - Niced processes executing in user mode", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Iowait - Waiting for I/O to complete", + "range": true, + "refId": "E", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"irq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Irq - Servicing interrupts", + "range": true, + "refId": "F", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"softirq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Softirq - Servicing softirqs", + "range": true, + "refId": "G", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"steal\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Steal - Time spent in other operating systems when running in a virtualized environment", + "range": true, + "refId": "H", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Idle - Waiting for something to happen", + "range": true, + "refId": "J", + "step": 240 + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap - Swap memory usage" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused - Free memory unassigned" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Hardware Corrupted - *./" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "normal" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 24, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"} - node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Apps - Memory used by user-space applications", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "PageTables - Memory used to map between virtual and physical memory addresses", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SwapCache - Memory that keeps track of pages that have been fetched from swap but not yet been modified", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Slab - Memory used by the kernel to cache data structures for its own use (caches like inode, dentry, etc)", + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Cache - Parked file data (file content) cache", + "refId": "E", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Buffers - Block device (e.g. harddisk) cache", + "refId": "F", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Unused - Free memory unassigned", + "refId": "G", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Swap - Swap space used", + "refId": "H", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_HardwareCorrupted_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working", + "refId": "I", + "step": 240 + } + ], + "title": "Memory Stack", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bits out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "receive_packets_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "receive_packets_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "transmit_packets_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "transmit_packets_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 84, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Transmit", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 156, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} - node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}}", + "refId": "A", + "step": 240 + } + ], + "title": "Disk Space Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "IO read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 229, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", + "intervalFactor": 4, + "legendFormat": "{{device}} - Reads completed", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} - Writes completed", + "refId": "B", + "step": 240 + } + ], + "title": "Disk IOps", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "io time" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*read*./" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byType", + "options": "time" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "hidden" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 42, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{device}} - Successfully read bytes", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{device}} - Successfully written bytes", + "refId": "B", + "step": 240 + } + ], + "title": "I/O Usage Read / Write", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "%util", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "io time" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byType", + "options": "time" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "hidden" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 127, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"} [$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}}", + "refId": "A", + "step": 240 + } + ], + "title": "I/O Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "percentage", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 70, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^Guest - /" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#5195ce", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^GuestNice - /" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#c15c17", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 319, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))", + "hide": false, + "legendFormat": "Guest - Time spent running a virtual CPU for a guest operating system", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))", + "hide": false, + "legendFormat": "GuestNice - Time spent running a niced guest (virtual CPU for guest operating system)", + "range": true, + "refId": "B" + } + ], + "title": "CPU spent seconds in guests (VMs)", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "CPU / Memory / Net / Disk", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 266, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 54 + }, + "id": 136, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Inactive_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Inactive - Memory which has been less recently used. It is more eligible to be reclaimed for other purposes", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Active_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Active - Memory that has been used more recently and usually not reclaimed unless absolutely necessary", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Active / Inactive", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*CommitLimit - *./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 54 + }, + "id": 135, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Committed_AS_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Committed_AS - Amount of memory presently allocated on the system", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_CommitLimit_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "CommitLimit - Amount of memory currently available to be allocated on the system", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Committed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 191, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Inactive_file_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Inactive_file - File-backed memory on inactive LRU list", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Inactive_anon_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Inactive_anon - Anonymous and swap cache on inactive LRU list, including tmpfs (shmem)", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Active_file_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Active_file - File-backed memory on active LRU list", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Active_anon_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Active_anon - Anonymous and swap cache on active least-recently-used (LRU) list, including tmpfs", + "refId": "D", + "step": 240 + } + ], + "title": "Memory Active / Inactive Detail", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 64 + }, + "id": 130, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Writeback_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Writeback - Memory which is actively being written back to disk", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_WritebackTmp_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "WritebackTmp - Memory used by FUSE for temporary writeback buffers", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Dirty_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Dirty - Memory which is waiting to get written back to the disk", + "refId": "C", + "step": 240 + } + ], + "title": "Memory Writeback and Dirty", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 74 + }, + "id": 138, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Mapped_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Mapped - Used memory in mapped pages files which have been mapped, such as libraries", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Shmem_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Shmem - Used shared memory (shared between several processes, thus including RAM disks)", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_ShmemHugePages_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_ShmemPmdMapped_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "ShmemPmdMapped - Amount of shared (shmem/tmpfs) memory backed by huge pages", + "refId": "D", + "step": 240 + } + ], + "title": "Memory Shared and Mapped", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 74 + }, + "id": 131, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_SUnreclaim_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SUnreclaim - Part of Slab, that cannot be reclaimed on memory pressure", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SReclaimable - Part of Slab, that might be reclaimed, such as caches", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Slab", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 84 + }, + "id": 70, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_VmallocChunk_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "VmallocChunk - Largest contiguous block of vmalloc area which is free", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_VmallocTotal_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "VmallocTotal - Total size of vmalloc memory area", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_VmallocUsed_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "VmallocUsed - Amount of vmalloc area which is used", + "refId": "C", + "step": 240 + } + ], + "title": "Memory Vmalloc", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 84 + }, + "id": 159, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Bounce_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Bounce - Memory used for block device bounce buffers", + "refId": "A", + "step": 240 + } + ], + "title": "Memory Bounce", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Inactive *./" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 94 + }, + "id": 129, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_AnonHugePages_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "AnonHugePages - Memory in anonymous huge pages", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_AnonPages_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "AnonPages - Memory in user pages not backed by files", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Anonymous", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 94 + }, + "id": 160, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_KernelStack_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "KernelStack - Kernel memory stack. This is not reclaimable", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Percpu_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "PerCPU - Per CPU memory allocated dynamically by loadable modules", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Kernel / CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "pages", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 104 + }, + "id": 140, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_HugePages_Free{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "HugePages_Free - Huge pages in the pool that are not yet allocated", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_HugePages_Rsvd{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "HugePages_Rsvd - Huge pages for which a commitment to allocate from the pool has been made, but no allocation has yet been made", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_HugePages_Surp{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "HugePages_Surp - Huge pages in the pool above the value in /proc/sys/vm/nr_hugepages", + "refId": "C", + "step": 240 + } + ], + "title": "Memory HugePages Counter", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 104 + }, + "id": 71, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_HugePages_Total{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "HugePages - Total size of the pool of huge pages", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Hugepagesize - Huge Page size", + "refId": "B", + "step": 240 + } + ], + "title": "Memory HugePages Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 114 + }, + "id": 128, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_DirectMap1G_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "DirectMap1G - Amount of pages mapped as this size", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_DirectMap2M_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "DirectMap2M - Amount of pages mapped as this size", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_DirectMap4k_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "DirectMap4K - Amount of pages mapped as this size", + "refId": "C", + "step": 240 + } + ], + "title": "Memory DirectMap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 114 + }, + "id": 137, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Unevictable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Unevictable - Amount of unevictable memory that can't be swapped out for a variety of reasons", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Mlocked_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "MLocked - Size of pages locked to memory using the mlock() system call", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Unevictable and MLocked", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 124 + }, + "id": 132, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_NFS_Unstable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "NFS Unstable - Memory in NFS pages sent to the server, but not yet committed to the storage", + "refId": "A", + "step": 240 + } + ], + "title": "Memory NFS", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Memory Meminfo", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 267, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "pages out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 176, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_vmstat_pgpgin{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pagesin - Page in operations", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_vmstat_pgpgout{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pagesout - Page out operations", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Pages In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "pages out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 22, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_vmstat_pswpin{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pswpin - Pages swapped in", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_vmstat_pswpout{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pswpout - Pages swapped out", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Pages Swap In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "faults", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pgfault - Page major and minor fault operations" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "normal" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 175, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pgfault - Page major and minor fault operations", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pgmajfault - Major page fault operations", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pgminfault - Minor page fault operations", + "refId": "C", + "step": 240 + } + ], + "title": "Memory Page Faults", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 307, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_vmstat_oom_kill{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "oom killer invocations ", + "refId": "A", + "step": 240 + } + ], + "title": "OOM Killer", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Memory Vmstat", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 293, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Variation*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 260, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_timex_estimated_error_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Estimated error in seconds", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_timex_offset_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Time offset in between local system and reference clock", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_timex_maxerror_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Maximum error in seconds", + "refId": "C", + "step": 240 + } + ], + "title": "Time Synchronized Drift", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 291, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_timex_loop_time_constant{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Phase-locked loop time adjust", + "refId": "A", + "step": 240 + } + ], + "title": "Time PLL Adjust", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Variation*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 66 + }, + "id": 168, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_timex_sync_status{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Is clock synchronized to a reliable server (1 = yes, 0 = no)", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_timex_frequency_adjustment_ratio{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Local clock frequency adjustment", + "refId": "B", + "step": 240 + } + ], + "title": "Time Synchronized Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 66 + }, + "id": 294, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_timex_tick_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Seconds between clock ticks", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_timex_tai_offset_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "International Atomic Time (TAI) offset", + "refId": "B", + "step": 240 + } + ], + "title": "Time Misc", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "System Timesync", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 312, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 62, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_procs_blocked{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Processes blocked waiting for I/O to complete", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_procs_running{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Processes in runnable state", + "refId": "B", + "step": 240 + } + ], + "title": "Processes Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 315, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_processes_state{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ state }}", + "refId": "A", + "step": 240 + } + ], + "title": "Processes State", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "forks / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 53 + }, + "id": 148, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_forks_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Processes forks second", + "refId": "A", + "step": 240 + } + ], + "title": "Processes Forks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max.*/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 53 + }, + "id": 149, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Processes virtual memory size in bytes", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "process_resident_memory_max_bytes{instance=\"$node\",job=\"$job\"}", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Maximum amount of virtual memory available in bytes", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Processes virtual memory size in bytes", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(process_virtual_memory_max_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Maximum amount of virtual memory available in bytes", + "refId": "D", + "step": 240 + } + ], + "title": "Processes Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "PIDs limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 63 + }, + "id": 313, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_processes_pids{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Number of PIDs", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_processes_max_processes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "PIDs limit", + "refId": "B", + "step": 240 + } + ], + "title": "PIDs Number and Limit", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*waiting.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 63 + }, + "id": 305, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }} - seconds spent running a process", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }} - seconds spent by processing waiting for this CPU", + "refId": "B", + "step": 240 + } + ], + "title": "Process schedule stats Running / Waiting", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Threads limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 73 + }, + "id": 314, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_processes_threads{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Allocated threads", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_processes_max_threads{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Threads limit", + "refId": "B", + "step": 240 + } + ], + "title": "Threads Number and Limit", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "System Processes", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 269, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 8, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_context_switches_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Context switches", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_intr_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Interrupts", + "refId": "B", + "step": 240 + } + ], + "title": "Context Switches / Interrupts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 7, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_load1{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Load 1m", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_load5{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Load 5m", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_load15{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Load 15m", + "refId": "C", + "step": 240 + } + ], + "title": "System Load", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "hertz" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Max" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 10 + }, + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": false, + "viz": false + } + }, + { + "id": "custom.fillBelowTo", + "value": "Min" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Min" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": false, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 321, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "node_cpu_scaling_frequency_hertz{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }}", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "avg(node_cpu_scaling_frequency_max_hertz{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Max", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "avg(node_cpu_scaling_frequency_min_hertz{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Min", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "CPU Frequency Scaling", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "https://docs.kernel.org/accounting/psi.html", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Memory some" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Memory full" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "I/O some" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "I/O full" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 322, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(node_pressure_cpu_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "CPU some", + "range": true, + "refId": "CPU some", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(node_pressure_memory_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Memory some", + "range": true, + "refId": "Memory some", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(node_pressure_memory_stalled_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Memory full", + "range": true, + "refId": "Memory full", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(node_pressure_io_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "I/O some", + "range": true, + "refId": "I/O some", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(node_pressure_io_stalled_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "I/O full", + "range": true, + "refId": "I/O full", + "step": 240 + } + ], + "title": "Pressure Stall Information", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Critical*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 46 + }, + "id": 259, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_interrupts_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ type }} - {{ info }}", + "refId": "A", + "step": 240 + } + ], + "title": "Interrupts Detail", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 46 + }, + "id": 306, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_schedstat_timeslices_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }}", + "refId": "A", + "step": 240 + } + ], + "title": "Schedule timeslices executed by each cpu", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 151, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_entropy_available_bits{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Entropy available to random number generators", + "refId": "A", + "step": 240 + } + ], + "title": "Entropy", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 308, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(process_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Time spent", + "refId": "A", + "step": 240 + } + ], + "title": "CPU time spent in user and system contexts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 66 + }, + "id": 64, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "process_max_fds{instance=\"$node\",job=\"$job\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Maximum open file descriptors", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "process_open_fds{instance=\"$node\",job=\"$job\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Open file descriptors", + "refId": "B", + "step": 240 + } + ], + "title": "File Descriptors", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "System Misc", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 304, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "temperature", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "celsius" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Critical*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 158, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_hwmon_temp_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }} temp", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_hwmon_temp_crit_alarm_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }} Critical Alarm", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_hwmon_temp_crit_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }} Critical", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_hwmon_temp_crit_hyst_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }} Critical Historical", + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_hwmon_temp_max_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }} Max", + "refId": "E", + "step": 240 + } + ], + "title": "Hardware temperature monitor", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 300, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_cooling_device_cur_state{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Current {{ name }} in {{ type }}", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_cooling_device_max_state{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Max {{ name }} in {{ type }}", + "refId": "B", + "step": 240 + } + ], + "title": "Throttle cooling device", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 69 + }, + "id": 302, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_power_supply_online{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ power_supply }} online", + "refId": "A", + "step": 240 + } + ], + "title": "Power supply", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Hardware Misc", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 296, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 46 + }, + "id": 297, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_systemd_socket_accepted_connections_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ name }} Connections", + "refId": "A", + "step": 240 + } + ], + "title": "Systemd Sockets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Failed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF9830", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Deactivating" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFCB7D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Activating" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C8F2C2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 46 + }, + "id": 298, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"activating\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Activating", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"active\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Active", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"deactivating\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Deactivating", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"failed\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Failed", + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"inactive\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Inactive", + "refId": "E", + "step": 240 + } + ], + "title": "Systemd Units State", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Systemd", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 270, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The number (after merges) of I/O requests completed per second for the device", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "IO read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 9, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "intervalFactor": 4, + "legendFormat": "{{device}} - Reads completed", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} - Writes completed", + "refId": "B", + "step": 240 + } + ], + "title": "Disk IOps Completed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The number of bytes read from or written to the device per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 33, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "{{device}} - Read bytes", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Written bytes", + "refId": "B", + "step": 240 + } + ], + "title": "Disk R/W Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The average time for requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "time. read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 57 + }, + "id": 37, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_read_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}} - Read wait time avg", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_write_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Write wait time avg", + "refId": "B", + "step": 240 + } + ], + "title": "Disk Average Wait Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The average queue length of the requests that were issued to the device", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "aqu-sz", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 57 + }, + "id": 35, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_io_time_weighted_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}}", + "refId": "A", + "step": 240 + } + ], + "title": "Average Queue Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The number of read and write requests merged per second that were queued to the device", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "I/Os", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 133, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_reads_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} - Read merged", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_writes_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} - Write merged", + "refId": "B", + "step": 240 + } + ], + "title": "Disk R/W Merged", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of elapsed time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100% for devices serving requests serially. But for devices serving requests in parallel, such as RAID arrays and modern SSDs, this number does not reflect their performance limits.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "%util", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 36, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}} - IO", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_discard_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}} - discard", + "refId": "B", + "step": 240 + } + ], + "title": "Time Spent Doing I/Os", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The number of outstanding requests at the instant the sample was taken. Incremented as requests are given to appropriate struct request_queue and decremented as they finish.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Outstanding req.", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 77 + }, + "id": 34, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_disk_io_now{instance=\"$node\",job=\"$job\"}", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}} - IO now", + "refId": "A", + "step": 240 + } + ], + "title": "Instantaneous Queue Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "IOs", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 77 + }, + "id": 301, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_discards_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}} - Discards completed", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_discards_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Discards merged", + "refId": "B", + "step": 240 + } + ], + "title": "Disk IOps Discards completed / merged", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Storage Disk", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 271, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 62 + }, + "id": 43, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - Available", + "metric": "", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_filesystem_free_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - Free", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - Size", + "refId": "C", + "step": 240 + } + ], + "title": "Filesystem space available", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "file nodes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 62 + }, + "id": 41, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_filesystem_files_free{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - Free file nodes", + "refId": "A", + "step": 240 + } + ], + "title": "File Nodes Free", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "files", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 72 + }, + "id": 28, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_filefd_maximum{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Max open files", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_filefd_allocated{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Open files", + "refId": "B", + "step": 240 + } + ], + "title": "File Descriptor", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "file Nodes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 72 + }, + "id": 219, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_filesystem_files{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - File nodes total", + "refId": "A", + "step": 240 + } + ], + "title": "File Nodes Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "/ ReadOnly" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 82 + }, + "id": 44, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_filesystem_readonly{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - ReadOnly", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_filesystem_device_error{instance=\"$node\",job=\"$job\",device!~'rootfs',fstype!~'tmpfs'}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - Device error", + "refId": "B", + "step": 240 + } + ], + "title": "Filesystem in ReadOnly / Error", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Storage Filesystem", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 272, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "receive_packets_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "receive_packets_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "transmit_packets_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "transmit_packets_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 60, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_receive_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_transmit_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Transmit", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic by Packets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 142, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_receive_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive errors", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_transmit_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Rransmit errors", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 57 + }, + "id": 143, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_receive_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive drop", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_transmit_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Transmit drop", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Drop", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 57 + }, + "id": 141, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_receive_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive compressed", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_transmit_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Transmit compressed", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Compressed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 146, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_receive_multicast_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive multicast", + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic Multicast", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 144, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_receive_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive fifo", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_transmit_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Transmit fifo", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Fifo", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 77 + }, + "id": 145, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_receive_frame_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive frame", + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic Frame", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 77 + }, + "id": 231, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_transmit_carrier_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Statistic transmit_carrier", + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic Carrier", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 87 + }, + "id": 232, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_transmit_colls_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Transmit colls", + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic Colls", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "entries", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "NF conntrack limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 87 + }, + "id": 61, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_nf_conntrack_entries{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "NF conntrack entries", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_nf_conntrack_entries_limit{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "NF conntrack limit", + "refId": "B", + "step": 240 + } + ], + "title": "NF Contrack", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Entries", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 97 + }, + "id": 230, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_arp_entries{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ device }} - ARP entries", + "refId": "A", + "step": 240 + } + ], + "title": "ARP Entries", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 97 + }, + "id": 288, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_network_mtu_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ device }} - Bytes", + "refId": "A", + "step": 240 + } + ], + "title": "MTU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 107 + }, + "id": 280, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_network_speed_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ device }} - Speed", + "refId": "A", + "step": 240 + } + ], + "title": "Speed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 107 + }, + "id": 289, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_network_transmit_queue_length{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ device }} - Interface transmit queue length", + "refId": "A", + "step": 240 + } + ], + "title": "Queue Length", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packetes drop (-) / process (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Dropped.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 117 + }, + "id": 290, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_softnet_processed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{cpu}} - Processed", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_softnet_dropped_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{cpu}} - Dropped", + "refId": "B", + "step": 240 + } + ], + "title": "Softnet Packets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 117 + }, + "id": 310, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_softnet_times_squeezed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{cpu}} - Squeezed", + "refId": "A", + "step": 240 + } + ], + "title": "Softnet Out of Quota", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 127 + }, + "id": 309, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_network_up{operstate=\"up\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{interface}} - Operational state UP", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_network_carrier{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{device}} - Physical link state", + "refId": "B" + } + ], + "title": "Network Operational Status", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Network Traffic", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 273, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 63, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_TCP_alloc{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP_alloc - Allocated sockets", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_TCP_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP_inuse - Tcp sockets currently in use", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_TCP_mem{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP_mem - Used memory for tcp", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_TCP_orphan{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP_orphan - Orphan sockets", + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_TCP_tw{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP_tw - Sockets waiting close", + "refId": "E", + "step": 240 + } + ], + "title": "Sockstat TCP", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 124, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_UDPLITE_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDPLITE_inuse - Udplite sockets currently in use", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_UDP_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP_inuse - Udp sockets currently in use", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_UDP_mem{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP_mem - Used memory for udp", + "refId": "C", + "step": 240 + } + ], + "title": "Sockstat UDP", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 125, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_FRAG_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "FRAG_inuse - Frag sockets currently in use", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_RAW_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "RAW_inuse - Raw sockets currently in use", + "refId": "C", + "step": 240 + } + ], + "title": "Sockstat FRAG / RAW", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 220, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_TCP_mem_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "mem_bytes - TCP sockets in that state", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_UDP_mem_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "mem_bytes - UDP sockets in that state", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_FRAG_memory{instance=\"$node\",job=\"$job\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "FRAG_memory - Used memory for frag", + "refId": "C" + } + ], + "title": "Sockstat Memory Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "sockets", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 68 + }, + "id": 126, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_sockets_used{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Sockets_used - Sockets currently in use", + "refId": "A", + "step": 240 + } + ], + "title": "Sockstat Used", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Network Sockstat", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 274, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "octets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 221, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_IpExt_InOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "InOctets - Received octets", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_IpExt_OutOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "OutOctets - Sent octets", + "refId": "B", + "step": 240 + } + ], + "title": "Netstat IP In / Out Octets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "datagrams", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 81, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Ip_Forwarding{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Forwarding - IP forwarding", + "refId": "A", + "step": 240 + } + ], + "title": "Netstat IP Forwarding", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "messages out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 115, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Icmp_InMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "InMsgs - Messages which the entity received. Note that this counter includes all those counted by icmpInErrors", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Icmp_OutMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "OutMsgs - Messages which this entity attempted to send. Note that this counter includes all those counted by icmpOutErrors", + "refId": "B", + "step": 240 + } + ], + "title": "ICMP In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "messages out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 50, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Icmp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "InErrors - Messages which the entity received but determined as having ICMP-specific errors (bad ICMP checksums, bad length, etc.)", + "refId": "A", + "step": 240 + } + ], + "title": "ICMP Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "datagrams out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Snd.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 69 + }, + "id": 55, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Udp_InDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "InDatagrams - Datagrams received", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Udp_OutDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "OutDatagrams - Datagrams sent", + "refId": "B", + "step": 240 + } + ], + "title": "UDP In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "datagrams", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 69 + }, + "id": 109, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Udp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "InErrors - UDP Datagrams that could not be delivered to an application", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Udp_NoPorts{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "NoPorts - UDP Datagrams received on a port with no listener", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_UdpLite_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "InErrors Lite - UDPLite Datagrams that could not be delivered to an application", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Udp_RcvbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "RcvbufErrors - UDP buffer errors received", + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Udp_SndbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "SndbufErrors - UDP buffer errors send", + "refId": "E", + "step": 240 + } + ], + "title": "UDP Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "datagrams out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Snd.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 79 + }, + "id": 299, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Tcp_InSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "InSegs - Segments received, including those received in error. This count includes segments received on currently established connections", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Tcp_OutSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "OutSegs - Segments sent, including those on current connections but excluding those containing only retransmitted octets", + "refId": "B", + "step": 240 + } + ], + "title": "TCP In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 79 + }, + "id": 104, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_TcpExt_ListenOverflows{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "ListenOverflows - Times the listen queue of a socket overflowed", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_TcpExt_ListenDrops{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "ListenDrops - SYNs to LISTEN sockets ignored", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_TcpExt_TCPSynRetrans{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCPSynRetrans - SYN-SYN/ACK retransmits to break down retransmissions in SYN, fast/timeout retransmits", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Tcp_RetransSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "RetransSegs - Segments retransmitted - that is, the number of TCP segments transmitted containing one or more previously transmitted octets", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Tcp_InErrs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "InErrs - Segments received in error (e.g., bad TCP checksums)", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Tcp_OutRsts{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "OutRsts - Segments sent with RST flag", + "refId": "F" + } + ], + "title": "TCP Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "connections", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*MaxConn *./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 89 + }, + "id": 85, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_netstat_Tcp_CurrEstab{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "CurrEstab - TCP connections for which the current state is either ESTABLISHED or CLOSE- WAIT", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_netstat_Tcp_MaxConn{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "MaxConn - Limit on the total number of TCP connections the entity can support (Dynamic is \"-1\")", + "refId": "B", + "step": 240 + } + ], + "title": "TCP Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Sent.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 89 + }, + "id": 91, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_TcpExt_SyncookiesFailed{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "SyncookiesFailed - Invalid SYN cookies received", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_TcpExt_SyncookiesRecv{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "SyncookiesRecv - SYN cookies received", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_TcpExt_SyncookiesSent{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "SyncookiesSent - SYN cookies sent", + "refId": "C", + "step": 240 + } + ], + "title": "TCP SynCookie", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "connections", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 99 + }, + "id": 82, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Tcp_ActiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "ActiveOpens - TCP connections that have made a direct transition to the SYN-SENT state from the CLOSED state", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Tcp_PassiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "PassiveOpens - TCP connections that have made a direct transition to the SYN-RCVD state from the LISTEN state", + "refId": "B", + "step": 240 + } + ], + "title": "TCP Direct Transition", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Enable with --collector.tcpstat argument on node-exporter", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "connections", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 99 + }, + "id": 320, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"established\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "established - TCP sockets in established state", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"fin_wait2\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "fin_wait2 - TCP sockets in fin_wait2 state", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"listen\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "listen - TCP sockets in listen state", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"time_wait\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "time_wait - TCP sockets in time_wait state", + "range": true, + "refId": "D", + "step": 240 + } + ], + "title": "TCP Stat", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Network Netstat", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 279, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 66 + }, + "id": 40, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_scrape_collector_duration_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{collector}} - Scrape duration", + "refId": "A", + "step": 240 + } + ], + "title": "Node Exporter Scrape Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*error.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + }, + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 66 + }, + "id": 157, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_scrape_collector_success{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{collector}} - Scrape success", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_textfile_scrape_error{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{collector}} - Scrape textfile error (1 = true)", + "refId": "B", + "step": 240 + } + ], + "title": "Node Exporter Scrape", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Node Exporter", + "type": "row" + } + ], + "refresh": "", + "revision": 1, + "schemaVersion": 38, + "tags": [ + "linux" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "node", + "value": "node" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Job", + "multi": false, + "name": "job", + "options": [], + "query": { + "query": "label_values(node_uname_info, job)", + "refId": "Prometheus-job-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(node_uname_info{job=\"$job\"}, instance)", + "hide": 0, + "includeAll": false, + "label": "Host", + "multi": false, + "name": "node", + "options": [], + "query": { + "query": "label_values(node_uname_info{job=\"$job\"}, instance)", + "refId": "Prometheus-node-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", + "value": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+" + }, + "hide": 2, + "includeAll": false, + "multi": false, + "name": "diskdevices", + "options": [ + { + "selected": true, + "text": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", + "value": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+" + } + ], + "query": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Node Exporter Full", + "uid": "rYdddlPWk", + "version": 2, + "weekStart": "" +} diff --git a/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml b/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml index b5f03034..2d6a44da 100644 --- a/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml +++ b/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml @@ -11,6 +11,13 @@ scrape_configs: static_configs: - targets: ['localhost:9090'] + - job_name: node + static_configs: + # Add node-exporter targets to be monitored below + # - targets: ['example-host:9100'] + # labels: + # instance: 'my-host' + - job_name: 'blackbox' scrape_interval: 10s metrics_path: /probe diff --git a/stack_orchestrator/data/stacks/monitoring/README.md b/stack_orchestrator/data/stacks/monitoring/README.md index feced015..7b410478 100644 --- a/stack_orchestrator/data/stacks/monitoring/README.md +++ b/stack_orchestrator/data/stacks/monitoring/README.md @@ -2,9 +2,12 @@ * Instructions to setup and run a Prometheus server and Grafana dashboard * Comes with the following built-in exporters / dashboards: - * [Prometheus Blackbox Exporter](https://grafana.com/grafana/dashboards/7587-prometheus-blackbox-exporter/) - for tracking HTTP endpoints - * [NodeJS Application Dashboard](https://grafana.com/grafana/dashboards/11159-nodejs-application-dashboard/) - for default NodeJS metrics * Chain Head Exporter - for tracking chain heads given external ETH RPC endpoints + * Watchers dashboard + * [Prometheus Blackbox](https://grafana.com/grafana/dashboards/7587-prometheus-blackbox-exporter/) - for tracking HTTP endpoints + * [NodeJS Application Dashboard](https://grafana.com/grafana/dashboards/11159-nodejs-application-dashboard/) - for default NodeJS metrics + * [PostgreSQL Database](https://grafana.com/grafana/dashboards/9628-postgresql-database/) - for monitoring Postgres dbs + * [Node Exporter Full](https://grafana.com/grafana/dashboards/1860-node-exporter-full/) - for monitoring system metrics * See [monitoring-watchers.md](./monitoring-watchers.md) for an example usage of the stack with pre-configured dashboards for watchers ## Setup @@ -72,7 +75,20 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --de - targets: [':'] ``` -* Blackbox: update the `blackbox` job to add any endpoints to be monitored on the Blackbox dashboard: +* Node exporter: update the `node` job to add any node-exporter targets to be monitored: + + ```yml + ... + - job_name: 'node' + ... + static_configs: + # Add node-exporter targets to be monitored below + - targets: [example-host:9100] + labels: + instance: 'my-host' + ``` + +* Blackbox (in-stack exporter): update the `blackbox` job to add any endpoints to be monitored on the Blackbox dashboard: ```yml ... @@ -85,7 +101,7 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --de - ``` -* Postgres: +* Postgres (in-stack exporter): * Update the `postgres` job to add Postgres db targets to be monitored: ```yml @@ -111,6 +127,8 @@ Place the dashboard json files in grafana dashboards config directory (`monitori Set the following env variables in the deployment env config file (`monitoring-deployment/config.env`): ```bash + # For chain-head exporter + # External ETH RPC endpoint (ethereum) # (Optional, default: https://mainnet.infura.io/v3) CERC_ETH_RPC_ENDPOINT= diff --git a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md index 676ca463..23335efb 100644 --- a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md +++ b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md @@ -1,6 +1,6 @@ # Monitoring Watchers -Instructions to setup and run monitoring stack with pre-configured watcher dashboards +Instructions to setup and run monitoring stack with configured Watchers dashboard ## Create a deployment @@ -98,7 +98,7 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers chain: 'filecoin' ``` -Add scrape config as done above for any additional watcher to add it to the watcher dashboard. +Add scrape config as done above for any additional watcher to add it to the Watchers dashboard. ### Env diff --git a/stack_orchestrator/data/stacks/node-exporter/README.md b/stack_orchestrator/data/stacks/node-exporter/README.md new file mode 100644 index 00000000..30376af9 --- /dev/null +++ b/stack_orchestrator/data/stacks/node-exporter/README.md @@ -0,0 +1,17 @@ +# node-exporter + +## Start the stack + +```bash +laconic-so --stack node-exporter deploy up +``` + +* The host node's metrics can be accessed at `http://localhost:9100/metrics` + +## Clean up + +Stop the node-exporter running in background: + +```bash +laconic-so --stack node-exporter deploy down +``` diff --git a/stack_orchestrator/data/stacks/node-exporter/stack.yml b/stack_orchestrator/data/stacks/node-exporter/stack.yml new file mode 100644 index 00000000..a9b3b7b3 --- /dev/null +++ b/stack_orchestrator/data/stacks/node-exporter/stack.yml @@ -0,0 +1,6 @@ +version: "0.1" +name: node-exporter +repos: +containers: +pods: + - node-exporter From 837e4438006bb142e8ec9be47e2a12bd8bc9fc9a Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Thu, 21 Dec 2023 18:05:40 -0600 Subject: [PATCH 60/75] Support application removal requests. (#697) * Support application removal request. * Git should never prompt when deploying a webapp --- .../webapp/deploy_webapp_from_registry.py | 36 ++-- .../webapp/undeploy_webapp_from_registry.py | 189 ++++++++++++++++++ stack_orchestrator/deploy/webapp/util.py | 38 +++- stack_orchestrator/main.py | 6 +- 4 files changed, 243 insertions(+), 26 deletions(-) create mode 100644 stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py index 02731992..2e4544eb 100644 --- a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -26,7 +26,8 @@ from stack_orchestrator.deploy.webapp import deploy_webapp from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient, build_container_image, push_container_image, file_hash, deploy_to_k8s, publish_deployment, - hostname_for_deployment_request, generate_hostname_for_app) + hostname_for_deployment_request, generate_hostname_for_app, + match_owner) def process_app_deployment_request( @@ -57,19 +58,12 @@ def process_app_deployment_request( dns_crn = f"{dns_record_namespace}/{fqdn}" dns_record = laconic.get_record(dns_crn) if dns_record: - dns_record_owners = dns_record.owners - dns_request_owners = [] - if dns_record.request: - prev_request = laconic.get_record(dns_record.request, require=True) - dns_request_owners = prev_request.owners + matched_owner = match_owner(app_deployment_request, dns_record) + if not matched_owner and dns_record.request: + matched_owner = match_owner(app_deployment_request, laconic.get_record(dns_record.request, require=True)) - owner_match = None - - for owner in app_deployment_request.owners: - if owner in dns_request_owners or owner in dns_record_owners: - owner_match = owner - if owner_match: - print("Matched DnsRecord ownership to", owner) + if matched_owner: + print("Matched DnsRecord ownership:", matched_owner) else: raise Exception("Unable to confirm ownership of DnsRecord %s for request %s" % (dns_record.id, app_deployment_request.id)) @@ -153,7 +147,6 @@ def dump_known_requests(filename, requests): @click.command() @click.option("--kube-config", help="Provide a config file for a k8s deployment") -@click.option("--kube-config", help="Provide a config file for a k8s deployment") @click.option("--laconic-config", help="Provide a config file for laconicd", required=True) @click.option("--image-registry", help="Provide a container image registry url for this k8s cluster") @click.option("--deployment-parent-dir", help="Create deployment directories beneath this directory", required=True) @@ -237,9 +230,20 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_ if d.attributes.request: deployments_by_request[d.attributes.request] = d + # Find removal requests. + cancellation_requests = {} + removal_requests = laconic.app_deployment_removal_requests() + for r in removal_requests: + if r.attributes.request: + cancellation_requests[r.attributes.request] = r + requests_to_execute = [] for r in requests_by_name.values(): - if r.id not in deployments_by_request: + if r.id in cancellation_requests and match_owner(cancellation_requests[r.id], r): + print(f"Found deployment cancellation request for {r.id} at {cancellation_requests[r.id].id}") + elif r.id in deployments_by_request: + print(f"Found satisfied request for {r.id} at {deployments_by_request[r.id].id}") + else: if r.id not in previous_requests: print(f"Request {r.id} needs to processed.") requests_to_execute.append(r) @@ -247,8 +251,6 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_ print( f"Skipping unsatisfied request {r.id} because we have seen it before." ) - else: - print(f"Found satisfied request {r.id} at {deployments_by_request[r.id].names[0]}") print("Found %d unsatisfied request(s) to process." % len(requests_to_execute)) diff --git a/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py new file mode 100644 index 00000000..6c5cac85 --- /dev/null +++ b/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py @@ -0,0 +1,189 @@ +# Copyright ©2023 Vulcanize +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import json +import os +import shutil +import subprocess +import sys + +import click + +from stack_orchestrator.deploy.webapp.util import LaconicRegistryClient, match_owner + + +def process_app_removal_request(ctx, + laconic: LaconicRegistryClient, + app_removal_request, + deployment_parent_dir, + delete_volumes, + delete_names): + deployment_record = laconic.get_record(app_removal_request.attributes.deployment, require=True) + dns_record = laconic.get_record(deployment_record.attributes.dns, require=True) + deployment_dir = os.path.join(deployment_parent_dir, dns_record.attributes.name) + + if not os.path.exists(deployment_dir): + raise Exception("Deployment directory %s does not exist." % deployment_dir) + + # Check if the removal request is from the owner of the DnsRecord or deployment record. + matched_owner = match_owner(app_removal_request, deployment_record, dns_record) + + # Or of the original deployment request. + if not matched_owner and deployment_record.request: + matched_owner = match_owner(app_removal_request, laconic.get_record(deployment_record.request, require=True)) + + if matched_owner: + print("Matched deployment ownership:", matched_owner) + else: + raise Exception("Unable to confirm ownership of deployment %s for removal request %s" % + (deployment_record.id, app_removal_request.id)) + + # TODO(telackey): Call the function directly. The easiest way to build the correct click context is to + # exec the process, but it would be better to refactor so we could just call down_operation with the + # necessary parameters + down_command = [sys.argv[0], "deployment", "--dir", deployment_dir, "down"] + if delete_volumes: + down_command.append("--delete-volumes") + result = subprocess.run(down_command) + result.check_returncode() + + removal_record = { + "record": { + "type": "ApplicationDeploymentRemovalRecord", + "version": "1.0.0", + "request": app_removal_request.id, + "deployment": deployment_record.id, + } + } + laconic.publish(removal_record) + + if delete_names: + if deployment_record.names: + for name in deployment_record.names: + laconic.delete_name(name) + + if dns_record.names: + for name in dns_record.names: + laconic.delete_name(name) + + if delete_volumes: + shutil.rmtree(deployment_dir) + + +def load_known_requests(filename): + if filename and os.path.exists(filename): + return json.load(open(filename, "r")) + return {} + + +def dump_known_requests(filename, requests): + if not filename: + return + known_requests = load_known_requests(filename) + for r in requests: + known_requests[r.id] = r.createTime + json.dump(known_requests, open(filename, "w")) + + +@click.command() +@click.option("--laconic-config", help="Provide a config file for laconicd", required=True) +@click.option("--deployment-parent-dir", help="Create deployment directories beneath this directory", required=True) +@click.option("--request-id", help="The ApplicationDeploymentRemovalRequest to process") +@click.option("--discover", help="Discover and process all pending ApplicationDeploymentRemovalRequests", + is_flag=True, default=False) +@click.option("--state-file", help="File to store state about previously seen requests.") +@click.option("--only-update-state", help="Only update the state file, don't process any requests anything.", is_flag=True) +@click.option("--delete-names/--preserve-names", help="Delete all names associated with removed deployments.", default=True) +@click.option("--delete-volumes/--preserve-volumes", default=True, help="delete data volumes") +@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True) +@click.pass_context +def command(ctx, laconic_config, deployment_parent_dir, + request_id, discover, state_file, only_update_state, + delete_names, delete_volumes, dry_run): + if request_id and discover: + print("Cannot specify both --request-id and --discover", file=sys.stderr) + sys.exit(2) + + if not request_id and not discover: + print("Must specify either --request-id or --discover", file=sys.stderr) + sys.exit(2) + + if only_update_state and not state_file: + print("--only-update-state requires --state-file", file=sys.stderr) + sys.exit(2) + + laconic = LaconicRegistryClient(laconic_config) + + # Find deployment removal requests. + # single request + if request_id: + requests = [laconic.get_record(request_id, require=True)] + # TODO: assert record type + # all requests + elif discover: + requests = laconic.app_deployment_removal_requests() + + if only_update_state: + if not dry_run: + dump_known_requests(state_file, requests) + return + + previous_requests = load_known_requests(state_file) + requests.sort(key=lambda r: r.createTime) + + # Find deployments. + deployments = {} + for d in laconic.app_deployments(all=True): + deployments[d.id] = d + + # Find removal requests. + removals_by_deployment = {} + removals_by_request = {} + for r in laconic.app_deployment_removals(): + if r.attributes.deployment: + # TODO: should we handle CRNs? + removals_by_deployment[r.attributes.deployment] = r + + requests_to_execute = [] + for r in requests: + if not r.attributes.deployment: + print(f"Skipping removal request {r.id} since it was a cancellation.") + elif r.id in removals_by_request: + print(f"Found satisfied request for {r.id} at {removals_by_request[r.id].id}") + elif r.attributes.deployment in removals_by_deployment: + print( + f"Found removal record for indicated deployment {r.attributes.deployment} at " + f"{removals_by_deployment[r.attributes.deployment].id}") + else: + if r.id not in previous_requests: + print(f"Request {r.id} needs to processed.") + requests_to_execute.append(r) + else: + print(f"Skipping unsatisfied request {r.id} because we have seen it before.") + + print("Found %d unsatisfied request(s) to process." % len(requests_to_execute)) + + if not dry_run: + for r in requests_to_execute: + try: + process_app_removal_request( + ctx, + laconic, + r, + os.path.abspath(deployment_parent_dir), + delete_volumes, + delete_names + ) + finally: + dump_known_requests(state_file, [r]) diff --git a/stack_orchestrator/deploy/webapp/util.py b/stack_orchestrator/deploy/webapp/util.py index f95d3f3e..ef98117b 100644 --- a/stack_orchestrator/deploy/webapp/util.py +++ b/stack_orchestrator/deploy/webapp/util.py @@ -49,6 +49,14 @@ def cmd(*vargs): raise err +def match_owner(recordA, *records): + for owner in recordA.owners: + for otherRecord in records: + if owner in otherRecord.owners: + return owner + return None + + class LaconicRegistryClient: def __init__(self, config_file): self.config_file = config_file @@ -146,11 +154,17 @@ class LaconicRegistryClient: raise Exception("Cannot locate record:", name_or_id) return None - def app_deployment_requests(self): - return self.list_records({"type": "ApplicationDeploymentRequest"}, True) + def app_deployment_requests(self, all=True): + return self.list_records({"type": "ApplicationDeploymentRequest"}, all) - def app_deployments(self): - return self.list_records({"type": "ApplicationDeploymentRecord"}) + def app_deployments(self, all=True): + return self.list_records({"type": "ApplicationDeploymentRecord"}, all) + + def app_deployment_removal_requests(self, all=True): + return self.list_records({"type": "ApplicationDeploymentRemovalRequest"}, all) + + def app_deployment_removals(self, all=True): + return self.list_records({"type": "ApplicationDeploymentRemovalRecord"}, all) def publish(self, record, names=[]): tmpdir = tempfile.mkdtemp() @@ -165,11 +179,17 @@ class LaconicRegistryClient: cmd("laconic", "-c", self.config_file, "cns", "record", "publish", "--filename", record_fname) )["id"] for name in names: - cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, new_record_id) + self.set_name(name, new_record_id) return new_record_id finally: cmd("rm", "-rf", tmpdir) + def set_name(self, name, record_id): + cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, record_id) + + def delete_name(self, name): + cmd("laconic", "-c", self.config_file, "cns", "name", "delete", name) + def file_hash(filename): return hashlib.sha1(open(filename).read().encode()).hexdigest() @@ -187,9 +207,11 @@ def build_container_image(app_record, tag, extra_build_args=[]): print(f"Cloning repository {repo} to {clone_dir} ...") if ref: # TODO: Determing branch or hash, and use depth 1 if we can. - result = subprocess.run(["git", "clone", repo, clone_dir]) - result.check_returncode() - subprocess.check_call(["git", "checkout", ref], cwd=clone_dir) + git_env = dict(os.environ.copy()) + # Never prompt + git_env["GIT_TERMINAL_PROMPT"] = "0" + subprocess.check_call(["git", "clone", repo, clone_dir], env=git_env) + subprocess.check_call(["git", "checkout", ref], cwd=clone_dir, env=git_env) else: result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir]) result.check_returncode() diff --git a/stack_orchestrator/main.py b/stack_orchestrator/main.py index fabc58ff..a4644412 100644 --- a/stack_orchestrator/main.py +++ b/stack_orchestrator/main.py @@ -20,7 +20,10 @@ from stack_orchestrator.repos import setup_repositories from stack_orchestrator.build import build_containers from stack_orchestrator.build import build_npms from stack_orchestrator.build import build_webapp -from stack_orchestrator.deploy.webapp import run_webapp, deploy_webapp, deploy_webapp_from_registry +from stack_orchestrator.deploy.webapp import (run_webapp, + deploy_webapp, + deploy_webapp_from_registry, + undeploy_webapp_from_registry) from stack_orchestrator.deploy import deploy from stack_orchestrator import version from stack_orchestrator.deploy import deployment @@ -54,6 +57,7 @@ cli.add_command(build_webapp.command, "build-webapp") cli.add_command(run_webapp.command, "run-webapp") cli.add_command(deploy_webapp.command, "deploy-webapp") cli.add_command(deploy_webapp_from_registry.command, "deploy-webapp-from-registry") +cli.add_command(undeploy_webapp_from_registry.command, "undeploy-webapp-from-registry") cli.add_command(deploy.command, "deploy") # deploy is an alias for deploy-system cli.add_command(deploy.command, "deploy-system") cli.add_command(deployment.command, "deployment") From eae2af7cccfe8b55d6f1772fe25a245f4d32a3e8 Mon Sep 17 00:00:00 2001 From: Nabarun Gogoi Date: Mon, 8 Jan 2024 11:51:54 +0530 Subject: [PATCH 61/75] Upgrade watcher versions (#699) --- stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml | 2 +- stack_orchestrator/data/stacks/sushiswap-v3/stack.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml index 3f9dd43e..847df33c 100644 --- a/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml +++ b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml @@ -2,7 +2,7 @@ version: "1.0" name: merkl-sushiswap-v3 description: "SushiSwap v3 watcher stack" repos: - - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.4 + - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.6 containers: - cerc/watcher-merkl-sushiswap-v3 pods: diff --git a/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml b/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml index 90766012..12103d62 100644 --- a/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml +++ b/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml @@ -2,7 +2,7 @@ version: "1.0" name: sushiswap-v3 description: "SushiSwap v3 watcher stack" repos: - - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.5 + - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.6 containers: - cerc/watcher-sushiswap-v3 pods: From a3eb3c0bb0162b10ea7e22df56baa2bc54dd147b Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Mon, 8 Jan 2024 17:25:30 +0530 Subject: [PATCH 62/75] Setup basic alerting for watchers in monitoring stack (#698) * Provision Grafana alert contactpoints and policies for Slack * Add watcher alert rules * Update watcher monitoring instructions * Add listening port flag to node exporter command * Add reference links --- .../compose/docker-compose-node-exporter.yml | 3 +- .../provisioning/alerting/contactpoints.yml | 14 + .../provisioning/alerting/policies.yml | 15 + .../config/monitoring/watcher-alert-rules.yml | 973 ++++++++++++++++++ .../stacks/monitoring/monitoring-watchers.md | 23 + 5 files changed, 1026 insertions(+), 2 deletions(-) create mode 100644 stack_orchestrator/data/config/monitoring/grafana/provisioning/alerting/contactpoints.yml create mode 100644 stack_orchestrator/data/config/monitoring/grafana/provisioning/alerting/policies.yml create mode 100644 stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml diff --git a/stack_orchestrator/data/compose/docker-compose-node-exporter.yml b/stack_orchestrator/data/compose/docker-compose-node-exporter.yml index 8b49c269..6fa02d6d 100644 --- a/stack_orchestrator/data/compose/docker-compose-node-exporter.yml +++ b/stack_orchestrator/data/compose/docker-compose-node-exporter.yml @@ -5,12 +5,11 @@ services: image: prom/node-exporter:latest restart: unless-stopped command: + - '--web.listen-address=:9100' - '--path.rootfs=/host' - '--collector.systemd' - '--collector.processes' network_mode: host pid: host - ports: - - 9100 volumes: - '/:/host:ro,rslave' diff --git a/stack_orchestrator/data/config/monitoring/grafana/provisioning/alerting/contactpoints.yml b/stack_orchestrator/data/config/monitoring/grafana/provisioning/alerting/contactpoints.yml new file mode 100644 index 00000000..5e924946 --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/provisioning/alerting/contactpoints.yml @@ -0,0 +1,14 @@ +# https://www.clever-cloud.com/blog/features/2021/12/03/slack-alerts-for-grafana/ + +apiVersion: 1 + +contactPoints: + - orgId: 1 + name: SlackNotifier + receivers: + - uid: a71b06e3-58b6-41fe-af65-fbbb29653951 + type: slack + settings: + # Slack hook URL (see https://api.slack.com/messaging/webhooks) + url: + disableResolveMessage: false diff --git a/stack_orchestrator/data/config/monitoring/grafana/provisioning/alerting/policies.yml b/stack_orchestrator/data/config/monitoring/grafana/provisioning/alerting/policies.yml new file mode 100644 index 00000000..b1beb31d --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/provisioning/alerting/policies.yml @@ -0,0 +1,15 @@ +# https://grafana.com/docs/grafana/latest/alerting/alerting-rules/create-notification-policy/ + +apiVersion: 1 + +policies: + - orgId: 1 + receiver: grafana-default-email + group_by: + - grafana_folder + - alertname + routes: + - receiver: SlackNotifier + object_matchers: + # Add matchers below + # - ['grafana_folder', '=', 'MyAlerts'] diff --git a/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml b/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml new file mode 100644 index 00000000..7e26ba14 --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml @@ -0,0 +1,973 @@ +# https://grafana.com/docs/grafana/latest/alerting/alerting-rules/create-grafana-managed-rule/ + +apiVersion: 1 + +groups: + - orgId: 1 + name: watcher + folder: WatcherAlerts + interval: 30s + rules: + # Azimuth + - uid: azimuth_diff_external + title: azimuth_watcher_head_tracking + condition: condition + data: + - refId: diff + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + disableTextWrap: false + editorMode: code + expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"} + fullMetaSearch: false + includeNullMetadata: true + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: diff + useBackend: false + - refId: latest_external + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: latest_block_number{chain="ethereum"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_external + - refId: latest_indexed + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_indexed + - refId: condition + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: ${diff} >= 16 + intervalMs: 1000 + maxDataPoints: 43200 + refId: condition + type: math + dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca + panelId: 24 + noDataState: Alerting + execErrState: Alerting + for: 15m + annotations: + __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca + __panelId__: "24" + summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} + isPaused: false + - uid: censures_diff_external + title: censures_watcher_head_tracking + condition: condition + data: + - refId: diff + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + disableTextWrap: false + editorMode: code + expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"} + fullMetaSearch: false + includeNullMetadata: true + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: diff + useBackend: false + - refId: latest_external + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: latest_block_number{chain="ethereum"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_external + - refId: latest_indexed + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_indexed + - refId: condition + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: ${diff} >= 16 + intervalMs: 1000 + maxDataPoints: 43200 + refId: condition + type: math + dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca + panelId: 24 + noDataState: Alerting + execErrState: Alerting + for: 15m + annotations: + __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca + __panelId__: "24" + summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} + isPaused: false + - uid: claims_diff_external + title: claims_watcher_head_tracking + condition: condition + data: + - refId: diff + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + disableTextWrap: false + editorMode: code + expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"} + fullMetaSearch: false + includeNullMetadata: true + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: diff + useBackend: false + - refId: latest_external + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: latest_block_number{chain="ethereum"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_external + - refId: latest_indexed + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_indexed + - refId: condition + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: ${diff} >= 16 + intervalMs: 1000 + maxDataPoints: 43200 + refId: condition + type: math + dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca + panelId: 24 + noDataState: Alerting + execErrState: Alerting + for: 15m + annotations: + __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca + __panelId__: "24" + summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} + isPaused: false + - uid: conditional_star_release_diff_external + title: conditional_star_release_watcher_head_tracking + condition: condition + data: + - refId: diff + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + disableTextWrap: false + editorMode: code + expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"} + fullMetaSearch: false + includeNullMetadata: true + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: diff + useBackend: false + - refId: latest_external + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: latest_block_number{chain="ethereum"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_external + - refId: latest_indexed + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_indexed + - refId: condition + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: ${diff} >= 16 + intervalMs: 1000 + maxDataPoints: 43200 + refId: condition + type: math + dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca + panelId: 24 + noDataState: Alerting + execErrState: Alerting + for: 15m + annotations: + __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca + __panelId__: "24" + summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} + isPaused: false + - uid: delegated_sending_diff_external + title: delegated_sending_watcher_head_tracking + condition: condition + data: + - refId: diff + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + disableTextWrap: false + editorMode: code + expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"} + fullMetaSearch: false + includeNullMetadata: true + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: diff + useBackend: false + - refId: latest_external + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: latest_block_number{chain="ethereum"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_external + - refId: latest_indexed + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_indexed + - refId: condition + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: ${diff} >= 16 + intervalMs: 1000 + maxDataPoints: 43200 + refId: condition + type: math + dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca + panelId: 24 + noDataState: Alerting + execErrState: Alerting + for: 15m + annotations: + __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca + __panelId__: "24" + summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} + isPaused: false + - uid: ecliptic_diff_external + title: ecliptic_watcher_head_tracking + condition: condition + data: + - refId: diff + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + disableTextWrap: false + editorMode: code + expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"} + fullMetaSearch: false + includeNullMetadata: true + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: diff + useBackend: false + - refId: latest_external + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: latest_block_number{chain="ethereum"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_external + - refId: latest_indexed + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_indexed + - refId: condition + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: ${diff} >= 16 + intervalMs: 1000 + maxDataPoints: 43200 + refId: condition + type: math + dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca + panelId: 24 + noDataState: Alerting + execErrState: Alerting + for: 15m + annotations: + __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca + __panelId__: "24" + summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} + isPaused: false + - uid: linear_star_release_diff_external + title: linear_star_release_watcher_head_tracking + condition: condition + data: + - refId: diff + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + disableTextWrap: false + editorMode: code + expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="linear_star_release", kind="latest_indexed"} + fullMetaSearch: false + includeNullMetadata: true + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: diff + useBackend: false + - refId: latest_external + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: latest_block_number{chain="ethereum"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_external + - refId: latest_indexed + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_indexed + - refId: condition + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: ${diff} >= 16 + intervalMs: 1000 + maxDataPoints: 43200 + refId: condition + type: math + dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca + panelId: 24 + noDataState: Alerting + execErrState: Alerting + for: 15m + annotations: + __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca + __panelId__: "24" + summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} + isPaused: false + - uid: polls_diff_external + title: polls_watcher_head_tracking + condition: condition + data: + - refId: diff + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + disableTextWrap: false + editorMode: code + expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"} + fullMetaSearch: false + includeNullMetadata: true + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: diff + useBackend: false + - refId: latest_external + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: latest_block_number{chain="ethereum"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_external + - refId: latest_indexed + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_indexed + - refId: condition + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: ${diff} >= 16 + intervalMs: 1000 + maxDataPoints: 43200 + refId: condition + type: math + dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca + panelId: 24 + noDataState: Alerting + execErrState: Alerting + for: 15m + annotations: + __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca + __panelId__: "24" + summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} + isPaused: false + + # Sushi + - uid: sushiswap_diff_external + title: sushiswap_watcher_head_tracking + condition: condition + data: + - refId: diff + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + disableTextWrap: false + editorMode: code + expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"} + fullMetaSearch: false + includeNullMetadata: true + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: diff + useBackend: false + - refId: latest_external + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: latest_block_number{chain="filecoin"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_external + - refId: latest_indexed + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_indexed + - refId: condition + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: ${diff} >= 16 + intervalMs: 1000 + maxDataPoints: 43200 + refId: condition + type: math + dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca + panelId: 24 + noDataState: Alerting + execErrState: Alerting + for: 15m + annotations: + __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca + __panelId__: "24" + summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} + isPaused: false + - uid: merkl_sushiswap_diff_external + title: merkl_sushiswap_watcher_head_tracking + condition: condition + data: + - refId: diff + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + disableTextWrap: false + editorMode: code + expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"} + fullMetaSearch: false + includeNullMetadata: true + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: diff + useBackend: false + - refId: latest_external + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: latest_block_number{chain="filecoin"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_external + - refId: latest_indexed + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_indexed + - refId: condition + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: ${diff} >= 16 + intervalMs: 1000 + maxDataPoints: 43200 + refId: condition + type: math + dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca + panelId: 24 + noDataState: Alerting + execErrState: Alerting + for: 15m + annotations: + __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca + __panelId__: "24" + summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} + isPaused: false diff --git a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md index 23335efb..3673b530 100644 --- a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md +++ b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md @@ -35,6 +35,8 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-watchers-spec ## Configure +### Prometheus scrape config + Add the following scrape configs to prometheus config file (`monitoring-watchers-deployment/config/monitoring/prometheus/prometheus.yml`) in the deployment folder: ```yml @@ -100,6 +102,27 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers Add scrape config as done above for any additional watcher to add it to the Watchers dashboard. +### Grafana alerts config + +Place the pre-configured watcher alerts rules in Grafana provisioning directory: + + ```bash + cp monitoring-watchers-deployment/config/monitoring/watcher-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/ + ``` + +Update the alerting contact points config (`monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/contactpoints.yml`) with desired contact points + +Add corresponding routes to the notification policies config (`monitoring-watchers-deployment/monitoring/grafana/provisioning/alerting/policies.yaml`) with appropriate object-matchers: + + ```yml + ... + routes: + - receiver: SlackNotifier + object_matchers: + # Add matchers below + - ['grafana_folder', '=', 'WatcherAlerts'] + ``` + ### Env Set the following env variables in the deployment env config file (`monitoring-watchers-deployment/config.env`): From b83030f63b01311c32a43d4ad59adb2d241b1acf Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Tue, 9 Jan 2024 22:53:43 -0600 Subject: [PATCH 63/75] Use custom act with gitea. (#700) * Use custom act with gitea. * Make sure wget is available * Fix repo url --- .gitea/workflows/test-webapp.yml | 2 ++ .../container-build/cerc-act-runner-task-executor/build.sh | 4 +++- stack_orchestrator/data/stacks/package-registry/stack.yml | 6 ++++-- stack_orchestrator/deploy/deployment_create.py | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/test-webapp.yml b/.gitea/workflows/test-webapp.yml index 9fbf84b2..8b83c606 100644 --- a/.gitea/workflows/test-webapp.yml +++ b/.gitea/workflows/test-webapp.yml @@ -41,6 +41,8 @@ jobs: run: ./scripts/create_build_tag_file.sh - name: "Build local shiv package" run: ./scripts/build_shiv_package.sh + - name: "Install wget" # 20240109 - Only needed until the executors are updated. + run: apt update && apt install -y wget - name: Start dockerd # Also needed until we can incorporate into the executor run: | dockerd -H $DOCKER_HOST --userland-proxy=false & diff --git a/stack_orchestrator/data/container-build/cerc-act-runner-task-executor/build.sh b/stack_orchestrator/data/container-build/cerc-act-runner-task-executor/build.sh index b625ed4b..939d6898 100755 --- a/stack_orchestrator/data/container-build/cerc-act-runner-task-executor/build.sh +++ b/stack_orchestrator/data/container-build/cerc-act-runner-task-executor/build.sh @@ -2,4 +2,6 @@ # Build a local version of the task executor for act-runner source ${CERC_CONTAINER_BASE_DIR}/build-base.sh SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -docker build -t cerc/act-runner-task-executor:local -f ${CERC_REPO_BASE_DIR}/hosting/act-runner/Dockerfile.task-executor ${build_command_args} ${SCRIPT_DIR} + +cd ${CERC_REPO_BASE_DIR}/hosting/act-runner +docker build -t cerc/act-runner-task-executor:local -f Dockerfile.task-executor ${build_command_args} . diff --git a/stack_orchestrator/data/stacks/package-registry/stack.yml b/stack_orchestrator/data/stacks/package-registry/stack.yml index f6367ab1..8f56d200 100644 --- a/stack_orchestrator/data/stacks/package-registry/stack.yml +++ b/stack_orchestrator/data/stacks/package-registry/stack.yml @@ -2,9 +2,11 @@ version: "1.1" name: package-registry description: "Local Package Registry" repos: - - git.vdb.to/cerc-io/hosting - - gitea.com/gitea/act_runner + - github.com/telackey/gitea@telackey/entrypoint + - git.vdb.to/cerc-io/hosting@dboreham/fix-dind + - gitea.com/telackey/act_runner@telackey/entrypoint containers: + - cerc/gitea - cerc/act-runner - cerc/act-runner-task-executor pods: diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py index 9eaea30c..138fc1bb 100644 --- a/stack_orchestrator/deploy/deployment_create.py +++ b/stack_orchestrator/deploy/deployment_create.py @@ -444,7 +444,7 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw deployer_config_generator = getDeployerConfigGenerator(deployment_type) # TODO: make deployment_dir_path a Path above deployer_config_generator.generate(deployment_dir_path) - call_stack_deploy_create(deployment_context, [network_dir, initial_peers]) + call_stack_deploy_create(deployment_context, [network_dir, initial_peers, deployment_command_context]) # TODO: this code should be in the stack .py files but From db8aec52aa708d100bb440fe121b08f5ad74d835 Mon Sep 17 00:00:00 2001 From: Nabarun Gogoi Date: Mon, 15 Jan 2024 16:56:06 +0530 Subject: [PATCH 64/75] Pin commit hash of asset list repo in osmosis frontend app (#703) --- .../data/compose/docker-compose-osmosis-front-end-urbit.yml | 1 + .../data/compose/docker-compose-osmosis-front-end.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/stack_orchestrator/data/compose/docker-compose-osmosis-front-end-urbit.yml b/stack_orchestrator/data/compose/docker-compose-osmosis-front-end-urbit.yml index eebdfb01..5012de13 100644 --- a/stack_orchestrator/data/compose/docker-compose-osmosis-front-end-urbit.yml +++ b/stack_orchestrator/data/compose/docker-compose-osmosis-front-end-urbit.yml @@ -6,6 +6,7 @@ services: restart: on-failure environment: - NEXT_PUBLIC_WEB_API_BASE_URL=${CERC_WEB_API_BASE_URL} + - ASSET_LIST_COMMIT_HASH=a326bcefc51372b4912be5a2a2fa84a5d142a438 - NEXT_PUBLIC_BASEPATH=/apps/osmosis - NEXT_PUBLIC_URBIT_DEPLOYMENT=true working_dir: /app/packages/web diff --git a/stack_orchestrator/data/compose/docker-compose-osmosis-front-end.yml b/stack_orchestrator/data/compose/docker-compose-osmosis-front-end.yml index 0ada9367..4178383e 100644 --- a/stack_orchestrator/data/compose/docker-compose-osmosis-front-end.yml +++ b/stack_orchestrator/data/compose/docker-compose-osmosis-front-end.yml @@ -6,6 +6,7 @@ services: restart: on-failure environment: - NEXT_PUBLIC_WEB_API_BASE_URL=${CERC_WEB_API_BASE_URL} + - ASSET_LIST_COMMIT_HASH=a326bcefc51372b4912be5a2a2fa84a5d142a438 working_dir: /app/packages/web command: ["./build-app.sh"] volumes: From 4b3ea7c30f6650fb73de29e8d5133b037ab6917d Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Mon, 15 Jan 2024 12:10:48 -0600 Subject: [PATCH 65/75] Update independent act-runner stack to use custom act as well. (#702) * Update independent act-runner stack to use custom act as well. * Remove branches which are not needed or already merged. --- stack_orchestrator/data/stacks/act-runner/stack.yml | 2 +- stack_orchestrator/data/stacks/package-registry/stack.yml | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/stack_orchestrator/data/stacks/act-runner/stack.yml b/stack_orchestrator/data/stacks/act-runner/stack.yml index a236fccf..473bc01e 100644 --- a/stack_orchestrator/data/stacks/act-runner/stack.yml +++ b/stack_orchestrator/data/stacks/act-runner/stack.yml @@ -3,7 +3,7 @@ name: act-runner description: "Local act-runner" repos: - git.vdb.to/cerc-io/hosting - - gitea.com/gitea/act_runner + - gitea.com/telackey/act_runner@telackey/entrypoint containers: - cerc/act-runner - cerc/act-runner-task-executor diff --git a/stack_orchestrator/data/stacks/package-registry/stack.yml b/stack_orchestrator/data/stacks/package-registry/stack.yml index 8f56d200..e4b06a8c 100644 --- a/stack_orchestrator/data/stacks/package-registry/stack.yml +++ b/stack_orchestrator/data/stacks/package-registry/stack.yml @@ -2,11 +2,9 @@ version: "1.1" name: package-registry description: "Local Package Registry" repos: - - github.com/telackey/gitea@telackey/entrypoint - - git.vdb.to/cerc-io/hosting@dboreham/fix-dind + - git.vdb.to/cerc-io/hosting - gitea.com/telackey/act_runner@telackey/entrypoint containers: - - cerc/gitea - cerc/act-runner - cerc/act-runner-task-executor pods: From 0587813dd06d5218b6b2af89fa81e5fa559cf89b Mon Sep 17 00:00:00 2001 From: Thomas E Lackey Date: Mon, 15 Jan 2024 12:12:59 -0600 Subject: [PATCH 66/75] Create next.config.js if it is missing. (#701) * Create next.config.js if it is missing. * Add comment. --- .../container-build/cerc-nextjs-base/scripts/build-app.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh index 759ba255..40bcbeed 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh @@ -21,6 +21,11 @@ WORK_DIR="${1:-/app}" cd "${WORK_DIR}" || exit 1 +# If this file doesn't exist at all, we'll get errors below. +if [ ! -f "next.config.js" ]; then + touch next.config.js +fi + if [ ! -f "next.config.dist" ]; then cp next.config.js next.config.dist fi From 1f9653e6f7d8a4ef1e9745888157f5625a8d4350 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 16 Jan 2024 15:55:58 -0700 Subject: [PATCH 67/75] Fix kind mode and add k8s deployment test (#704) * Fix kind mode and add k8s deployment test * Fix lint errors --- stack_orchestrator/deploy/k8s/cluster_info.py | 6 +- stack_orchestrator/deploy/k8s/deploy_k8s.py | 63 ++++++++------- stack_orchestrator/deploy/k8s/helpers.py | 12 ++- tests/k8s-deploy/run-deploy-test.sh | 76 ++++++++++++++++--- 4 files changed, 115 insertions(+), 42 deletions(-) diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 7718c777..85fd63a8 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -168,8 +168,8 @@ class ClusterInfo: result.append(pv) return result - # to suit the deployment, and also annotate the container specs to point at said volumes - def get_deployment(self): + # TODO: put things like image pull policy into an object-scope struct + def get_deployment(self, image_pull_policy: str = None): containers = [] for pod_name in self.parsed_pod_yaml_map: pod = self.parsed_pod_yaml_map[pod_name] @@ -189,7 +189,7 @@ class ClusterInfo: container = client.V1Container( name=container_name, image=image_to_use, - image_pull_policy="Always", + image_pull_policy=image_pull_policy, env=envs_from_environment_variables_map(self.environment_variables.map), ports=[client.V1ContainerPort(container_port=port)], volume_mounts=volume_mounts, diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index bf82ebdf..0a339fe9 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -111,7 +111,7 @@ class K8sDeployer(Deployer): print("PVCs created:") print(f"{pvc_resp}") # Process compose files into a Deployment - deployment = self.cluster_info.get_deployment() + deployment = self.cluster_info.get_deployment(image_pull_policy=None if self.is_kind() else "Always") # Create the k8s objects if opts.o.debug: print(f"Sending this deployment: {deployment}") @@ -132,18 +132,18 @@ class K8sDeployer(Deployer): print("Service created:") print(f"{service_resp}") - # TODO: disable ingress for kind - ingress: client.V1Ingress = self.cluster_info.get_ingress() + if not self.is_kind(): + ingress: client.V1Ingress = self.cluster_info.get_ingress() - if opts.o.debug: - print(f"Sending this ingress: {ingress}") - ingress_resp = self.networking_api.create_namespaced_ingress( - namespace=self.k8s_namespace, - body=ingress - ) - if opts.o.debug: - print("Ingress created:") - print(f"{ingress_resp}") + if opts.o.debug: + print(f"Sending this ingress: {ingress}") + ingress_resp = self.networking_api.create_namespaced_ingress( + namespace=self.k8s_namespace, + body=ingress + ) + if opts.o.debug: + print("Ingress created:") + print(f"{ingress_resp}") def down(self, timeout, volumes): self.connect_api() @@ -196,16 +196,16 @@ class K8sDeployer(Deployer): except client.exceptions.ApiException as e: _check_delete_exception(e) - # TODO: disable ingress for kind - ingress: client.V1Ingress = self.cluster_info.get_ingress() - if opts.o.debug: - print(f"Deleting this ingress: {ingress}") - try: - self.networking_api.delete_namespaced_ingress( - name=ingress.metadata.name, namespace=self.k8s_namespace - ) - except client.exceptions.ApiException as e: - _check_delete_exception(e) + if not self.is_kind(): + ingress: client.V1Ingress = self.cluster_info.get_ingress() + if opts.o.debug: + print(f"Deleting this ingress: {ingress}") + try: + self.networking_api.delete_namespaced_ingress( + name=ingress.metadata.name, namespace=self.k8s_namespace + ) + except client.exceptions.ApiException as e: + _check_delete_exception(e) if self.is_kind(): # Destroy the kind cluster @@ -219,7 +219,7 @@ class K8sDeployer(Deployer): if all_pods.items: for p in all_pods.items: - if self.cluster_info.app_name in p.metadata.name: + if f"{self.cluster_info.app_name}-deployment" in p.metadata.name: pods.append(p) if not pods: @@ -266,7 +266,7 @@ class K8sDeployer(Deployer): ret = [] for p in pods.items: - if self.cluster_info.app_name in p.metadata.name: + if f"{self.cluster_info.app_name}-deployment" in p.metadata.name: pod_ip = p.status.pod_ip ports = AttrDict() for c in p.spec.containers: @@ -299,11 +299,20 @@ class K8sDeployer(Deployer): def logs(self, services, tail, follow, stream): self.connect_api() - pods = pods_in_deployment(self.core_api, "test-deployment") + pods = pods_in_deployment(self.core_api, self.cluster_info.app_name) if len(pods) > 1: print("Warning: more than one pod in the deployment") - k8s_pod_name = pods[0] - log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test") + if len(pods) == 0: + log_data = "******* Pods not running ********\n" + else: + k8s_pod_name = pods[0] + # If the pod is not yet started, the logs request below will throw an exception + try: + log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test") + except client.exceptions.ApiException as e: + if opts.o.debug: + print(f"Error from read_namespaced_pod_log: {e}") + log_data = "******* No logs available ********\n" return log_stream_from_string(log_data) def update(self): diff --git a/stack_orchestrator/deploy/k8s/helpers.py b/stack_orchestrator/deploy/k8s/helpers.py index 9f968dbf..62545dfd 100644 --- a/stack_orchestrator/deploy/k8s/helpers.py +++ b/stack_orchestrator/deploy/k8s/helpers.py @@ -21,6 +21,7 @@ from typing import Set, Mapping, List from stack_orchestrator.opts import opts from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names +from stack_orchestrator.deploy.deployer import DeployerException def _run_command(command: str): @@ -29,10 +30,13 @@ def _run_command(command: str): result = subprocess.run(command, shell=True) if opts.o.debug: print(f"Result: {result}") + return result def create_cluster(name: str, config_file: str): - _run_command(f"kind create cluster --name {name} --config {config_file}") + result = _run_command(f"kind create cluster --name {name} --config {config_file}") + if result.returncode != 0: + raise DeployerException(f"kind create cluster failed: {result}") def destroy_cluster(name: str): @@ -41,12 +45,14 @@ def destroy_cluster(name: str): def load_images_into_kind(kind_cluster_name: str, image_set: Set[str]): for image in image_set: - _run_command(f"kind load docker-image {image} --name {kind_cluster_name}") + result = _run_command(f"kind load docker-image {image} --name {kind_cluster_name}") + if result.returncode != 0: + raise DeployerException(f"kind create cluster failed: {result}") def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str): pods = [] - pod_response = core_api.list_namespaced_pod(namespace="default", label_selector="app=test-app") + pod_response = core_api.list_namespaced_pod(namespace="default", label_selector=f"app={deployment_name}") if opts.o.debug: print(f"pod_response: {pod_response}") for pod_info in pod_response.items: diff --git a/tests/k8s-deploy/run-deploy-test.sh b/tests/k8s-deploy/run-deploy-test.sh index b7ee9dd0..5f1c0102 100755 --- a/tests/k8s-deploy/run-deploy-test.sh +++ b/tests/k8s-deploy/run-deploy-test.sh @@ -1,14 +1,59 @@ #!/usr/bin/env bash set -e if [ -n "$CERC_SCRIPT_DEBUG" ]; then - set -x + set -x + # Dump environment variables for debugging + echo "Environment variables:" + env fi + +# Helper functions: TODO move into a separate file +wait_for_pods_started () { + for i in {1..5} + do + local ps_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir ps ) + + if [[ "$ps_output" == *"Running containers:"* ]]; then + # if ready, return + return + else + # if not ready, wait + sleep 5 + fi + done + # Timed out, error exit + echo "waiting for pods to start: FAILED" + delete_cluster_exit +} + +wait_for_log_output () { + for i in {1..5} + do + + local log_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs ) + + if [[ ! -z "$log_output" ]]; then + # if ready, return + return + else + # if not ready, wait + sleep 5 + fi + done + # Timed out, error exit + echo "waiting for pods log content: FAILED" + delete_cluster_exit +} + + +delete_cluster_exit () { + $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes + exit 1 +} + # Note: eventually this test should be folded into ../deploy/ # but keeping it separate for now for convenience TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 ) -# Dump environment variables for debugging -echo "Environment variables:" -env # Set a non-default repo dir export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir echo "Testing this package: $TEST_TARGET_SO" @@ -53,23 +98,36 @@ fi echo "deploy create output file test: passed" # Try to start the deployment $TEST_TARGET_SO deployment --dir $test_deployment_dir start -# TODO: add a check to see if the container is up -# Sleep because k8s not up yet -sleep 30 +wait_for_pods_started # Check logs command works +wait_for_log_output log_output_3=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs ) if [[ "$log_output_3" == *"Filesystem is fresh"* ]]; then echo "deployment logs test: passed" else echo "deployment logs test: FAILED" - exit 1 + delete_cluster_exit fi # Check the config variable CERC_TEST_PARAM_1 was passed correctly if [[ "$log_output_3" == *"Test-param-1: PASSED"* ]]; then echo "deployment config test: passed" else echo "deployment config test: FAILED" - exit 1 + delete_cluster_exit +fi +# Stop then start again and check the volume was preserved +$TEST_TARGET_SO deployment --dir $test_deployment_dir stop +# Sleep a bit just in case +sleep 2 +$TEST_TARGET_SO deployment --dir $test_deployment_dir start +wait_for_pods_started +wait_for_log_output +log_output_4=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs ) +if [[ "$log_output_4" == *"Filesystem is old"* ]]; then + echo "Retain volumes test: passed" +else + echo "Retain volumes test: FAILED" + delete_cluster_exit fi # Stop and clean up $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes From 90cebdb7a68dbbc4c84b9c871807a602b0808128 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 16 Jan 2024 16:16:07 -0700 Subject: [PATCH 68/75] Add CI script for k8s deployment test (#705) --- .gitea/workflows/test-k8s-deploy.yml | 15 ++++++--------- tests/scripts/install-kind.sh | 5 +++++ tests/scripts/install-kubectl.sh | 5 +++++ 3 files changed, 16 insertions(+), 9 deletions(-) create mode 100755 tests/scripts/install-kind.sh create mode 100755 tests/scripts/install-kubectl.sh diff --git a/.gitea/workflows/test-k8s-deploy.yml b/.gitea/workflows/test-k8s-deploy.yml index 84cce91a..2856d403 100644 --- a/.gitea/workflows/test-k8s-deploy.yml +++ b/.gitea/workflows/test-k8s-deploy.yml @@ -44,12 +44,9 @@ jobs: - name: Start dockerd # Also needed until we can incorporate into the executor run: | dockerd -H $DOCKER_HOST --userland-proxy=false & - sleep 5 - - name: "Install Go" - uses: actions/setup-go@v4 - with: - go-version: '1.21' - - name: "Install Kind" - run: go install sigs.k8s.io/kind@v0.20.0 - - name: "Debug Kind" - run: kind create cluster --retain && docker logs kind-control-plane + - name: "Install kind" + run: ./tests/scripts/install-kind.sh + - name: "Install Kubectl" + run: ./tests/scripts/install-kubectl.sh + - name: "Run k8s deployment test" + run: ./tests/k8s-deploy/run-deploy-test.sh diff --git a/tests/scripts/install-kind.sh b/tests/scripts/install-kind.sh new file mode 100755 index 00000000..254c3288 --- /dev/null +++ b/tests/scripts/install-kind.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# TODO: handle ARM +curl --silent -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 +chmod +x ./kind +mv ./kind /usr/local/bin diff --git a/tests/scripts/install-kubectl.sh b/tests/scripts/install-kubectl.sh new file mode 100755 index 00000000..7a5062fe --- /dev/null +++ b/tests/scripts/install-kubectl.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# TODO: handle ARM +curl --silent -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" +chmod +x ./kubectl +mv ./kubectl /usr/local/bin From 02c49d66f5b90cc41a2492e7ec5f6da7f84505e8 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 16 Jan 2024 17:07:03 -0700 Subject: [PATCH 69/75] Add debug output to check container --- .gitea/workflows/test-k8s-deploy.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitea/workflows/test-k8s-deploy.yml b/.gitea/workflows/test-k8s-deploy.yml index 2856d403..3e6d53ae 100644 --- a/.gitea/workflows/test-k8s-deploy.yml +++ b/.gitea/workflows/test-k8s-deploy.yml @@ -44,6 +44,8 @@ jobs: - name: Start dockerd # Also needed until we can incorporate into the executor run: | dockerd -H $DOCKER_HOST --userland-proxy=false & + - name: "Check container" + run: cat /usr/local/bin/entrypoint.sh - name: "Install kind" run: ./tests/scripts/install-kind.sh - name: "Install Kubectl" From c642e5d490c02a5bcf5f9e09e2faf110571fcaa1 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 16 Jan 2024 17:09:58 -0700 Subject: [PATCH 70/75] Try different runner --- .gitea/workflows/test-k8s-deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitea/workflows/test-k8s-deploy.yml b/.gitea/workflows/test-k8s-deploy.yml index 3e6d53ae..5d33f221 100644 --- a/.gitea/workflows/test-k8s-deploy.yml +++ b/.gitea/workflows/test-k8s-deploy.yml @@ -16,8 +16,8 @@ env: jobs: test: - name: "Run deploy test suite" - runs-on: ubuntu-latest + name: "Run deploy test suite on kind/k8s" + runs-on: ubuntu-22.04-with-syn-ethdb steps: - name: "Clone project repository" uses: actions/checkout@v3 From 9877cfaf85309f9174f91cad14b1f6e9d3014e75 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 16 Jan 2024 20:08:32 -0700 Subject: [PATCH 71/75] Update for new runner --- .gitea/workflows/test-k8s-deploy.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.gitea/workflows/test-k8s-deploy.yml b/.gitea/workflows/test-k8s-deploy.yml index 5d33f221..b3aabb02 100644 --- a/.gitea/workflows/test-k8s-deploy.yml +++ b/.gitea/workflows/test-k8s-deploy.yml @@ -10,10 +10,6 @@ on: paths-ignore: - '.gitea/workflows/triggers/*' -# Needed until we can incorporate docker startup into the executor container -env: - DOCKER_HOST: unix:///var/run/dind.sock - jobs: test: name: "Run deploy test suite on kind/k8s" @@ -41,11 +37,6 @@ jobs: run: ./scripts/create_build_tag_file.sh - name: "Build local shiv package" run: ./scripts/build_shiv_package.sh - - name: Start dockerd # Also needed until we can incorporate into the executor - run: | - dockerd -H $DOCKER_HOST --userland-proxy=false & - - name: "Check container" - run: cat /usr/local/bin/entrypoint.sh - name: "Install kind" run: ./tests/scripts/install-kind.sh - name: "Install Kubectl" From 635aa7037b267ff8a4ab25cad6c5cf827481b8d2 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 16 Jan 2024 21:15:21 -0700 Subject: [PATCH 72/75] Build test container --- tests/k8s-deploy/run-deploy-test.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/k8s-deploy/run-deploy-test.sh b/tests/k8s-deploy/run-deploy-test.sh index 5f1c0102..15eb2d3b 100755 --- a/tests/k8s-deploy/run-deploy-test.sh +++ b/tests/k8s-deploy/run-deploy-test.sh @@ -63,7 +63,9 @@ echo "Version reported is: ${reported_version_string}" echo "Cloning repositories into: $CERC_REPO_BASE_DIR" rm -rf $CERC_REPO_BASE_DIR mkdir -p $CERC_REPO_BASE_DIR -# Test basic stack-orchestrator deploy +$TEST_TARGET_SO --stack test setup-repositories +$TEST_TARGET_SO --stack test build-containers +# Test basic stack-orchestrator deploy to k8s test_deployment_dir=$CERC_REPO_BASE_DIR/test-deployment-dir test_deployment_spec=$CERC_REPO_BASE_DIR/test-deployment-spec.yml $TEST_TARGET_SO --stack test deploy --deploy-to k8s-kind init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED From 282e17556643ec684d1962b7e12f6f7a8c746c07 Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Wed, 17 Jan 2024 14:02:10 +0530 Subject: [PATCH 73/75] Remove unnecessary hyperlinks and pin image versions (#706) * Remove invalid dashboard and panel ids from alert rules * Pin grafana and prometheus versions * Configure custom grafana server URL --- .../data/compose/docker-compose-grafana.yml | 4 +- .../compose/docker-compose-prom-server.yml | 2 +- .../config/monitoring/watcher-alert-rules.yml | 40 ------------------- .../data/stacks/monitoring/README.md | 4 ++ .../stacks/monitoring/monitoring-watchers.md | 4 ++ 5 files changed, 12 insertions(+), 42 deletions(-) diff --git a/stack_orchestrator/data/compose/docker-compose-grafana.yml b/stack_orchestrator/data/compose/docker-compose-grafana.yml index 4aa8d1e1..6d32e37b 100644 --- a/stack_orchestrator/data/compose/docker-compose-grafana.yml +++ b/stack_orchestrator/data/compose/docker-compose-grafana.yml @@ -2,8 +2,10 @@ version: "3.7" services: grafana: - image: grafana/grafana + image: grafana/grafana:10.2.2 restart: always + environment: + GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL} volumes: - ../config/monitoring/grafana/provisioning:/etc/grafana/provisioning - ../config/monitoring/grafana/dashboards:/etc/grafana/dashboards diff --git a/stack_orchestrator/data/compose/docker-compose-prom-server.yml b/stack_orchestrator/data/compose/docker-compose-prom-server.yml index b085e295..9095b6dc 100644 --- a/stack_orchestrator/data/compose/docker-compose-prom-server.yml +++ b/stack_orchestrator/data/compose/docker-compose-prom-server.yml @@ -2,7 +2,7 @@ version: "3.7" services: prometheus: - image: prom/prometheus + image: prom/prometheus:v2.49.1 restart: always volumes: - ../config/monitoring/prometheus:/etc/prometheus diff --git a/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml b/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml index 7e26ba14..9df9472f 100644 --- a/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml +++ b/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml @@ -95,14 +95,10 @@ groups: maxDataPoints: 43200 refId: condition type: math - dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca - panelId: 24 noDataState: Alerting execErrState: Alerting for: 15m annotations: - __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca - __panelId__: "24" summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} isPaused: false - uid: censures_diff_external @@ -191,14 +187,10 @@ groups: maxDataPoints: 43200 refId: condition type: math - dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca - panelId: 24 noDataState: Alerting execErrState: Alerting for: 15m annotations: - __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca - __panelId__: "24" summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} isPaused: false - uid: claims_diff_external @@ -287,14 +279,10 @@ groups: maxDataPoints: 43200 refId: condition type: math - dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca - panelId: 24 noDataState: Alerting execErrState: Alerting for: 15m annotations: - __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca - __panelId__: "24" summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} isPaused: false - uid: conditional_star_release_diff_external @@ -383,14 +371,10 @@ groups: maxDataPoints: 43200 refId: condition type: math - dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca - panelId: 24 noDataState: Alerting execErrState: Alerting for: 15m annotations: - __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca - __panelId__: "24" summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} isPaused: false - uid: delegated_sending_diff_external @@ -479,14 +463,10 @@ groups: maxDataPoints: 43200 refId: condition type: math - dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca - panelId: 24 noDataState: Alerting execErrState: Alerting for: 15m annotations: - __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca - __panelId__: "24" summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} isPaused: false - uid: ecliptic_diff_external @@ -575,14 +555,10 @@ groups: maxDataPoints: 43200 refId: condition type: math - dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca - panelId: 24 noDataState: Alerting execErrState: Alerting for: 15m annotations: - __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca - __panelId__: "24" summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} isPaused: false - uid: linear_star_release_diff_external @@ -671,14 +647,10 @@ groups: maxDataPoints: 43200 refId: condition type: math - dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca - panelId: 24 noDataState: Alerting execErrState: Alerting for: 15m annotations: - __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca - __panelId__: "24" summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} isPaused: false - uid: polls_diff_external @@ -767,14 +739,10 @@ groups: maxDataPoints: 43200 refId: condition type: math - dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca - panelId: 24 noDataState: Alerting execErrState: Alerting for: 15m annotations: - __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca - __panelId__: "24" summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} isPaused: false @@ -865,14 +833,10 @@ groups: maxDataPoints: 43200 refId: condition type: math - dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca - panelId: 24 noDataState: Alerting execErrState: Alerting for: 15m annotations: - __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca - __panelId__: "24" summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} isPaused: false - uid: merkl_sushiswap_diff_external @@ -961,13 +925,9 @@ groups: maxDataPoints: 43200 refId: condition type: math - dashboardUid: cb9b746a-9abc-482e-9214-5231e0dd75ca - panelId: 24 noDataState: Alerting execErrState: Alerting for: 15m annotations: - __dashboardUid__: cb9b746a-9abc-482e-9214-5231e0dd75ca - __panelId__: "24" summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} isPaused: false diff --git a/stack_orchestrator/data/stacks/monitoring/README.md b/stack_orchestrator/data/stacks/monitoring/README.md index 7b410478..74b1bff8 100644 --- a/stack_orchestrator/data/stacks/monitoring/README.md +++ b/stack_orchestrator/data/stacks/monitoring/README.md @@ -140,6 +140,10 @@ Set the following env variables in the deployment env config file (`monitoring-d # External ETH RPC endpoint (filecoin) # (Optional, default: https://api.node.glif.io/rpc/v1) CERC_FIL_RPC_ENDPOINT= + + # Grafana server host URL (used in various links in alerts, etc.) + # (Optional, default: http://localhost:3000) + GF_SERVER_ROOT_URL= ``` ## Start the stack diff --git a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md index 3673b530..bce2d84b 100644 --- a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md +++ b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md @@ -130,6 +130,10 @@ Set the following env variables in the deployment env config file (`monitoring-w ```bash # Infura key to be used CERC_INFURA_KEY= + + # Grafana server host URL to be used + # (Optional, default: http://localhost:3000) + GF_SERVER_ROOT_URL= ``` ## Start the stack From 50509203d19003fe71b75fa95caa6d79f392a7d3 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Wed, 17 Jan 2024 08:26:51 -0700 Subject: [PATCH 74/75] Set k8s deploy test to manual trigger --- .gitea/workflows/test-k8s-deploy.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.gitea/workflows/test-k8s-deploy.yml b/.gitea/workflows/test-k8s-deploy.yml index b3aabb02..edd87177 100644 --- a/.gitea/workflows/test-k8s-deploy.yml +++ b/.gitea/workflows/test-k8s-deploy.yml @@ -1,14 +1,11 @@ name: K8s Deploy Test on: - pull_request: - branches: '*' push: - branches: - - main - - ci-test - paths-ignore: - - '.gitea/workflows/triggers/*' + branches: '*' + paths: + - '!**' + - '.gitea/workflows/triggers/fixturenet-laconicd-test' jobs: test: From b83465767d0975ddaff5f43a0889f6cf29303b4a Mon Sep 17 00:00:00 2001 From: David Boreham Date: Wed, 17 Jan 2024 08:27:09 -0700 Subject: [PATCH 75/75] Set k8s deploy test to manual trigger --- .gitea/workflows/triggers/test-k8s-deploy | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitea/workflows/triggers/test-k8s-deploy diff --git a/.gitea/workflows/triggers/test-k8s-deploy b/.gitea/workflows/triggers/test-k8s-deploy new file mode 100644 index 00000000..519ff4ac --- /dev/null +++ b/.gitea/workflows/triggers/test-k8s-deploy @@ -0,0 +1 @@ +Change this file to trigger running the test-k8s-deploy CI job