diff --git a/.gitea/workflows/test-k8s-deploy.yml b/.gitea/workflows/test-k8s-deploy.yml index 1835d9d4..93a445cf 100644 --- a/.gitea/workflows/test-k8s-deploy.yml +++ b/.gitea/workflows/test-k8s-deploy.yml @@ -1,23 +1,16 @@ name: K8s Deploy Test on: - pull_request: - branches: '*' push: - branches: - - main - - ci-test - paths-ignore: - - '.gitea/workflows/triggers/*' - -# Needed until we can incorporate docker startup into the executor container -env: - DOCKER_HOST: unix:///var/run/dind.sock + branches: '*' + paths: + - '!**' + - '.gitea/workflows/triggers/fixturenet-laconicd-test' jobs: test: - name: "Run deploy test suite" - runs-on: ubuntu-latest + name: "Run deploy test suite on kind/k8s" + runs-on: ubuntu-22.04-with-syn-ethdb steps: - name: "Clone project repository" uses: actions/checkout@v3 @@ -41,19 +34,10 @@ jobs: run: ./scripts/create_build_tag_file.sh - name: "Build local shiv package" run: ./scripts/build_shiv_package.sh - - name: Start dockerd # Also needed until we can incorporate into the executor - run: | - dockerd -H $DOCKER_HOST --userland-proxy=false & - sleep 5 - - name: "Install Go" - uses: actions/setup-go@v4 - with: - go-version: '1.21' - - name: "Install Kind" - run: go install sigs.k8s.io/kind@v0.20.0 - - name: "Debug Kind" - continue-on-error: true - run: kind create cluster --retain - - name: "Get Kind Logs" - continue-on-error: true - run: docker logs kind-control-plane + - name: "Install kind" + run: ./tests/scripts/install-kind.sh + - name: "Install Kubectl" + run: ./tests/scripts/install-kubectl.sh + - name: "Run k8s deployment test" + run: ./tests/k8s-deploy/run-deploy-test.sh + diff --git a/.gitea/workflows/test-webapp.yml b/.gitea/workflows/test-webapp.yml index 9fbf84b2..8b83c606 100644 --- a/.gitea/workflows/test-webapp.yml +++ b/.gitea/workflows/test-webapp.yml @@ -41,6 +41,8 @@ jobs: run: ./scripts/create_build_tag_file.sh - name: "Build local shiv package" run: ./scripts/build_shiv_package.sh + - name: "Install wget" # 20240109 - Only needed until the executors are updated. + run: apt update && apt install -y wget - name: Start dockerd # Also needed until we can incorporate into the executor run: | dockerd -H $DOCKER_HOST --userland-proxy=false & diff --git a/.gitea/workflows/triggers/test-k8s-deploy b/.gitea/workflows/triggers/test-k8s-deploy new file mode 100644 index 00000000..519ff4ac --- /dev/null +++ b/.gitea/workflows/triggers/test-k8s-deploy @@ -0,0 +1 @@ +Change this file to trigger running the test-k8s-deploy CI job diff --git a/docs/webapp.md b/docs/webapp.md new file mode 100644 index 00000000..fcf4ffcb --- /dev/null +++ b/docs/webapp.md @@ -0,0 +1,64 @@ +### Building and Running Webapps + +It is possible to build and run Next.js webapps using the `build-webapp` and `run-webapp` subcommands. + +To make it easier to build once and deploy into different environments and with different configuration, +compilation and static page generation are separated in the `build-webapp` and `run-webapp` steps. + +This offers much more flexibilty than standard Next.js build methods, since any environment variables accessed +via `process.env`, whether for pages or for API, will have values drawn from their runtime deployment environment, +not their build environment. + +## Building + +Building usually requires no additional configuration. By default, the Next.js version specified in `package.json` +is used, and either `yarn` or `npm` will be used automatically depending on which lock files are present. These +can be overidden with the build arguments `CERC_NEXT_VERSION` and `CERC_BUILD_TOOL` respectively. For example: `--extra-build-args "--build-arg CERC_NEXT_VERSION=13.4.12"` + +**Example**: +``` +$ cd ~/cerc +$ git clone git@git.vdb.to:cerc-io/test-progressive-web-app.git +$ laconic-so build-webapp --source-repo ~/cerc/test-progressive-web-app +... + +Built host container for ~/cerc/test-progressive-web-app with tag: + + cerc/test-progressive-web-app:local + +To test locally run: + + laconic-so run-webapp --image cerc/test-progressive-web-app:local --env-file /path/to/environment.env + +``` + +## Running + +With `run-webapp` a new container will be launched on the local machine, with runtime configuration provided by `--env-file` (if specified) and published on an available port. Multiple instances can be launched with different configuration. + +**Example**: +``` +# Production env +$ laconic-so run-webapp --image cerc/test-progressive-web-app:local --env-file /path/to/environment/production.env + +Image: cerc/test-progressive-web-app:local +ID: 4c6e893bf436b3e91a2b92ce37e30e499685131705700bd92a90d2eb14eefd05 +URL: http://localhost:32768 + +# Dev env +$ laconic-so run-webapp --image cerc/test-progressive-web-app:local --env-file /path/to/environment/dev.env + +Image: cerc/test-progressive-web-app:local +ID: 9ab96494f563aafb6c057d88df58f9eca81b90f8721a4e068493a289a976051c +URL: http://localhost:32769 +``` + +## Deploying + +Use the subcommand `deploy-webapp create` to make a deployment directory that can be subsequently deployed to a Kubernetes cluster. +Example commands are shown below, assuming that the webapp container image `cerc/test-progressive-web-app:local` has already been built: +``` +$ laconic-so deploy-webapp create --kube-config ~/kubectl/k8s-kubeconfig.yaml --image-registry registry.digitalocean.com/laconic-registry --deployment-dir webapp-k8s-deployment --image cerc/test-progressive-web-app:local --url https://test-pwa-app.hosting.laconic.com/ --env-file test-webapp.env +$ laconic-so deployment --dir webapp-k8s-deployment push-images +$ laconic-so deployment --dir webapp-k8s-deployment start +``` diff --git a/stack_orchestrator/build/build_containers.py b/stack_orchestrator/build/build_containers.py index 5b2748cc..e987c504 100644 --- a/stack_orchestrator/build/build_containers.py +++ b/stack_orchestrator/build/build_containers.py @@ -27,7 +27,7 @@ import subprocess import click import importlib.resources from pathlib import Path -from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config +from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external from stack_orchestrator.base import get_npm_registry_url # TODO: find a place for this @@ -58,7 +58,8 @@ def make_container_build_env(dev_root_path: str, return container_build_env -def process_container(container, +def process_container(stack: str, + container, container_build_dir: str, container_build_env: dict, dev_root_path: str, @@ -69,12 +70,29 @@ def process_container(container, ): if not quiet: print(f"Building: {container}") - build_dir = os.path.join(container_build_dir, container.replace("/", "-")) - build_script_filename = os.path.join(build_dir, "build.sh") + + default_container_tag = f"{container}:local" + container_build_env.update({"CERC_DEFAULT_CONTAINER_IMAGE_TAG": default_container_tag}) + + # Check if this is in an external stack + if stack_is_external(stack): + container_parent_dir = Path(stack).joinpath("container-build") + temp_build_dir = container_parent_dir.joinpath(container.replace("/", "-")) + temp_build_script_filename = temp_build_dir.joinpath("build.sh") + # Now check if the container exists in the external stack. + if not temp_build_script_filename.exists(): + # If not, revert to building an internal container + container_parent_dir = container_build_dir + else: + container_parent_dir = container_build_dir + + build_dir = container_parent_dir.joinpath(container.replace("/", "-")) + build_script_filename = build_dir.joinpath("build.sh") + if verbose: print(f"Build script filename: {build_script_filename}") if os.path.exists(build_script_filename): - build_command = build_script_filename + build_command = build_script_filename.as_posix() else: if verbose: print(f"No script file found: {build_script_filename}, using default build script") @@ -84,7 +102,7 @@ def process_container(container, repo_full_path = os.path.join(dev_root_path, repo_dir) repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir build_command = os.path.join(container_build_dir, - "default-build.sh") + f" {container}:local {repo_dir_or_build_dir}" + "default-build.sh") + f" {default_container_tag} {repo_dir_or_build_dir}" if not dry_run: if verbose: print(f"Executing: {build_command} with environment: {container_build_env}") @@ -158,7 +176,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args): for container in containers_in_scope: if include_exclude_check(container, include, exclude): - process_container(container, container_build_dir, container_build_env, + process_container(stack, container, container_build_dir, container_build_env, dev_root_path, quiet, verbose, dry_run, continue_on_error) else: if verbose: diff --git a/stack_orchestrator/build/build_webapp.py b/stack_orchestrator/build/build_webapp.py index f4668c5d..287347eb 100644 --- a/stack_orchestrator/build/build_webapp.py +++ b/stack_orchestrator/build/build_webapp.py @@ -32,8 +32,9 @@ from stack_orchestrator.build import build_containers @click.option('--source-repo', help="directory containing the webapp to build", required=True) @click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild") @click.option("--extra-build-args", help="Supply extra arguments to build") +@click.option("--tag", help="Container tag (default: cerc/:local)") @click.pass_context -def command(ctx, base_container, source_repo, force_rebuild, extra_build_args): +def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, tag): '''build the specified webapp container''' quiet = ctx.obj.quiet @@ -60,17 +61,21 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args): container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug, force_rebuild, extra_build_args) - build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet, + build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet, verbose, dry_run, continue_on_error) # Now build the target webapp. We use the same build script, but with a different Dockerfile and work dir. + container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true" container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"] = os.path.abspath(source_repo) container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = os.path.join(container_build_dir, base_container.replace("/", "-"), "Dockerfile.webapp") - webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1] - container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local" + if not tag: + webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1] + container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local" + else: + container_build_env["CERC_CONTAINER_BUILD_TAG"] = tag - build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet, + build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet, verbose, dry_run, continue_on_error) diff --git a/stack_orchestrator/constants.py b/stack_orchestrator/constants.py new file mode 100644 index 00000000..596b0c1b --- /dev/null +++ b/stack_orchestrator/constants.py @@ -0,0 +1,32 @@ +# Copyright © 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +cluster_name_prefix = "laconic-" +stack_file_name = "stack.yml" +spec_file_name = "spec.yml" +config_file_name = "config.env" +deployment_file_name = "deployment.yml" +compose_dir_name = "compose" +compose_deploy_type = "compose" +k8s_kind_deploy_type = "k8s-kind" +k8s_deploy_type = "k8s" +cluster_id_key = "cluster-id" +kube_config_key = "kube-config" +deploy_to_key = "deploy-to" +network_key = "network" +http_proxy_key = "http-proxy" +image_resigtry_key = "image-registry" +kind_config_filename = "kind-config.yml" +kube_config_filename = "kubeconfig.yml" diff --git a/stack_orchestrator/data/compose/docker-compose-fixturenet-laconic-console.yml b/stack_orchestrator/data/compose/docker-compose-fixturenet-laconic-console.yml index da2fd95f..a186e761 100644 --- a/stack_orchestrator/data/compose/docker-compose-fixturenet-laconic-console.yml +++ b/stack_orchestrator/data/compose/docker-compose-fixturenet-laconic-console.yml @@ -4,6 +4,6 @@ services: image: cerc/laconic-console-host:local environment: - CERC_WEBAPP_FILES_DIR=${CERC_WEBAPP_FILES_DIR:-/usr/local/share/.config/yarn/global/node_modules/@cerc-io/console-app/dist/production} - - LACONIC_HOSTED_ENDPOINT=${LACONIC_HOSTED_ENDPOINT:-http://localhost} + - LACONIC_HOSTED_ENDPOINT=${LACONIC_HOSTED_ENDPOINT:-http://localhost:9473} ports: - "80" diff --git a/stack_orchestrator/data/compose/docker-compose-fixturenet-laconicd.yml b/stack_orchestrator/data/compose/docker-compose-fixturenet-laconicd.yml index 641229d4..7b48f60d 100644 --- a/stack_orchestrator/data/compose/docker-compose-fixturenet-laconicd.yml +++ b/stack_orchestrator/data/compose/docker-compose-fixturenet-laconicd.yml @@ -5,7 +5,7 @@ services: command: ["sh", "/docker-entrypoint-scripts.d/create-fixturenet.sh"] volumes: # The cosmos-sdk node's database directory: - - laconicd-data:/root/.laconicd/data + - laconicd-data:/root/.laconicd # TODO: look at folding these scripts into the container - ../config/fixturenet-laconicd/create-fixturenet.sh:/docker-entrypoint-scripts.d/create-fixturenet.sh - ../config/fixturenet-laconicd/export-mykey.sh:/docker-entrypoint-scripts.d/export-mykey.sh diff --git a/stack_orchestrator/data/compose/docker-compose-fixturenet-optimism.yml b/stack_orchestrator/data/compose/docker-compose-fixturenet-optimism.yml index ddf7e290..fe1eac50 100644 --- a/stack_orchestrator/data/compose/docker-compose-fixturenet-optimism.yml +++ b/stack_orchestrator/data/compose/docker-compose-fixturenet-optimism.yml @@ -6,8 +6,8 @@ services: # Deploys the L1 smart contracts (outputs to volume l1_deployment) fixturenet-optimism-contracts: restart: on-failure - hostname: fixturenet-optimism-contracts image: cerc/optimism-contracts:local + hostname: fixturenet-optimism-contracts env_file: - ../config/fixturenet-optimism/l1-params.env environment: @@ -17,27 +17,49 @@ services: CERC_L1_ACCOUNTS_CSV_URL: ${CERC_L1_ACCOUNTS_CSV_URL} CERC_L1_ADDRESS: ${CERC_L1_ADDRESS} CERC_L1_PRIV_KEY: ${CERC_L1_PRIV_KEY} - CERC_L1_ADDRESS_2: ${CERC_L1_ADDRESS_2} - CERC_L1_PRIV_KEY_2: ${CERC_L1_PRIV_KEY_2} - # Waits for L1 endpoint to be up before running the script - command: | - "./wait-for-it.sh -h ${CERC_L1_HOST:-$${DEFAULT_CERC_L1_HOST}} -p ${CERC_L1_PORT:-$${DEFAULT_CERC_L1_PORT}} -s -t 60 -- ./run.sh" volumes: - ../config/network/wait-for-it.sh:/app/packages/contracts-bedrock/wait-for-it.sh - - ../config/optimism-contracts/hardhat-tasks/verify-contract-deployment.ts:/app/packages/contracts-bedrock/tasks/verify-contract-deployment.ts - - ../config/optimism-contracts/hardhat-tasks/rekey-json.ts:/app/packages/contracts-bedrock/tasks/rekey-json.ts - - ../config/optimism-contracts/hardhat-tasks/send-balance.ts:/app/packages/contracts-bedrock/tasks/send-balance.ts - - ../config/fixturenet-optimism/optimism-contracts/update-config.js:/app/packages/contracts-bedrock/update-config.js - - ../config/fixturenet-optimism/optimism-contracts/run.sh:/app/packages/contracts-bedrock/run.sh + - ../config/fixturenet-optimism/optimism-contracts/deploy-contracts.sh:/app/packages/contracts-bedrock/deploy-contracts.sh - l2_accounts:/l2-accounts - - l1_deployment:/app/packages/contracts-bedrock + - l1_deployment:/l1-deployment + - l2_config:/l2-config + # Waits for L1 endpoint to be up before running the contract deploy script + command: | + "./wait-for-it.sh -h ${CERC_L1_HOST:-$${DEFAULT_CERC_L1_HOST}} -p ${CERC_L1_PORT:-$${DEFAULT_CERC_L1_PORT}} -s -t 60 -- ./deploy-contracts.sh" + + # Initializes and runs the L2 execution client (outputs to volume l2_geth_data) + op-geth: + restart: always + image: cerc/optimism-l2geth:local + hostname: op-geth + depends_on: + op-node: + condition: service_started + volumes: + - ../config/fixturenet-optimism/run-op-geth.sh:/run-op-geth.sh + - l2_config:/l2-config:ro + - l2_accounts:/l2-accounts:ro + - l2_geth_data:/datadir + entrypoint: "sh" + command: "/run-op-geth.sh" + ports: + - "8545" + - "8546" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost:8545"] + interval: 30s + timeout: 10s + retries: 100 + start_period: 10s extra_hosts: - "host.docker.internal:host-gateway" - # Generates the config files required for L2 (outputs to volume l2_config) - op-node-l2-config-gen: - restart: on-failure + # Runs the L2 consensus client (Sequencer node) + # Generates the L2 config files if not already present (outputs to volume l2_config) + op-node: + restart: always image: cerc/optimism-op-node:local + hostname: op-node depends_on: fixturenet-optimism-contracts: condition: service_completed_successfully @@ -47,61 +69,19 @@ services: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_L1_RPC: ${CERC_L1_RPC} volumes: - - ../config/fixturenet-optimism/generate-l2-config.sh:/app/generate-l2-config.sh - - l1_deployment:/contracts-bedrock:ro - - l2_config:/app - command: ["sh", "/app/generate-l2-config.sh"] - extra_hosts: - - "host.docker.internal:host-gateway" - - # Initializes and runs the L2 execution client (outputs to volume l2_geth_data) - op-geth: - restart: always - image: cerc/optimism-l2geth:local - depends_on: - op-node-l2-config-gen: - condition: service_started - volumes: - - ../config/fixturenet-optimism/run-op-geth.sh:/run-op-geth.sh - - l2_config:/op-node:ro + - ../config/fixturenet-optimism/run-op-node.sh:/run-op-node.sh + - l1_deployment:/l1-deployment:ro + - l2_config:/l2-config - l2_accounts:/l2-accounts:ro - - l2_geth_data:/datadir entrypoint: "sh" - command: "/run-op-geth.sh" + command: "/run-op-node.sh" ports: - - "0.0.0.0:8545:8545" - - "0.0.0.0:8546:8546" - healthcheck: - test: ["CMD", "nc", "-vz", "localhost:8545"] - interval: 30s - timeout: 10s - retries: 10 - start_period: 10s - - # Runs the L2 consensus client (Sequencer node) - op-node: - restart: always - image: cerc/optimism-op-node:local - depends_on: - op-geth: - condition: service_healthy - env_file: - - ../config/fixturenet-optimism/l1-params.env - environment: - CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} - CERC_L1_RPC: ${CERC_L1_RPC} - volumes: - - ../config/fixturenet-optimism/run-op-node.sh:/app/run-op-node.sh - - l2_config:/op-node-data:ro - - l2_accounts:/l2-accounts:ro - command: ["sh", "/app/run-op-node.sh"] - ports: - - "0.0.0.0:8547:8547" + - "8547" healthcheck: test: ["CMD", "nc", "-vz", "localhost:8547"] interval: 30s timeout: 10s - retries: 10 + retries: 100 start_period: 10s extra_hosts: - "host.docker.internal:host-gateway" @@ -110,6 +90,7 @@ services: op-batcher: restart: always image: cerc/optimism-op-batcher:local + hostname: op-batcher depends_on: op-node: condition: service_healthy @@ -129,7 +110,7 @@ services: command: | "/wait-for-it.sh -h ${CERC_L1_HOST:-$${DEFAULT_CERC_L1_HOST}} -p ${CERC_L1_PORT:-$${DEFAULT_CERC_L1_PORT}} -s -t 60 -- /run-op-batcher.sh" ports: - - "127.0.0.1:8548:8548" + - "8548" extra_hosts: - "host.docker.internal:host-gateway" @@ -137,25 +118,29 @@ services: op-proposer: restart: always image: cerc/optimism-op-proposer:local + hostname: op-proposer depends_on: op-node: condition: service_healthy + op-geth: + condition: service_healthy env_file: - ../config/fixturenet-optimism/l1-params.env environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_L1_RPC: ${CERC_L1_RPC} + CERC_L1_CHAIN_ID: ${CERC_L1_CHAIN_ID} volumes: - ../config/network/wait-for-it.sh:/wait-for-it.sh - ../config/fixturenet-optimism/run-op-proposer.sh:/run-op-proposer.sh - - l1_deployment:/contracts-bedrock:ro + - l1_deployment:/l1-deployment:ro - l2_accounts:/l2-accounts:ro entrypoint: ["sh", "-c"] # Waits for L1 endpoint to be up before running the proposer command: | "/wait-for-it.sh -h ${CERC_L1_HOST:-$${DEFAULT_CERC_L1_HOST}} -p ${CERC_L1_PORT:-$${DEFAULT_CERC_L1_PORT}} -s -t 60 -- /run-op-proposer.sh" ports: - - "127.0.0.1:8560:8560" + - "8560" extra_hosts: - "host.docker.internal:host-gateway" diff --git a/stack_orchestrator/data/compose/docker-compose-fixturenet-urbit.yml b/stack_orchestrator/data/compose/docker-compose-fixturenet-urbit.yml new file mode 100644 index 00000000..bc64cd90 --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-fixturenet-urbit.yml @@ -0,0 +1,36 @@ +version: '3.7' + +services: + # Runs an Urbit fake ship and attempts an app installation using given data + # Uploads the app glob to given IPFS endpoint + # From urbit_app_builds volume: + # - takes app build from ${CERC_URBIT_APP}/build (waits for it to appear) + # - takes additional mark files from ${CERC_URBIT_APP}/mar + # - takes the docket file from ${CERC_URBIT_APP}/desk.docket-0 + urbit-fake-ship: + restart: unless-stopped + image: tloncorp/vere + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_URBIT_APP: ${CERC_URBIT_APP} + CERC_ENABLE_APP_INSTALL: ${CERC_ENABLE_APP_INSTALL:-true} + CERC_IPFS_GLOB_HOST_ENDPOINT: ${CERC_IPFS_GLOB_HOST_ENDPOINT:-http://ipfs:5001} + CERC_IPFS_SERVER_ENDPOINT: ${CERC_IPFS_SERVER_ENDPOINT:-http://ipfs:8080} + entrypoint: ["bash", "-c", "./run-urbit-ship.sh && ./deploy-app.sh && tail -f /dev/null"] + volumes: + - urbit_data:/urbit + - urbit_app_builds:/app-builds + - ../config/urbit/run-urbit-ship.sh:/urbit/run-urbit-ship.sh + - ../config/urbit/deploy-app.sh:/urbit/deploy-app.sh + ports: + - "80" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "80"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 10s + +volumes: + urbit_data: + urbit_app_builds: diff --git a/stack_orchestrator/data/compose/docker-compose-grafana.yml b/stack_orchestrator/data/compose/docker-compose-grafana.yml new file mode 100644 index 00000000..6d32e37b --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-grafana.yml @@ -0,0 +1,23 @@ +version: "3.7" + +services: + grafana: + image: grafana/grafana:10.2.2 + restart: always + environment: + GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL} + volumes: + - ../config/monitoring/grafana/provisioning:/etc/grafana/provisioning + - ../config/monitoring/grafana/dashboards:/etc/grafana/dashboards + - grafana_storage:/var/lib/grafana + ports: + - "3000" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "3000"] + interval: 30s + timeout: 5s + retries: 10 + start_period: 3s + +volumes: + grafana_storage: diff --git a/stack_orchestrator/data/compose/docker-compose-kubo.yml b/stack_orchestrator/data/compose/docker-compose-kubo.yml index f5f8b06e..b50dd15a 100644 --- a/stack_orchestrator/data/compose/docker-compose-kubo.yml +++ b/stack_orchestrator/data/compose/docker-compose-kubo.yml @@ -1,13 +1,24 @@ version: "3.2" + # See: https://docs.ipfs.tech/install/run-ipfs-inside-docker/#set-up services: ipfs: - image: ipfs/kubo:master-2023-02-20-714a968 + image: ipfs/kubo:v0.24.0 restart: always volumes: - - ./ipfs/import:/import - - ./ipfs/data:/data/ipfs + - ipfs-import:/import + - ipfs-data:/data/ipfs ports: - - "0.0.0.0:8080:8080" - - "0.0.0.0:4001:4001" + - "4001" + - "8080" - "0.0.0.0:5001:5001" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "5001"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 10s + +volumes: + ipfs-import: + ipfs-data: diff --git a/stack_orchestrator/data/compose/docker-compose-node-exporter.yml b/stack_orchestrator/data/compose/docker-compose-node-exporter.yml new file mode 100644 index 00000000..6fa02d6d --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-node-exporter.yml @@ -0,0 +1,15 @@ +version: '3.8' + +services: + node-exporter: + image: prom/node-exporter:latest + restart: unless-stopped + command: + - '--web.listen-address=:9100' + - '--path.rootfs=/host' + - '--collector.systemd' + - '--collector.processes' + network_mode: host + pid: host + volumes: + - '/:/host:ro,rslave' diff --git a/stack_orchestrator/data/compose/docker-compose-osmosis-front-end-urbit.yml b/stack_orchestrator/data/compose/docker-compose-osmosis-front-end-urbit.yml new file mode 100644 index 00000000..5012de13 --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-osmosis-front-end-urbit.yml @@ -0,0 +1,22 @@ +version: "3.2" + +services: + osmosis-front-end: + image: cerc/osmosis-front-end-urbit:local + restart: on-failure + environment: + - NEXT_PUBLIC_WEB_API_BASE_URL=${CERC_WEB_API_BASE_URL} + - ASSET_LIST_COMMIT_HASH=a326bcefc51372b4912be5a2a2fa84a5d142a438 + - NEXT_PUBLIC_BASEPATH=/apps/osmosis + - NEXT_PUBLIC_URBIT_DEPLOYMENT=true + working_dir: /app/packages/web + command: ["./build-app-for-urbit.sh"] + volumes: + - ../config/osmosis/build-app-for-urbit.sh:/app/packages/web/build-app-for-urbit.sh + - ../config/osmosis/.env.production:/app/packages/web/.env.production + - urbit_app_builds:/app-builds + - ../config/osmosis/urbit-files/mar:/app/packages/web/mar + - ../config/osmosis/urbit-files/desk.docket-0:/app/packages/web/desk.docket-0 + +volumes: + urbit_app_builds: diff --git a/stack_orchestrator/data/compose/docker-compose-osmosis-front-end.yml b/stack_orchestrator/data/compose/docker-compose-osmosis-front-end.yml new file mode 100644 index 00000000..4178383e --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-osmosis-front-end.yml @@ -0,0 +1,36 @@ +version: "3.2" + +services: + osmosis-front-end: + image: cerc/osmosis-front-end:local + restart: on-failure + environment: + - NEXT_PUBLIC_WEB_API_BASE_URL=${CERC_WEB_API_BASE_URL} + - ASSET_LIST_COMMIT_HASH=a326bcefc51372b4912be5a2a2fa84a5d142a438 + working_dir: /app/packages/web + command: ["./build-app.sh"] + volumes: + - ../config/osmosis/build-app.sh:/app/packages/web/build-app.sh + - ../config/osmosis/.env.production:/app/packages/web/.env.production + - app_builds:/app-builds + + nginx: + image: nginx:1.23-alpine + restart: always + depends_on: + osmosis-front-end: + condition: service_completed_successfully + volumes: + - ../config/osmosis/nginx:/etc/nginx/conf.d + - app_builds:/usr/share/nginx + ports: + - "80" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "80"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + +volumes: + app_builds: diff --git a/stack_orchestrator/data/compose/docker-compose-prom-server.yml b/stack_orchestrator/data/compose/docker-compose-prom-server.yml new file mode 100644 index 00000000..9095b6dc --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-prom-server.yml @@ -0,0 +1,57 @@ +version: "3.7" + +services: + prometheus: + image: prom/prometheus:v2.49.1 + restart: always + volumes: + - ../config/monitoring/prometheus:/etc/prometheus + - prometheus_data:/prometheus + ports: + - "9090" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "9090"] + interval: 30s + timeout: 5s + retries: 10 + start_period: 3s + extra_hosts: + - "host.docker.internal:host-gateway" + + blackbox: + image: prom/blackbox-exporter:latest + restart: always + volumes: + - ../config/monitoring/blackbox.yml:/etc/blackbox_exporter/config.yml + ports: + - '9115' + extra_hosts: + - "host.docker.internal:host-gateway" + + chain-head-exporter: + image: cerc/watcher-ts:local + restart: always + working_dir: /app/packages/cli + environment: + ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT} + FIL_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT} + ETH_RPC_API_KEY: ${CERC_INFURA_KEY} + PORT: ${CERC_METRICS_PORT} + command: ["sh", "-c", "yarn export-metrics:chain-heads"] + ports: + - '5000' + extra_hosts: + - "host.docker.internal:host-gateway" + + postgres-exporter: + image: quay.io/prometheuscommunity/postgres-exporter + restart: always + volumes: + - ../config/monitoring/postgres-exporter.yml:/postgres_exporter.yml + ports: + - '9187' + extra_hosts: + - "host.docker.internal:host-gateway" + +volumes: + prometheus_data: diff --git a/stack_orchestrator/data/compose/docker-compose-proxy-server.yml b/stack_orchestrator/data/compose/docker-compose-proxy-server.yml new file mode 100644 index 00000000..dc672e0a --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-proxy-server.yml @@ -0,0 +1,22 @@ +version: "3.2" + +services: + proxy-server: + image: cerc/watcher-ts:local + restart: on-failure + working_dir: /app/packages/cli + environment: + ENABLE_PROXY: ${CERC_ENABLE_PROXY:-true} + PROXY_UPSTREAM: ${CERC_PROXY_UPSTREAM} + PROXY_ORIGIN_HEADER: ${CERC_PROXY_ORIGIN_HEADER} + command: ["sh", "-c", "./run.sh"] + volumes: + - ../config/proxy-server/run.sh:/app/packages/cli/run.sh + ports: + - "4000" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "4000"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 10s diff --git a/stack_orchestrator/data/compose/docker-compose-uniswap-interface.yml b/stack_orchestrator/data/compose/docker-compose-uniswap-interface.yml new file mode 100644 index 00000000..334ab0cd --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-uniswap-interface.yml @@ -0,0 +1,18 @@ +version: "3.2" + +services: + uniswap-interface: + image: cerc/uniswap-interface:local + restart: on-failure + environment: + - REACT_APP_INFURA_KEY=${CERC_INFURA_KEY} + - REACT_APP_AWS_API_ENDPOINT=${CERC_UNISWAP_GQL} + command: ["./build-app.sh"] + volumes: + - ../config/uniswap-interface/build-app.sh:/app/build-app.sh + - urbit_app_builds:/app-builds + - ../config/uniswap-interface/urbit-files/mar:/app/mar + - ../config/uniswap-interface/urbit-files/desk.docket-0:/app/desk.docket-0 + +volumes: + urbit_app_builds: diff --git a/stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml b/stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml index 327c77fc..48e77082 100644 --- a/stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml +++ b/stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml @@ -10,6 +10,7 @@ services: - POSTGRES_MULTIPLE_DATABASES=azimuth-watcher,azimuth-watcher-job-queue,censures-watcher,censures-watcher-job-queue,claims-watcher,claims-watcher-job-queue,conditional-star-release-watcher,conditional-star-release-watcher-job-queue,delegated-sending-watcher,delegated-sending-watcher-job-queue,ecliptic-watcher,ecliptic-watcher-job-queue,linear-star-release-watcher,linear-star-release-watcher-job-queue,polls-watcher,polls-watcher-job-queue - POSTGRES_EXTENSION=azimuth-watcher-job-queue:pgcrypto,censures-watcher-job-queue:pgcrypto,claims-watcher-job-queue:pgcrypto,conditional-star-release-watcher-job-queue:pgcrypto,delegated-sending-watcher-job-queue:pgcrypto,ecliptic-watcher-job-queue:pgcrypto,linear-star-release-watcher-job-queue:pgcrypto,polls-watcher-job-queue:pgcrypto, - POSTGRES_PASSWORD=password + command: ["postgres", "-c", "max_connections=200"] volumes: - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh - watcher_db_data:/var/lib/postgresql/data @@ -22,6 +23,38 @@ services: retries: 15 start_period: 10s + # Starts the azimuth-watcher job runner + azimuth-watcher-job-runner: + image: cerc/watcher-azimuth:local + restart: unless-stopped + depends_on: + watcher-db: + condition: service_healthy + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CERC_HISTORICAL_BLOCK_RANGE: 500 + CONTRACT_ADDRESS: 0x223c067F8CF28ae173EE5CafEa60cA44C335fecB + CONTRACT_NAME: Azimuth + STARTING_BLOCK: 6784880 + working_dir: /app/packages/azimuth-watcher + command: "./start-job-runner.sh" + volumes: + - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/azimuth-watcher/environments/watcher-config-template.toml + - ../config/watcher-azimuth/merge-toml.js:/app/packages/azimuth-watcher/merge-toml.js + - ../config/watcher-azimuth/start-job-runner.sh:/app/packages/azimuth-watcher/start-job-runner.sh + ports: + - "9000" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "9000"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + # Starts the azimuth-watcher server azimuth-watcher-server: image: cerc/watcher-azimuth:local @@ -29,8 +62,8 @@ services: depends_on: watcher-db: condition: service_healthy - env_file: - - ../config/watcher-azimuth/watcher-params.env + azimuth-watcher-job-runner: + condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} @@ -52,6 +85,37 @@ services: extra_hosts: - "host.docker.internal:host-gateway" + # Starts the censures-watcher job runner + censures-watcher-job-runner: + image: cerc/watcher-azimuth:local + restart: unless-stopped + depends_on: + watcher-db: + condition: service_healthy + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CONTRACT_ADDRESS: 0x325f68d32BdEe6Ed86E7235ff2480e2A433D6189 + CONTRACT_NAME: Censures + STARTING_BLOCK: 6784954 + working_dir: /app/packages/censures-watcher + command: "./start-job-runner.sh" + volumes: + - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/censures-watcher/environments/watcher-config-template.toml + - ../config/watcher-azimuth/merge-toml.js:/app/packages/censures-watcher/merge-toml.js + - ../config/watcher-azimuth/start-job-runner.sh:/app/packages/censures-watcher/start-job-runner.sh + ports: + - "9002" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "9002"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + # Starts the censures-watcher server censures-watcher-server: image: cerc/watcher-azimuth:local @@ -59,8 +123,8 @@ services: depends_on: watcher-db: condition: service_healthy - env_file: - - ../config/watcher-azimuth/watcher-params.env + censures-watcher-job-runner: + condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} @@ -82,6 +146,37 @@ services: extra_hosts: - "host.docker.internal:host-gateway" + # Starts the claims-watcher job runner + claims-watcher-job-runner: + image: cerc/watcher-azimuth:local + restart: unless-stopped + depends_on: + watcher-db: + condition: service_healthy + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CONTRACT_ADDRESS: 0xe7e7f69b34D7d9Bd8d61Fb22C33b22708947971A + CONTRACT_NAME: Claims + STARTING_BLOCK: 6784941 + working_dir: /app/packages/claims-watcher + command: "./start-job-runner.sh" + volumes: + - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/claims-watcher/environments/watcher-config-template.toml + - ../config/watcher-azimuth/merge-toml.js:/app/packages/claims-watcher/merge-toml.js + - ../config/watcher-azimuth/start-job-runner.sh:/app/packages/claims-watcher/start-job-runner.sh + ports: + - "9004" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "9004"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + # Starts the claims-watcher server claims-watcher-server: image: cerc/watcher-azimuth:local @@ -89,8 +184,8 @@ services: depends_on: watcher-db: condition: service_healthy - env_file: - - ../config/watcher-azimuth/watcher-params.env + claims-watcher-job-runner: + condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} @@ -112,6 +207,37 @@ services: extra_hosts: - "host.docker.internal:host-gateway" + # Starts the conditional-star-release-watcher job runner + conditional-star-release-watcher-job-runner: + image: cerc/watcher-azimuth:local + restart: unless-stopped + depends_on: + watcher-db: + condition: service_healthy + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CONTRACT_ADDRESS: 0x8C241098C3D3498Fe1261421633FD57986D74AeA + CONTRACT_NAME: ConditionalStarRelease + STARTING_BLOCK: 6828004 + working_dir: /app/packages/conditional-star-release-watcher + command: "./start-job-runner.sh" + volumes: + - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/conditional-star-release-watcher/environments/watcher-config-template.toml + - ../config/watcher-azimuth/merge-toml.js:/app/packages/conditional-star-release-watcher/merge-toml.js + - ../config/watcher-azimuth/start-job-runner.sh:/app/packages/conditional-star-release-watcher/start-job-runner.sh + ports: + - "9006" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "9006"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + # Starts the conditional-star-release-watcher server conditional-star-release-watcher-server: image: cerc/watcher-azimuth:local @@ -119,8 +245,8 @@ services: depends_on: watcher-db: condition: service_healthy - env_file: - - ../config/watcher-azimuth/watcher-params.env + conditional-star-release-watcher-job-runner: + condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} @@ -142,6 +268,37 @@ services: extra_hosts: - "host.docker.internal:host-gateway" + # Starts the delegated-sending-watcher job runner + delegated-sending-watcher-job-runner: + image: cerc/watcher-azimuth:local + restart: unless-stopped + depends_on: + watcher-db: + condition: service_healthy + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CONTRACT_ADDRESS: 0xf6b461fE1aD4bd2ce25B23Fe0aff2ac19B3dFA76 + CONTRACT_NAME: DelegatedSending + STARTING_BLOCK: 6784956 + working_dir: /app/packages/delegated-sending-watcher + command: "./start-job-runner.sh" + volumes: + - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/delegated-sending-watcher/environments/watcher-config-template.toml + - ../config/watcher-azimuth/merge-toml.js:/app/packages/delegated-sending-watcher/merge-toml.js + - ../config/watcher-azimuth/start-job-runner.sh:/app/packages/delegated-sending-watcher/start-job-runner.sh + ports: + - "9008" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "9008"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + # Starts the delegated-sending-watcher server delegated-sending-watcher-server: image: cerc/watcher-azimuth:local @@ -149,8 +306,8 @@ services: depends_on: watcher-db: condition: service_healthy - env_file: - - ../config/watcher-azimuth/watcher-params.env + delegated-sending-watcher-job-runner: + condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} @@ -172,6 +329,37 @@ services: extra_hosts: - "host.docker.internal:host-gateway" + # Starts the ecliptic-watcher job runner + ecliptic-watcher-job-runner: + image: cerc/watcher-azimuth:local + restart: unless-stopped + depends_on: + watcher-db: + condition: service_healthy + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CONTRACT_ADDRESS: 0x33EeCbf908478C10614626A9D304bfe18B78DD73 + CONTRACT_NAME: Ecliptic + STARTING_BLOCK: 13692129 + working_dir: /app/packages/ecliptic-watcher + command: "./start-job-runner.sh" + volumes: + - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/ecliptic-watcher/environments/watcher-config-template.toml + - ../config/watcher-azimuth/merge-toml.js:/app/packages/ecliptic-watcher/merge-toml.js + - ../config/watcher-azimuth/start-job-runner.sh:/app/packages/ecliptic-watcher/start-job-runner.sh + ports: + - "9010" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "9010"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + # Starts the ecliptic-watcher server ecliptic-watcher-server: image: cerc/watcher-azimuth:local @@ -179,8 +367,8 @@ services: depends_on: watcher-db: condition: service_healthy - env_file: - - ../config/watcher-azimuth/watcher-params.env + ecliptic-watcher-job-runner: + condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} @@ -202,6 +390,37 @@ services: extra_hosts: - "host.docker.internal:host-gateway" + # Starts the linear-star-release-watcher job runner + linear-star-release-watcher-job-runner: + image: cerc/watcher-azimuth:local + restart: unless-stopped + depends_on: + watcher-db: + condition: service_healthy + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CONTRACT_ADDRESS: 0x86cd9cd0992F04231751E3761De45cEceA5d1801 + CONTRACT_NAME: LinearStarRelease + STARTING_BLOCK: 6784943 + working_dir: /app/packages/linear-star-release-watcher + command: "./start-job-runner.sh" + volumes: + - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/linear-star-release-watcher/environments/watcher-config-template.toml + - ../config/watcher-azimuth/merge-toml.js:/app/packages/linear-star-release-watcher/merge-toml.js + - ../config/watcher-azimuth/start-job-runner.sh:/app/packages/linear-star-release-watcher/start-job-runner.sh + ports: + - "9012" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "9012"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + # Starts the linear-star-release-watcher server linear-star-release-watcher-server: image: cerc/watcher-azimuth:local @@ -209,8 +428,8 @@ services: depends_on: watcher-db: condition: service_healthy - env_file: - - ../config/watcher-azimuth/watcher-params.env + linear-star-release-watcher-job-runner: + condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} @@ -232,6 +451,37 @@ services: extra_hosts: - "host.docker.internal:host-gateway" + # Starts the polls-watcher job runner + polls-watcher-job-runner: + image: cerc/watcher-azimuth:local + restart: unless-stopped + depends_on: + watcher-db: + condition: service_healthy + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} + CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL} + CONTRACT_ADDRESS: 0x7fEcaB617c868Bb5996d99D95200D2Fa708218e4 + CONTRACT_NAME: Polls + STARTING_BLOCK: 6784912 + working_dir: /app/packages/polls-watcher + command: "./start-job-runner.sh" + volumes: + - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/polls-watcher/environments/watcher-config-template.toml + - ../config/watcher-azimuth/merge-toml.js:/app/packages/polls-watcher/merge-toml.js + - ../config/watcher-azimuth/start-job-runner.sh:/app/packages/polls-watcher/start-job-runner.sh + ports: + - "9014" + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "9014"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + # Starts the polls-watcher server polls-watcher-server: image: cerc/watcher-azimuth:local @@ -239,8 +489,8 @@ services: depends_on: watcher-db: condition: service_healthy - env_file: - - ../config/watcher-azimuth/watcher-params.env + polls-watcher-job-runner: + condition: service_healthy environment: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC} diff --git a/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml b/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml new file mode 100644 index 00000000..aae7bb47 --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml @@ -0,0 +1,76 @@ +version: '3.2' + +services: + merkl-sushiswap-v3-watcher-db: + restart: unless-stopped + image: postgres:14-alpine + environment: + - POSTGRES_USER=vdbm + - POSTGRES_MULTIPLE_DATABASES=merkl-sushiswap-v3-watcher,merkl-sushiswap-v3-watcher-job-queue + - POSTGRES_EXTENSION=merkl-sushiswap-v3-watcher-job-queue:pgcrypto + - POSTGRES_PASSWORD=password + volumes: + - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh + - merkl_sushiswap_v3_watcher_db_data:/var/lib/postgresql/data + ports: + - "5432" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "5432"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 10s + + merkl-sushiswap-v3-watcher-job-runner: + restart: unless-stopped + depends_on: + merkl-sushiswap-v3-watcher-db: + condition: service_healthy + image: cerc/watcher-merkl-sushiswap-v3:local + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT} + command: ["bash", "./start-job-runner.sh"] + volumes: + - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml + - ../config/watcher-merkl-sushiswap-v3/start-job-runner.sh:/app/start-job-runner.sh + ports: + - "9002:9000" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "9000"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + merkl-sushiswap-v3-watcher-server: + restart: unless-stopped + depends_on: + merkl-sushiswap-v3-watcher-db: + condition: service_healthy + merkl-sushiswap-v3-watcher-job-runner: + condition: service_healthy + image: cerc/watcher-merkl-sushiswap-v3:local + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT} + command: ["bash", "./start-server.sh"] + volumes: + - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml + - ../config/watcher-merkl-sushiswap-v3/start-server.sh:/app/start-server.sh + ports: + - "127.0.0.1:3007:3008" + - "9003:9001" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "3008"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + +volumes: + merkl_sushiswap_v3_watcher_db_data: diff --git a/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml b/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml new file mode 100644 index 00000000..6c39320c --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml @@ -0,0 +1,76 @@ +version: '3.2' + +services: + sushiswap-v3-watcher-db: + restart: unless-stopped + image: postgres:14-alpine + environment: + - POSTGRES_USER=vdbm + - POSTGRES_MULTIPLE_DATABASES=sushiswap-v3-watcher,sushiswap-v3-watcher-job-queue + - POSTGRES_EXTENSION=sushiswap-v3-watcher-job-queue:pgcrypto + - POSTGRES_PASSWORD=password + volumes: + - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh + - sushiswap_v3_watcher_db_data:/var/lib/postgresql/data + ports: + - "5432" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "5432"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 10s + + sushiswap-v3-watcher-job-runner: + restart: unless-stopped + depends_on: + sushiswap-v3-watcher-db: + condition: service_healthy + image: cerc/watcher-sushiswap-v3:local + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT} + command: ["bash", "./start-job-runner.sh"] + volumes: + - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml + - ../config/watcher-sushiswap-v3/start-job-runner.sh:/app/start-job-runner.sh + ports: + - "9000:9000" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "9000"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + + sushiswap-v3-watcher-server: + restart: unless-stopped + depends_on: + sushiswap-v3-watcher-db: + condition: service_healthy + sushiswap-v3-watcher-job-runner: + condition: service_healthy + image: cerc/watcher-sushiswap-v3:local + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT} + command: ["bash", "./start-server.sh"] + volumes: + - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml + - ../config/watcher-sushiswap-v3/start-server.sh:/app/start-server.sh + ports: + - "127.0.0.1:3008:3008" + - "9001:9001" + healthcheck: + test: ["CMD", "nc", "-v", "localhost", "3008"] + interval: 20s + timeout: 5s + retries: 15 + start_period: 5s + extra_hosts: + - "host.docker.internal:host-gateway" + +volumes: + sushiswap_v3_watcher_db_data: diff --git a/stack_orchestrator/data/compose/docker-compose-webapp-template.yml b/stack_orchestrator/data/compose/docker-compose-webapp-template.yml new file mode 100644 index 00000000..b8697afa --- /dev/null +++ b/stack_orchestrator/data/compose/docker-compose-webapp-template.yml @@ -0,0 +1,8 @@ +services: + webapp: + image: cerc/webapp-container:local + restart: always + environment: + CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} + ports: + - "3000" diff --git a/stack_orchestrator/data/config/fixturenet-laconicd/create-fixturenet.sh b/stack_orchestrator/data/config/fixturenet-laconicd/create-fixturenet.sh index 9c30bff8..d444fcad 100644 --- a/stack_orchestrator/data/config/fixturenet-laconicd/create-fixturenet.sh +++ b/stack_orchestrator/data/config/fixturenet-laconicd/create-fixturenet.sh @@ -14,104 +14,111 @@ LOGLEVEL="info" TRACE="--trace" # TRACE="" -# validate dependencies are installed -command -v jq > /dev/null 2>&1 || { echo >&2 "jq not installed. More info: https://stedolan.github.io/jq/download/"; exit 1; } +if [ "$1" == "clean" ] || [ ! -d "$HOME/.laconicd/data/blockstore.db" ]; then + # validate dependencies are installed + command -v jq > /dev/null 2>&1 || { echo >&2 "jq not installed. More info: https://stedolan.github.io/jq/download/"; exit 1; } -# remove existing daemon and client -rm -rf ~/.laconic* + # remove existing daemon and client + rm -rf $HOME/.laconicd/* + rm -rf $HOME/.laconic/* -make install - -laconicd config keyring-backend $KEYRING -laconicd config chain-id $CHAINID - -# if $KEY exists it should be deleted -laconicd keys add $KEY --keyring-backend $KEYRING --algo $KEYALGO - -# Set moniker and chain-id for Ethermint (Moniker can be anything, chain-id must be an integer) -laconicd init $MONIKER --chain-id $CHAINID - -# Change parameter token denominations to aphoton -cat $HOME/.laconicd/config/genesis.json | jq '.app_state["staking"]["params"]["bond_denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json -cat $HOME/.laconicd/config/genesis.json | jq '.app_state["crisis"]["constant_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json -cat $HOME/.laconicd/config/genesis.json | jq '.app_state["gov"]["deposit_params"]["min_deposit"][0]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json -cat $HOME/.laconicd/config/genesis.json | jq '.app_state["mint"]["params"]["mint_denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json -# Custom modules -cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["record_rent"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json -cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json -cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_commit_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json -cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_reveal_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json -cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_minimum_bid"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json - -if [[ "$TEST_REGISTRY_EXPIRY" == "true" ]]; then - echo "Setting timers for expiry tests." - - cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["record_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json - cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_grace_period"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json - cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json -fi - -if [[ "$TEST_AUCTION_ENABLED" == "true" ]]; then - echo "Enabling auction and setting timers." - - cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_enabled"]=true' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json - cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json - cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_grace_period"]="300s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json - cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_commits_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json - cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_reveals_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json -fi - -# increase block time (?) -cat $HOME/.laconicd/config/genesis.json | jq '.consensus_params["block"]["time_iota_ms"]="1000"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json - -# Set gas limit in genesis -cat $HOME/.laconicd/config/genesis.json | jq '.consensus_params["block"]["max_gas"]="10000000"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json - -# disable produce empty block -if [[ "$OSTYPE" == "darwin"* ]]; then - sed -i '' 's/create_empty_blocks = true/create_empty_blocks = false/g' $HOME/.laconicd/config/config.toml - else - sed -i 's/create_empty_blocks = true/create_empty_blocks = false/g' $HOME/.laconicd/config/config.toml -fi - -if [[ $1 == "pending" ]]; then - if [[ "$OSTYPE" == "darwin"* ]]; then - sed -i '' 's/create_empty_blocks_interval = "0s"/create_empty_blocks_interval = "30s"/g' $HOME/.laconicd/config/config.toml - sed -i '' 's/timeout_propose = "3s"/timeout_propose = "30s"/g' $HOME/.laconicd/config/config.toml - sed -i '' 's/timeout_propose_delta = "500ms"/timeout_propose_delta = "5s"/g' $HOME/.laconicd/config/config.toml - sed -i '' 's/timeout_prevote = "1s"/timeout_prevote = "10s"/g' $HOME/.laconicd/config/config.toml - sed -i '' 's/timeout_prevote_delta = "500ms"/timeout_prevote_delta = "5s"/g' $HOME/.laconicd/config/config.toml - sed -i '' 's/timeout_precommit = "1s"/timeout_precommit = "10s"/g' $HOME/.laconicd/config/config.toml - sed -i '' 's/timeout_precommit_delta = "500ms"/timeout_precommit_delta = "5s"/g' $HOME/.laconicd/config/config.toml - sed -i '' 's/timeout_commit = "5s"/timeout_commit = "150s"/g' $HOME/.laconicd/config/config.toml - sed -i '' 's/timeout_broadcast_tx_commit = "10s"/timeout_broadcast_tx_commit = "150s"/g' $HOME/.laconicd/config/config.toml - else - sed -i 's/create_empty_blocks_interval = "0s"/create_empty_blocks_interval = "30s"/g' $HOME/.laconicd/config/config.toml - sed -i 's/timeout_propose = "3s"/timeout_propose = "30s"/g' $HOME/.laconicd/config/config.toml - sed -i 's/timeout_propose_delta = "500ms"/timeout_propose_delta = "5s"/g' $HOME/.laconicd/config/config.toml - sed -i 's/timeout_prevote = "1s"/timeout_prevote = "10s"/g' $HOME/.laconicd/config/config.toml - sed -i 's/timeout_prevote_delta = "500ms"/timeout_prevote_delta = "5s"/g' $HOME/.laconicd/config/config.toml - sed -i 's/timeout_precommit = "1s"/timeout_precommit = "10s"/g' $HOME/.laconicd/config/config.toml - sed -i 's/timeout_precommit_delta = "500ms"/timeout_precommit_delta = "5s"/g' $HOME/.laconicd/config/config.toml - sed -i 's/timeout_commit = "5s"/timeout_commit = "150s"/g' $HOME/.laconicd/config/config.toml - sed -i 's/timeout_broadcast_tx_commit = "10s"/timeout_broadcast_tx_commit = "150s"/g' $HOME/.laconicd/config/config.toml + if [ -n "`which make`" ]; then + make install fi -fi -# Allocate genesis accounts (cosmos formatted addresses) -laconicd add-genesis-account $KEY 100000000000000000000000000aphoton --keyring-backend $KEYRING + laconicd config keyring-backend $KEYRING + laconicd config chain-id $CHAINID -# Sign genesis transaction -laconicd gentx $KEY 1000000000000000000000aphoton --keyring-backend $KEYRING --chain-id $CHAINID + # if $KEY exists it should be deleted + laconicd keys add $KEY --keyring-backend $KEYRING --algo $KEYALGO -# Collect genesis tx -laconicd collect-gentxs + # Set moniker and chain-id for Ethermint (Moniker can be anything, chain-id must be an integer) + laconicd init $MONIKER --chain-id $CHAINID -# Run this to ensure everything worked and that the genesis file is setup correctly -laconicd validate-genesis + # Change parameter token denominations to aphoton + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["staking"]["params"]["bond_denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["crisis"]["constant_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["gov"]["deposit_params"]["min_deposit"][0]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["mint"]["params"]["mint_denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + # Custom modules + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["record_rent"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_commit_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_reveal_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_minimum_bid"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json -if [[ $1 == "pending" ]]; then - echo "pending mode is on, please wait for the first block committed." + if [[ "$TEST_REGISTRY_EXPIRY" == "true" ]]; then + echo "Setting timers for expiry tests." + + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["record_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_grace_period"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + fi + + if [[ "$TEST_AUCTION_ENABLED" == "true" ]]; then + echo "Enabling auction and setting timers." + + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_enabled"]=true' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_grace_period"]="300s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_commits_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_reveals_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + fi + + # increase block time (?) + cat $HOME/.laconicd/config/genesis.json | jq '.consensus_params["block"]["time_iota_ms"]="1000"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + + # Set gas limit in genesis + cat $HOME/.laconicd/config/genesis.json | jq '.consensus_params["block"]["max_gas"]="10000000"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json + + # disable produce empty block + if [[ "$OSTYPE" == "darwin"* ]]; then + sed -i '' 's/create_empty_blocks = true/create_empty_blocks = false/g' $HOME/.laconicd/config/config.toml + else + sed -i 's/create_empty_blocks = true/create_empty_blocks = false/g' $HOME/.laconicd/config/config.toml + fi + + if [[ $1 == "pending" ]]; then + if [[ "$OSTYPE" == "darwin"* ]]; then + sed -i '' 's/create_empty_blocks_interval = "0s"/create_empty_blocks_interval = "30s"/g' $HOME/.laconicd/config/config.toml + sed -i '' 's/timeout_propose = "3s"/timeout_propose = "30s"/g' $HOME/.laconicd/config/config.toml + sed -i '' 's/timeout_propose_delta = "500ms"/timeout_propose_delta = "5s"/g' $HOME/.laconicd/config/config.toml + sed -i '' 's/timeout_prevote = "1s"/timeout_prevote = "10s"/g' $HOME/.laconicd/config/config.toml + sed -i '' 's/timeout_prevote_delta = "500ms"/timeout_prevote_delta = "5s"/g' $HOME/.laconicd/config/config.toml + sed -i '' 's/timeout_precommit = "1s"/timeout_precommit = "10s"/g' $HOME/.laconicd/config/config.toml + sed -i '' 's/timeout_precommit_delta = "500ms"/timeout_precommit_delta = "5s"/g' $HOME/.laconicd/config/config.toml + sed -i '' 's/timeout_commit = "5s"/timeout_commit = "150s"/g' $HOME/.laconicd/config/config.toml + sed -i '' 's/timeout_broadcast_tx_commit = "10s"/timeout_broadcast_tx_commit = "150s"/g' $HOME/.laconicd/config/config.toml + else + sed -i 's/create_empty_blocks_interval = "0s"/create_empty_blocks_interval = "30s"/g' $HOME/.laconicd/config/config.toml + sed -i 's/timeout_propose = "3s"/timeout_propose = "30s"/g' $HOME/.laconicd/config/config.toml + sed -i 's/timeout_propose_delta = "500ms"/timeout_propose_delta = "5s"/g' $HOME/.laconicd/config/config.toml + sed -i 's/timeout_prevote = "1s"/timeout_prevote = "10s"/g' $HOME/.laconicd/config/config.toml + sed -i 's/timeout_prevote_delta = "500ms"/timeout_prevote_delta = "5s"/g' $HOME/.laconicd/config/config.toml + sed -i 's/timeout_precommit = "1s"/timeout_precommit = "10s"/g' $HOME/.laconicd/config/config.toml + sed -i 's/timeout_precommit_delta = "500ms"/timeout_precommit_delta = "5s"/g' $HOME/.laconicd/config/config.toml + sed -i 's/timeout_commit = "5s"/timeout_commit = "150s"/g' $HOME/.laconicd/config/config.toml + sed -i 's/timeout_broadcast_tx_commit = "10s"/timeout_broadcast_tx_commit = "150s"/g' $HOME/.laconicd/config/config.toml + fi + fi + + # Allocate genesis accounts (cosmos formatted addresses) + laconicd add-genesis-account $KEY 100000000000000000000000000aphoton --keyring-backend $KEYRING + + # Sign genesis transaction + laconicd gentx $KEY 1000000000000000000000aphoton --keyring-backend $KEYRING --chain-id $CHAINID + + # Collect genesis tx + laconicd collect-gentxs + + # Run this to ensure everything worked and that the genesis file is setup correctly + laconicd validate-genesis + + if [[ $1 == "pending" ]]; then + echo "pending mode is on, please wait for the first block committed." + fi +else + echo "Using existing database at $HOME/.laconicd. To replace, run '`basename $0` clean'" fi # Start the node (remove the --pruning=nothing flag if historical queries are not needed) diff --git a/stack_orchestrator/data/config/fixturenet-optimism/generate-l2-config.sh b/stack_orchestrator/data/config/fixturenet-optimism/generate-l2-config.sh deleted file mode 100755 index b10048d2..00000000 --- a/stack_orchestrator/data/config/fixturenet-optimism/generate-l2-config.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -set -e -if [ -n "$CERC_SCRIPT_DEBUG" ]; then - set -x -fi - -CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}" - -# Check existing config if it exists -if [ -f /app/jwt.txt ] && [ -f /app/rollup.json ]; then - echo "Found existing L2 config, cross-checking with L1 deployment config" - - SOURCE_L1_CONF=$(cat /contracts-bedrock/deploy-config/getting-started.json) - EXP_L1_BLOCKHASH=$(echo "$SOURCE_L1_CONF" | jq -r '.l1StartingBlockTag') - EXP_BATCHER=$(echo "$SOURCE_L1_CONF" | jq -r '.batchSenderAddress') - - GEN_L2_CONF=$(cat /app/rollup.json) - GEN_L1_BLOCKHASH=$(echo "$GEN_L2_CONF" | jq -r '.genesis.l1.hash') - GEN_BATCHER=$(echo "$GEN_L2_CONF" | jq -r '.genesis.system_config.batcherAddr') - - if [ "$EXP_L1_BLOCKHASH" = "$GEN_L1_BLOCKHASH" ] && [ "$EXP_BATCHER" = "$GEN_BATCHER" ]; then - echo "Config cross-checked, exiting" - exit 0 - fi - - echo "Existing L2 config doesn't match the L1 deployment config, please clear L2 config volume before starting" - exit 1 -fi - -op-node genesis l2 \ - --deploy-config /contracts-bedrock/deploy-config/getting-started.json \ - --deployment-dir /contracts-bedrock/deployments/getting-started/ \ - --outfile.l2 /app/genesis.json \ - --outfile.rollup /app/rollup.json \ - --l1-rpc $CERC_L1_RPC - -openssl rand -hex 32 > /app/jwt.txt diff --git a/stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/deploy-contracts.sh b/stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/deploy-contracts.sh new file mode 100755 index 00000000..23a2bc30 --- /dev/null +++ b/stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/deploy-contracts.sh @@ -0,0 +1,172 @@ +#!/bin/bash +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +CERC_L1_CHAIN_ID="${CERC_L1_CHAIN_ID:-${DEFAULT_CERC_L1_CHAIN_ID}}" +CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}" + +CERC_L1_ACCOUNTS_CSV_URL="${CERC_L1_ACCOUNTS_CSV_URL:-${DEFAULT_CERC_L1_ACCOUNTS_CSV_URL}}" + +export DEPLOYMENT_CONTEXT="$CERC_L1_CHAIN_ID" +# Optional create2 salt for deterministic deployment of contract implementations +export IMPL_SALT=$(openssl rand -hex 32) + +echo "Using L1 RPC endpoint ${CERC_L1_RPC}" + +# Exit if a deployment already exists (on restarts) +if [ -d "/l1-deployment/$DEPLOYMENT_CONTEXT" ]; then + echo "Deployment directory /l1-deployment/$DEPLOYMENT_CONTEXT, checking OptimismPortal deployment" + + OPTIMISM_PORTAL_ADDRESS=$(cat /l1-deployment/$DEPLOYMENT_CONTEXT/OptimismPortal.json | jq -r .address) + contract_code=$(cast code $OPTIMISM_PORTAL_ADDRESS --rpc-url $CERC_L1_RPC) + + if [ -z "${contract_code#0x}" ]; then + echo "Error: A deployment directory was found in the volume, but no contract code was found on-chain at the associated address. Please clear L1 deployment volume before restarting." + exit 1 + else + echo "Deployment found, exiting (successfully)." + exit 0 + fi +fi + +wait_for_block() { + local block="$1" # Block to wait for + local timeout="$2" # Max time to wait in seconds + + echo "Waiting for block $block." + i=0 + loops=$(($timeout/10)) + while [ -z "$block_result" ] && [[ "$i" -lt "$loops" ]]; do + sleep 10 + echo "Checking..." + block_result=$(cast block $block --rpc-url $CERC_L1_RPC | grep -E "(timestamp|hash|number)" || true) + i=$(($i + 1)) + done +} + +# We need four accounts and their private keys for the deployment: Admin, Proposer, Batcher, and Sequencer +# If $CERC_L1_ADDRESS and $CERC_L1_PRIV_KEY have been set, we'll assign it to Admin and generate/fund the remaining three accounts from it +# If not, we'll assume the L1 is the stack's own fixturenet-eth and use the pre-funded accounts/keys from $CERC_L1_ACCOUNTS_CSV_URL +if [ -n "$CERC_L1_ADDRESS" ] && [ -n "$CERC_L1_PRIV_KEY" ]; then + wallet1=$(cast wallet new) + wallet2=$(cast wallet new) + wallet3=$(cast wallet new) + # Admin + ADMIN=$CERC_L1_ADDRESS + ADMIN_KEY=$CERC_L1_PRIV_KEY + # Proposer + PROPOSER=$(echo "$wallet1" | awk '/Address:/{print $2}') + PROPOSER_KEY=$(echo "$wallet1" | awk '/Private key:/{print $3}') + # Batcher + BATCHER=$(echo "$wallet2" | awk '/Address:/{print $2}') + BATCHER_KEY=$(echo "$wallet2" | awk '/Private key:/{print $3}') + # Sequencer + SEQ=$(echo "$wallet3" | awk '/Address:/{print $2}') + SEQ_KEY=$(echo "$wallet3" | awk '/Private key:/{print $3}') + + echo "Funding accounts." + wait_for_block 1 300 + cast send --from $ADMIN --rpc-url $CERC_L1_RPC --value 5ether $PROPOSER --private-key $ADMIN_KEY + cast send --from $ADMIN --rpc-url $CERC_L1_RPC --value 10ether $BATCHER --private-key $ADMIN_KEY + cast send --from $ADMIN --rpc-url $CERC_L1_RPC --value 2ether $SEQ --private-key $ADMIN_KEY +else + curl -o accounts.csv $CERC_L1_ACCOUNTS_CSV_URL + # Admin + ADMIN=$(awk -F ',' 'NR == 1 {print $2}' accounts.csv) + ADMIN_KEY=$(awk -F ',' 'NR == 1 {print $3}' accounts.csv) + # Proposer + PROPOSER=$(awk -F ',' 'NR == 2 {print $2}' accounts.csv) + PROPOSER_KEY=$(awk -F ',' 'NR == 2 {print $3}' accounts.csv) + # Batcher + BATCHER=$(awk -F ',' 'NR == 3 {print $2}' accounts.csv) + BATCHER_KEY=$(awk -F ',' 'NR == 3 {print $3}' accounts.csv) + # Sequencer + SEQ=$(awk -F ',' 'NR == 4 {print $2}' accounts.csv) + SEQ_KEY=$(awk -F ',' 'NR == 4 {print $3}' accounts.csv) +fi + +echo "Using accounts:" +echo -e "Admin: $ADMIN\nProposer: $PROPOSER\nBatcher: $BATCHER\nSequencer: $SEQ" + +# These accounts will be needed by other containers, so write them to a shared volume +echo "Writing accounts/private keys to volume l2_accounts." +accounts_json=$(jq -n \ + --arg Admin "$ADMIN" --arg AdminKey "$ADMIN_KEY" \ + --arg Proposer "$PROPOSER" --arg ProposerKey "$PROPOSER_KEY" \ + --arg Batcher "$BATCHER" --arg BatcherKey "$BATCHER_KEY" \ + --arg Seq "$SEQ" --arg SeqKey "$SEQ_KEY" \ + '{Admin: $Admin, AdminKey: $AdminKey, Proposer: $Proposer, ProposerKey: $ProposerKey, Batcher: $Batcher, BatcherKey: $BatcherKey, Seq: $Seq, SeqKey: $SeqKey}') +echo "$accounts_json" > "/l2-accounts/accounts.json" + +# Get a finalized L1 block to set as the starting point for the L2 deployment +# If the chain is a freshly created fixturenet-eth, a finalized block won't be available for many minutes; rather than wait, we can use block 1 +echo "Checking L1 for finalized block..." +finalized=$(cast block finalized --rpc-url $CERC_L1_RPC | grep -E "(timestamp|hash|number)" || true) + +if [ -n "$finalized" ]; then + # finalized block was found + start_block=$finalized +else + # assume fresh chain and use block 1 instead + echo "No finalized block. Using block 1 instead." + # wait for 20 or so blocks to be safe + wait_for_block 24 300 + start_block=$(cast block 1 --rpc-url $CERC_L1_RPC | grep -E "(timestamp|hash|number)" || true) +fi + +if [ -z "$start_block" ]; then + echo "Unable to query chain for starting block. Exiting..." + exit 1 +fi + +BLOCKHASH=$(echo $start_block | awk -F ' ' '{print $2}') +HEIGHT=$(echo $start_block | awk -F ' ' '{print $4}') +TIMESTAMP=$(echo $start_block | awk -F ' ' '{print $6}') + +echo "Using block as deployment point:" +echo "Height: $HEIGHT" +echo "Hash: $BLOCKHASH" +echo "Timestamp: $TIMESTAMP" + +# Fill out the deployment template (./deploy-config/getting-started.json) with our values: +echo "Writing deployment config." +deploy_config_file="deploy-config/$DEPLOYMENT_CONTEXT.json" +cp deploy-config/getting-started.json $deploy_config_file +sed -i "s/\"l1ChainID\": .*/\"l1ChainID\": $DEPLOYMENT_CONTEXT,/g" $deploy_config_file +sed -i "s/ADMIN/$ADMIN/g" $deploy_config_file +sed -i "s/PROPOSER/$PROPOSER/g" $deploy_config_file +sed -i "s/BATCHER/$BATCHER/g" $deploy_config_file +sed -i "s/SEQUENCER/$SEQ/g" $deploy_config_file +sed -i "s/BLOCKHASH/$BLOCKHASH/g" $deploy_config_file +sed -i "s/TIMESTAMP/$TIMESTAMP/g" $deploy_config_file + +mkdir -p deployments/$DEPLOYMENT_CONTEXT + +# Deployment requires the create2 deterministic proxy contract be published on L1 at address 0x4e59b44847b379578588920ca78fbf26c0b4956c +# See: https://github.com/Arachnid/deterministic-deployment-proxy +echo "Deploying create2 proxy contract..." +echo "Funding deployment signer address" +deployment_signer="0x3fab184622dc19b6109349b94811493bf2a45362" +cast send --from $ADMIN --rpc-url $CERC_L1_RPC --value 0.5ether $deployment_signer --private-key $ADMIN_KEY +echo "Deploying contract..." +raw_bytes="0xf8a58085174876e800830186a08080b853604580600e600039806000f350fe7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe03601600081602082378035828234f58015156039578182fd5b8082525050506014600cf31ba02222222222222222222222222222222222222222222222222222222222222222a02222222222222222222222222222222222222222222222222222222222222222" + +cast publish --rpc-url $CERC_L1_RPC $raw_bytes + +# Create the L2 deployment +echo "Deploying L1 Optimism contracts..." +forge script scripts/Deploy.s.sol:Deploy --private-key $ADMIN_KEY --broadcast --rpc-url $CERC_L1_RPC +forge script scripts/Deploy.s.sol:Deploy --sig 'sync()' --private-key $ADMIN_KEY --broadcast --rpc-url $CERC_L1_RPC + +echo "*************************************" +echo "Done deploying contracts." + +# Copy files needed by other containers to the appropriate shared volumes +echo "Copying deployment artifacts volume l1_deployment and deploy-config to volume l2_config" +cp -a /app/packages/contracts-bedrock/deployments/$DEPLOYMENT_CONTEXT /l1-deployment +cp /app/packages/contracts-bedrock/deploy-config/$DEPLOYMENT_CONTEXT.json /l2-config +openssl rand -hex 32 > /l2-config/l2-jwt.txt + +echo "Deployment successful. Exiting" diff --git a/stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/run.sh b/stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/run.sh deleted file mode 100755 index d878c03f..00000000 --- a/stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/run.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/bin/bash -set -e -if [ -n "$CERC_SCRIPT_DEBUG" ]; then - set -x -fi - -CERC_L1_CHAIN_ID="${CERC_L1_CHAIN_ID:-${DEFAULT_CERC_L1_CHAIN_ID}}" -CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}" - -CERC_L1_ACCOUNTS_CSV_URL="${CERC_L1_ACCOUNTS_CSV_URL:-${DEFAULT_CERC_L1_ACCOUNTS_CSV_URL}}" - -echo "Using L1 RPC endpoint ${CERC_L1_RPC}" - -IMPORT_1="import './verify-contract-deployment'" -IMPORT_2="import './rekey-json'" -IMPORT_3="import './send-balance'" - -# Append mounted tasks to tasks/index.ts file if not present -if ! grep -Fxq "$IMPORT_1" tasks/index.ts; then - echo "$IMPORT_1" >> tasks/index.ts - echo "$IMPORT_2" >> tasks/index.ts - echo "$IMPORT_3" >> tasks/index.ts -fi - -# Update the chainId in the hardhat config -sed -i "/getting-started/ {n; s/.*chainId.*/ chainId: $CERC_L1_CHAIN_ID,/}" hardhat.config.ts - -# Exit if a deployment already exists (on restarts) -# Note: fixturenet-eth-geth currently starts fresh on a restart -if [ -d "deployments/getting-started" ]; then - echo "Deployment directory deployments/getting-started found, checking SystemDictator deployment" - - # Read JSON file into variable - SYSTEM_DICTATOR_DETAILS=$(cat deployments/getting-started/SystemDictator.json) - - # Parse JSON into variables - SYSTEM_DICTATOR_ADDRESS=$(echo "$SYSTEM_DICTATOR_DETAILS" | jq -r '.address') - SYSTEM_DICTATOR_TXHASH=$(echo "$SYSTEM_DICTATOR_DETAILS" | jq -r '.transactionHash') - - if yarn hardhat verify-contract-deployment --contract "${SYSTEM_DICTATOR_ADDRESS}" --transaction-hash "${SYSTEM_DICTATOR_TXHASH}"; then - echo "Deployment verfication successful, exiting" - exit 0 - else - echo "Deployment verfication failed, please clear L1 deployment volume before starting" - exit 1 - fi -fi - -# Generate the L2 account addresses -yarn hardhat rekey-json --output /l2-accounts/keys.json - -# Read JSON file into variable -KEYS_JSON=$(cat /l2-accounts/keys.json) - -# Parse JSON into variables -ADMIN_ADDRESS=$(echo "$KEYS_JSON" | jq -r '.Admin.address') -ADMIN_PRIV_KEY=$(echo "$KEYS_JSON" | jq -r '.Admin.privateKey') -PROPOSER_ADDRESS=$(echo "$KEYS_JSON" | jq -r '.Proposer.address') -BATCHER_ADDRESS=$(echo "$KEYS_JSON" | jq -r '.Batcher.address') -SEQUENCER_ADDRESS=$(echo "$KEYS_JSON" | jq -r '.Sequencer.address') - -# Get the private keys of L1 accounts -if [ -n "$CERC_L1_ACCOUNTS_CSV_URL" ] && \ - l1_accounts_response=$(curl -L --write-out '%{http_code}' --silent --output /dev/null "$CERC_L1_ACCOUNTS_CSV_URL") && \ - [ "$l1_accounts_response" -eq 200 ]; -then - echo "Fetching L1 account credentials using provided URL" - mkdir -p /geth-accounts - wget -O /geth-accounts/accounts.csv "$CERC_L1_ACCOUNTS_CSV_URL" - - CERC_L1_ADDRESS=$(head -n 1 /geth-accounts/accounts.csv | cut -d ',' -f 2) - CERC_L1_PRIV_KEY=$(head -n 1 /geth-accounts/accounts.csv | cut -d ',' -f 3) - CERC_L1_ADDRESS_2=$(awk -F, 'NR==2{print $(NF-1)}' /geth-accounts/accounts.csv) - CERC_L1_PRIV_KEY_2=$(awk -F, 'NR==2{print $NF}' /geth-accounts/accounts.csv) -else - echo "Couldn't fetch L1 account credentials, using them from env" -fi - -# Send balances to the above L2 addresses -yarn hardhat send-balance --to "${ADMIN_ADDRESS}" --amount 2 --private-key "${CERC_L1_PRIV_KEY}" --network getting-started -yarn hardhat send-balance --to "${PROPOSER_ADDRESS}" --amount 5 --private-key "${CERC_L1_PRIV_KEY}" --network getting-started -yarn hardhat send-balance --to "${BATCHER_ADDRESS}" --amount 1000 --private-key "${CERC_L1_PRIV_KEY}" --network getting-started - -echo "Balances sent to L2 accounts" - -# Select a finalized L1 block as the starting point for roll ups -until FINALIZED_BLOCK=$(cast block finalized --rpc-url "$CERC_L1_RPC"); do - echo "Waiting for a finalized L1 block to exist, retrying after 10s" - sleep 10 -done - -L1_BLOCKNUMBER=$(echo "$FINALIZED_BLOCK" | awk '/number/{print $2}') -L1_BLOCKHASH=$(echo "$FINALIZED_BLOCK" | awk '/hash/{print $2}') -L1_BLOCKTIMESTAMP=$(echo "$FINALIZED_BLOCK" | awk '/timestamp/{print $2}') - -echo "Selected L1 block ${L1_BLOCKNUMBER} as the starting block for roll ups" - -# Update the deployment config -sed -i 's/"l2OutputOracleStartingTimestamp": TIMESTAMP/"l2OutputOracleStartingTimestamp": '"$L1_BLOCKTIMESTAMP"'/g' deploy-config/getting-started.json -jq --arg chainid "$CERC_L1_CHAIN_ID" '.l1ChainID = ($chainid | tonumber)' deploy-config/getting-started.json > tmp.json && mv tmp.json deploy-config/getting-started.json - -node update-config.js deploy-config/getting-started.json "$ADMIN_ADDRESS" "$PROPOSER_ADDRESS" "$BATCHER_ADDRESS" "$SEQUENCER_ADDRESS" "$L1_BLOCKHASH" - -echo "Updated the deployment config" - -# Create a .env file -echo "L1_RPC=$CERC_L1_RPC" > .env -echo "PRIVATE_KEY_DEPLOYER=$ADMIN_PRIV_KEY" >> .env - -echo "Deploying the L1 smart contracts, this will take a while..." - -# Deploy the L1 smart contracts -yarn hardhat deploy --network getting-started --tags l1 - -echo "Deployed the L1 smart contracts" - -# Read Proxy contract's JSON and get the address -PROXY_JSON=$(cat deployments/getting-started/Proxy__OVM_L1StandardBridge.json) -PROXY_ADDRESS=$(echo "$PROXY_JSON" | jq -r '.address') - -# Send balance to the above Proxy contract in L1 for reflecting balance in L2 -# First account -yarn hardhat send-balance --to "${PROXY_ADDRESS}" --amount 1 --private-key "${CERC_L1_PRIV_KEY}" --network getting-started -# Second account -yarn hardhat send-balance --to "${PROXY_ADDRESS}" --amount 1 --private-key "${CERC_L1_PRIV_KEY_2}" --network getting-started - -echo "Balance sent to Proxy L2 contract" -echo "Use following accounts for transactions in L2:" -echo "${CERC_L1_ADDRESS}" -echo "${CERC_L1_ADDRESS_2}" -echo "Done" diff --git a/stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/update-config.js b/stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/update-config.js deleted file mode 100644 index 8a6c09d4..00000000 --- a/stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/update-config.js +++ /dev/null @@ -1,36 +0,0 @@ -const fs = require('fs') - -// Get the command-line argument -const configFile = process.argv[2] -const adminAddress = process.argv[3] -const proposerAddress = process.argv[4] -const batcherAddress = process.argv[5] -const sequencerAddress = process.argv[6] -const blockHash = process.argv[7] - -// Read the JSON file -const configData = fs.readFileSync(configFile) -const configObj = JSON.parse(configData) - -// Update the finalSystemOwner property with the ADMIN_ADDRESS value -configObj.finalSystemOwner = - configObj.portalGuardian = - configObj.controller = - configObj.l2OutputOracleChallenger = - configObj.proxyAdminOwner = - configObj.baseFeeVaultRecipient = - configObj.l1FeeVaultRecipient = - configObj.sequencerFeeVaultRecipient = - configObj.governanceTokenOwner = - adminAddress - -configObj.l2OutputOracleProposer = proposerAddress - -configObj.batchSenderAddress = batcherAddress - -configObj.p2pSequencerAddress = sequencerAddress - -configObj.l1StartingBlockTag = blockHash - -// Write the updated JSON object back to the file -fs.writeFileSync(configFile, JSON.stringify(configObj, null, 2)) diff --git a/stack_orchestrator/data/config/fixturenet-optimism/run-geth.sh b/stack_orchestrator/data/config/fixturenet-optimism/run-geth.sh new file mode 100755 index 00000000..b24fe867 --- /dev/null +++ b/stack_orchestrator/data/config/fixturenet-optimism/run-geth.sh @@ -0,0 +1,155 @@ +#!/bin/bash +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +# To facilitate deploying the Optimism contracts, a few additional arguments have been added to the geth start command +# Otherwise this script is unchanged from the image's default startup script + +ETHERBASE=`cat /opt/testnet/build/el/accounts.csv | head -1 | cut -d',' -f2` +NETWORK_ID=`cat /opt/testnet/el/el-config.yaml | grep 'chain_id' | awk '{ print $2 }'` +NETRESTRICT=`ip addr | grep inet | grep -v '127.0' | awk '{print $2}'` +CERC_ETH_DATADIR="${CERC_ETH_DATADIR:-$HOME/ethdata}" +CERC_PLUGINS_DIR="${CERC_PLUGINS_DIR:-/usr/local/lib/plugeth}" + +cd /opt/testnet/build/el +python3 -m http.server 9898 & +cd $HOME + +START_CMD="geth" +if [ "true" == "$CERC_REMOTE_DEBUG" ] && [ -x "/usr/local/bin/dlv" ]; then + START_CMD="/usr/local/bin/dlv --listen=:40000 --headless=true --api-version=2 --accept-multiclient exec /usr/local/bin/geth --continue --" +fi + +# See https://linuxconfig.org/how-to-propagate-a-signal-to-child-processes-from-a-bash-script +cleanup() { + echo "Signal received, cleaning up..." + + # Kill the child process first (CERC_REMOTE_DEBUG=true uses dlv which starts geth as a child process) + pkill -P ${geth_pid} + sleep 2 + kill $(jobs -p) + + wait + echo "Done" +} +trap 'cleanup' SIGINT SIGTERM + +if [ "true" == "$RUN_BOOTNODE" ]; then + $START_CMD \ + --datadir="${CERC_ETH_DATADIR}" \ + --nodekeyhex="${BOOTNODE_KEY}" \ + --nodiscover \ + --ipcdisable \ + --networkid=${NETWORK_ID} \ + --netrestrict="${NETRESTRICT}" \ + & + + geth_pid=$! +else + cd /opt/testnet/accounts + ./import_keys.sh + + echo -n "$JWT" > /opt/testnet/build/el/jwtsecret + + if [ "$CERC_RUN_STATEDIFF" == "detect" ] && [ -n "$CERC_STATEDIFF_DB_HOST" ]; then + dig_result=$(dig $CERC_STATEDIFF_DB_HOST +short) + dig_status_code=$? + if [[ $dig_status_code = 0 && -n $dig_result ]]; then + echo "Statediff DB at $CERC_STATEDIFF_DB_HOST" + CERC_RUN_STATEDIFF="true" + else + echo "No statediff DB available." + CERC_RUN_STATEDIFF="false" + fi + fi + + STATEDIFF_OPTS="" + if [ "$CERC_RUN_STATEDIFF" == "true" ]; then + ready=0 + echo "Waiting for statediff DB..." + while [ $ready -eq 0 ]; do + sleep 1 + export PGPASSWORD="$CERC_STATEDIFF_DB_PASSWORD" + result=$(psql -h "$CERC_STATEDIFF_DB_HOST" \ + -p "$CERC_STATEDIFF_DB_PORT" \ + -U "$CERC_STATEDIFF_DB_USER" \ + -d "$CERC_STATEDIFF_DB_NAME" \ + -t -c 'select max(version_id) from goose_db_version;' 2>/dev/null | awk '{ print $1 }') + if [ -n "$result" ]; then + echo "DB ready..." + if [ $result -ge $CERC_STATEDIFF_DB_GOOSE_MIN_VER ]; then + ready=1 + else + echo "DB not at required version (want $CERC_STATEDIFF_DB_GOOSE_MIN_VER, have $result)" + fi + fi + done + STATEDIFF_OPTS="--statediff \ + --statediff.db.host=$CERC_STATEDIFF_DB_HOST \ + --statediff.db.name=$CERC_STATEDIFF_DB_NAME \ + --statediff.db.nodeid=$CERC_STATEDIFF_DB_NODE_ID \ + --statediff.db.password=$CERC_STATEDIFF_DB_PASSWORD \ + --statediff.db.port=$CERC_STATEDIFF_DB_PORT \ + --statediff.db.user=$CERC_STATEDIFF_DB_USER \ + --statediff.db.logstatements=${CERC_STATEDIFF_DB_LOG_STATEMENTS:-false} \ + --statediff.db.copyfrom=${CERC_STATEDIFF_DB_COPY_FROM:-true} \ + --statediff.waitforsync=true \ + --statediff.workers=${CERC_STATEDIFF_WORKERS:-1} \ + --statediff.writing=true" + + if [ -d "${CERC_PLUGINS_DIR}" ]; then + # With plugeth, we separate the statediff options by prefixing with ' -- ' + STATEDIFF_OPTS="--pluginsdir "${CERC_PLUGINS_DIR}" -- ${STATEDIFF_OPTS}" + fi + fi + + # unlock account[0] + echo $ACCOUNT_PASSWORD > "$CERC_ETH_DATADIR/password" + + $START_CMD \ + --datadir="${CERC_ETH_DATADIR}" \ + --bootnodes="${ENODE}" \ + --allow-insecure-unlock \ + --password="${CERC_ETH_DATADIR}/password" \ + --unlock="$ETHERBASE" \ + --rpc.allow-unprotected-txs \ + --http \ + --http.addr="0.0.0.0" \ + --http.vhosts="*" \ + --http.api="${CERC_GETH_HTTP_APIS:-eth,web3,net,admin,personal,debug,statediff}" \ + --http.corsdomain="*" \ + --authrpc.addr="0.0.0.0" \ + --authrpc.vhosts="*" \ + --authrpc.jwtsecret="/opt/testnet/build/el/jwtsecret" \ + --ws \ + --ws.addr="0.0.0.0" \ + --ws.origins="*" \ + --ws.api="${CERC_GETH_WS_APIS:-eth,web3,net,admin,personal,debug,statediff}" \ + --http.corsdomain="*" \ + --networkid="${NETWORK_ID}" \ + --netrestrict="${NETRESTRICT}" \ + --gcmode archive \ + --txlookuplimit=0 \ + --cache.preimages \ + --syncmode=full \ + --mine \ + --miner.threads=1 \ + --metrics \ + --metrics.addr="0.0.0.0" \ + --verbosity=${CERC_GETH_VERBOSITY:-3} \ + --log.vmodule="${CERC_GETH_VMODULE:-statediff/*=5}" \ + --miner.etherbase="${ETHERBASE}" \ + ${STATEDIFF_OPTS} \ + & + + geth_pid=$! +fi + +wait $geth_pid + +if [ "true" == "$CERC_KEEP_RUNNING_AFTER_GETH_EXIT" ]; then + while [ 1 -eq 1 ]; do + sleep 60 + done +fi diff --git a/stack_orchestrator/data/config/fixturenet-optimism/run-op-batcher.sh b/stack_orchestrator/data/config/fixturenet-optimism/run-op-batcher.sh index 18955545..29a65d5d 100755 --- a/stack_orchestrator/data/config/fixturenet-optimism/run-op-batcher.sh +++ b/stack_orchestrator/data/config/fixturenet-optimism/run-op-batcher.sh @@ -6,22 +6,14 @@ fi CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}" -# Get Batcher key from keys.json -BATCHER_KEY=$(jq -r '.Batcher.privateKey' /l2-accounts/keys.json | tr -d '"') +# Start op-batcher +L2_RPC="http://op-geth:8545" +ROLLUP_RPC="http://op-node:8547" +BATCHER_KEY=$(cat /l2-accounts/accounts.json | jq -r .BatcherKey) -cleanup() { - echo "Signal received, cleaning up..." - kill ${batcher_pid} - - wait - echo "Done" -} -trap 'cleanup' INT TERM - -# Run op-batcher op-batcher \ - --l2-eth-rpc=http://op-geth:8545 \ - --rollup-rpc=http://op-node:8547 \ + --l2-eth-rpc=$L2_RPC \ + --rollup-rpc=$ROLLUP_RPC \ --poll-interval=1s \ --sub-safety-margin=6 \ --num-confirmations=1 \ @@ -32,8 +24,4 @@ op-batcher \ --rpc.enable-admin \ --max-channel-duration=1 \ --l1-eth-rpc=$CERC_L1_RPC \ - --private-key=$BATCHER_KEY \ - & - -batcher_pid=$! -wait $batcher_pid + --private-key="${BATCHER_KEY#0x}" diff --git a/stack_orchestrator/data/config/fixturenet-optimism/run-op-geth.sh b/stack_orchestrator/data/config/fixturenet-optimism/run-op-geth.sh index 8b521f85..9b06cedc 100755 --- a/stack_orchestrator/data/config/fixturenet-optimism/run-op-geth.sh +++ b/stack_orchestrator/data/config/fixturenet-optimism/run-op-geth.sh @@ -4,61 +4,36 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi -# TODO: Add in container build or use other tool -echo "Installing jq" -apk update && apk add jq +l2_genesis_file="/l2-config/genesis.json" -# Get Sequencer key from keys.json -SEQUENCER_KEY=$(jq -r '.Sequencer.privateKey' /l2-accounts/keys.json | tr -d '"') +# Check for genesis file; if necessary, wait on op-node to generate +timeout=300 # 5 minutes +start_time=$(date +%s) +elapsed_time=0 +echo "Checking for L2 genesis file at location $l2_genesis_file" +while [ ! -f "$l2_genesis_file" ] && [ $elapsed_time -lt $timeout ]; do + echo "Waiting for L2 genesis file to be generated..." + sleep 10 + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) +done -# Initialize op-geth if datadir/geth not found -if [ -f /op-node/jwt.txt ] && [ -d datadir/geth ]; then - echo "Found existing datadir, checking block signer key" - - BLOCK_SIGNER_KEY=$(cat datadir/block-signer-key) - - if [ "$SEQUENCER_KEY" = "$BLOCK_SIGNER_KEY" ]; then - echo "Sequencer and block signer keys match, skipping initialization" - else - echo "Sequencer and block signer keys don't match, please clear L2 geth data volume before starting" - exit 1 - fi -else - echo "Initializing op-geth" - - mkdir -p datadir - echo "pwd" > datadir/password - echo $SEQUENCER_KEY > datadir/block-signer-key - - geth account import --datadir=datadir --password=datadir/password datadir/block-signer-key - - while [ ! -f "/op-node/jwt.txt" ] - do - echo "Config files not created. Checking after 5 seconds." - sleep 5 - done - - echo "Config files created by op-node, proceeding with the initialization..." - - geth init --datadir=datadir /op-node/genesis.json - echo "Node Initialized" +if [ ! -f "$l2_genesis_file" ]; then + echo "L2 genesis file not found after timeout of $timeout seconds. Exiting..." + exit 1 fi -SEQUENCER_ADDRESS=$(jq -r '.Sequencer.address' /l2-accounts/keys.json | tr -d '"') -echo "SEQUENCER_ADDRESS: ${SEQUENCER_ADDRESS}" +# Initialize geth from our generated L2 genesis file (if not already initialized) +data_dir="/datadir" +if [ ! -d "$datadir/geth" ]; then + geth init --datadir=$data_dir $l2_genesis_file +fi -cleanup() { - echo "Signal received, cleaning up..." - kill ${geth_pid} +# Start op-geth +jwt_file="/l2-config/l2-jwt.txt" - wait - echo "Done" -} -trap 'cleanup' INT TERM - -# Run op-geth geth \ - --datadir ./datadir \ + --datadir=$data_dir \ --http \ --http.corsdomain="*" \ --http.vhosts="*" \ @@ -77,14 +52,5 @@ geth \ --authrpc.vhosts="*" \ --authrpc.addr=0.0.0.0 \ --authrpc.port=8551 \ - --authrpc.jwtsecret=/op-node/jwt.txt \ - --rollup.disabletxpoolgossip=true \ - --password=./datadir/password \ - --allow-insecure-unlock \ - --mine \ - --miner.etherbase=$SEQUENCER_ADDRESS \ - --unlock=$SEQUENCER_ADDRESS \ - & - -geth_pid=$! -wait $geth_pid + --authrpc.jwtsecret=$jwt_file \ + --rollup.disabletxpoolgossip=true diff --git a/stack_orchestrator/data/config/fixturenet-optimism/run-op-node.sh b/stack_orchestrator/data/config/fixturenet-optimism/run-op-node.sh index 516cf0a5..60a96855 100755 --- a/stack_orchestrator/data/config/fixturenet-optimism/run-op-node.sh +++ b/stack_orchestrator/data/config/fixturenet-optimism/run-op-node.sh @@ -4,23 +4,42 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi +CERC_L1_CHAIN_ID="${CERC_L1_CHAIN_ID:-${DEFAULT_CERC_L1_CHAIN_ID}}" CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}" +DEPLOYMENT_CONTEXT="$CERC_L1_CHAIN_ID" -# Get Sequencer key from keys.json -SEQUENCER_KEY=$(jq -r '.Sequencer.privateKey' /l2-accounts/keys.json | tr -d '"') +deploy_config_file="/l2-config/$DEPLOYMENT_CONTEXT.json" +deployment_dir="/l1-deployment/$DEPLOYMENT_CONTEXT" +genesis_outfile="/l2-config/genesis.json" +rollup_outfile="/l2-config/rollup.json" + +# Generate L2 genesis (if not already done) +if [ ! -f "$genesis_outfile" ] || [ ! -f "$rollup_outfile" ]; then + op-node genesis l2 \ + --deploy-config $deploy_config_file \ + --deployment-dir $deployment_dir \ + --outfile.l2 $genesis_outfile \ + --outfile.rollup $rollup_outfile \ + --l1-rpc $CERC_L1_RPC +fi + +# Start op-node +SEQ_KEY=$(cat /l2-accounts/accounts.json | jq -r .SeqKey) +jwt_file=/l2-config/l2-jwt.txt +L2_AUTH="http://op-geth:8551" +RPC_KIND=any # this can optionally be set to a preset for common node providers like Infura, Alchemy, etc. -# Run op-node op-node \ - --l2=http://op-geth:8551 \ - --l2.jwt-secret=/op-node-data/jwt.txt \ - --sequencer.enabled \ - --sequencer.l1-confs=3 \ - --verifier.l1-confs=3 \ - --rollup.config=/op-node-data/rollup.json \ - --rpc.addr=0.0.0.0 \ - --rpc.port=8547 \ - --p2p.disable \ - --rpc.enable-admin \ - --p2p.sequencer.key=$SEQUENCER_KEY \ - --l1=$CERC_L1_RPC \ - --l1.rpckind=any + --l2=$L2_AUTH \ + --l2.jwt-secret=$jwt_file \ + --sequencer.enabled \ + --sequencer.l1-confs=5 \ + --verifier.l1-confs=4 \ + --rollup.config=$rollup_outfile \ + --rpc.addr=0.0.0.0 \ + --rpc.port=8547 \ + --p2p.disable \ + --rpc.enable-admin \ + --p2p.sequencer.key="${SEQ_KEY#0x}" \ + --l1=$CERC_L1_RPC \ + --l1.rpckind=$RPC_KIND diff --git a/stack_orchestrator/data/config/fixturenet-optimism/run-op-proposer.sh b/stack_orchestrator/data/config/fixturenet-optimism/run-op-proposer.sh index 09746760..092705ca 100755 --- a/stack_orchestrator/data/config/fixturenet-optimism/run-op-proposer.sh +++ b/stack_orchestrator/data/config/fixturenet-optimism/run-op-proposer.sh @@ -5,32 +5,18 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then fi CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}" +CERC_L1_CHAIN_ID="${CERC_L1_CHAIN_ID:-${DEFAULT_CERC_L1_CHAIN_ID}}" +DEPLOYMENT_CONTEXT="$CERC_L1_CHAIN_ID" -# Read the L2OutputOracle contract address from the deployment -L2OO_DEPLOYMENT=$(cat /contracts-bedrock/deployments/getting-started/L2OutputOracle.json) -L2OO_ADDR=$(echo "$L2OO_DEPLOYMENT" | jq -r '.address') +# Start op-proposer +ROLLUP_RPC="http://op-node:8547" +PROPOSER_KEY=$(cat /l2-accounts/accounts.json | jq -r .ProposerKey) +L2OO_ADDR=$(cat /l1-deployment/$DEPLOYMENT_CONTEXT/L2OutputOracleProxy.json | jq -r .address) -# Get Proposer key from keys.json -PROPOSER_KEY=$(jq -r '.Proposer.privateKey' /l2-accounts/keys.json | tr -d '"') - -cleanup() { - echo "Signal received, cleaning up..." - kill ${proposer_pid} - - wait - echo "Done" -} -trap 'cleanup' INT TERM - -# Run op-proposer op-proposer \ - --poll-interval 12s \ - --rpc.port 8560 \ - --rollup-rpc http://op-node:8547 \ - --l2oo-address $L2OO_ADDR \ - --private-key $PROPOSER_KEY \ - --l1-eth-rpc $CERC_L1_RPC \ - & - -proposer_pid=$! -wait $proposer_pid + --poll-interval=12s \ + --rpc.port=8560 \ + --rollup-rpc=$ROLLUP_RPC \ + --l2oo-address="${L2OO_ADDR#0x}" \ + --private-key="${PROPOSER_KEY#0x}" \ + --l1-eth-rpc=$CERC_L1_RPC diff --git a/stack_orchestrator/data/config/monitoring/blackbox.yml b/stack_orchestrator/data/config/monitoring/blackbox.yml new file mode 100644 index 00000000..4a9d72eb --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/blackbox.yml @@ -0,0 +1,7 @@ +modules: + http_2xx: + prober: http + timeout: 5s + http: + valid_status_codes: [] #default to 2xx + method: GET diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/node-exporter-full.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/node-exporter-full.json new file mode 100644 index 00000000..106ca42c --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/node-exporter-full.json @@ -0,0 +1,23806 @@ +{ + "annotations": { + "list": [ + { + "$$hashKey": "object:1058", + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 1860, + "graphTooltip": 1, + "id": 5, + "links": [ + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "GitHub", + "type": "link", + "url": "https://github.com/rfmoz/grafana-dashboards" + }, + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "Grafana", + "type": "link", + "url": "https://grafana.com/grafana/dashboards/1860" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 261, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Quick CPU / Mem / Disk", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Busy state of all CPU cores together", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 85 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 20, + "links": [], + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))) * 100", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "CPU Busy", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Busy state of all CPU cores together (5 min average)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 85 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 155, + "links": [], + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg_over_time(node_load5{instance=\"$node\",job=\"$job\"}[$__rate_interval]) * 100 / on(instance) group_left sum by (instance)(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "Sys Load (5m avg)", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Busy state of all CPU cores together (15 min average)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 85 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 19, + "links": [], + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg_over_time(node_load15{instance=\"$node\",job=\"$job\"}[$__rate_interval]) * 100 / on(instance) group_left sum by (instance)(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]))", + "hide": false, + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "Sys Load (15m avg)", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Non available RAM memory", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 1 + }, + "hideTimeOverride": false, + "id": 16, + "links": [], + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "((avg_over_time(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - avg_over_time(node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])) / (avg_over_time(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) )) * 100", + "format": "time_series", + "hide": true, + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "100 - ((avg_over_time(node_memory_MemAvailable_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) * 100) / avg_over_time(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "B", + "step": 240 + } + ], + "title": "RAM Used", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Used Swap", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 10 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 25 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 21, + "links": [], + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "((avg_over_time(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - avg_over_time(node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])) / (avg_over_time(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) )) * 100", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "SWAP Used", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Used Root FS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 154, + "links": [], + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "100 - ((avg_over_time(node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}[$__rate_interval]) * 100) / avg_over_time(node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}[$__rate_interval]))", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "Root FS Used", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of CPU cores", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 18, + "y": 1 + }, + "id": 14, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU Cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "System uptime", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 20, + "y": 1 + }, + "hideTimeOverride": true, + "id": 15, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "node_time_seconds{instance=\"$node\",job=\"$job\"} - node_boot_time_seconds{instance=\"$node\",job=\"$job\"}", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total RootFS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 70 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 18, + "y": 3 + }, + "id": 23, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "RootFS Total", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total RAM", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 20, + "y": 3 + }, + "id": 75, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "RAM Total", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total SWAP", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 22, + "y": 3 + }, + "id": 18, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "SWAP Total", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 263, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Basic CPU / Mem / Net / Disk", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Basic CPU info", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Busy Iowait" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy Iowait" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy System" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy User" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy Other" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 77, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "width": 250 + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Busy System", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Busy User", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Busy Iowait", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=~\".*irq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Busy IRQs", + "range": true, + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!='idle',mode!='user',mode!='system',mode!='iowait',mode!='irq',mode!='softirq'}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Busy Other", + "range": true, + "refId": "E", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Idle", + "range": true, + "refId": "F", + "step": 240 + } + ], + "title": "CPU Basic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Basic memory usage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SWAP Used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap Used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM Cache + Buffer" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#DEDAF7", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "normal" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 78, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "RAM Total", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - (node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "RAM Used", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "RAM Cache + Buffer", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "RAM Free", + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SWAP Used", + "refId": "E", + "step": 240 + } + ], + "title": "Memory Basic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Basic network info per interface", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Recv_bytes_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_drop_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_errs_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CCA300", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_bytes_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_drop_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_errs_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CCA300", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_drop_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#967302", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_errs_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_bytes_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_drop_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#967302", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_errs_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 74, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "recv {{device}}", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "trans {{device}} ", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Basic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Disk space used of all filesystems mounted", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 152, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "100 - ((node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} * 100) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}}", + "refId": "A", + "step": 240 + } + ], + "title": "Disk Space Used Basic", + "type": "timeseries" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 265, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "percentage", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 70, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Idle - Waiting for something to happen" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Iowait - Waiting for I/O to complete" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Irq - Servicing interrupts" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Nice - Niced processes executing in user mode" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Softirq - Servicing softirqs" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Steal - Time spent in other operating systems when running in a virtualized environment" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCE2DE", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "System - Processes executing in kernel mode" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "User - Normal processes executing in user mode" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#5195CE", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 3, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 250 + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "System - Processes executing in kernel mode", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "User - Normal processes executing in user mode", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nice - Niced processes executing in user mode", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Iowait - Waiting for I/O to complete", + "range": true, + "refId": "E", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"irq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Irq - Servicing interrupts", + "range": true, + "refId": "F", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"softirq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Softirq - Servicing softirqs", + "range": true, + "refId": "G", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"steal\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Steal - Time spent in other operating systems when running in a virtualized environment", + "range": true, + "refId": "H", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Idle - Waiting for something to happen", + "range": true, + "refId": "J", + "step": 240 + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap - Swap memory usage" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused - Free memory unassigned" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Hardware Corrupted - *./" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "normal" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 24, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"} - node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Apps - Memory used by user-space applications", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "PageTables - Memory used to map between virtual and physical memory addresses", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SwapCache - Memory that keeps track of pages that have been fetched from swap but not yet been modified", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Slab - Memory used by the kernel to cache data structures for its own use (caches like inode, dentry, etc)", + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Cache - Parked file data (file content) cache", + "refId": "E", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Buffers - Block device (e.g. harddisk) cache", + "refId": "F", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Unused - Free memory unassigned", + "refId": "G", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Swap - Swap space used", + "refId": "H", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_HardwareCorrupted_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working", + "refId": "I", + "step": 240 + } + ], + "title": "Memory Stack", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bits out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "receive_packets_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "receive_packets_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "transmit_packets_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "transmit_packets_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 84, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Transmit", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 156, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} - node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}}", + "refId": "A", + "step": 240 + } + ], + "title": "Disk Space Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "IO read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 229, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", + "intervalFactor": 4, + "legendFormat": "{{device}} - Reads completed", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} - Writes completed", + "refId": "B", + "step": 240 + } + ], + "title": "Disk IOps", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "io time" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*read*./" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byType", + "options": "time" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "hidden" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 42, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{device}} - Successfully read bytes", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{device}} - Successfully written bytes", + "refId": "B", + "step": 240 + } + ], + "title": "I/O Usage Read / Write", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "%util", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "io time" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byType", + "options": "time" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "hidden" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 127, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"} [$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}}", + "refId": "A", + "step": 240 + } + ], + "title": "I/O Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "percentage", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 70, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^Guest - /" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#5195ce", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^GuestNice - /" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#c15c17", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 319, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))", + "hide": false, + "legendFormat": "Guest - Time spent running a virtual CPU for a guest operating system", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))", + "hide": false, + "legendFormat": "GuestNice - Time spent running a niced guest (virtual CPU for guest operating system)", + "range": true, + "refId": "B" + } + ], + "title": "CPU spent seconds in guests (VMs)", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "CPU / Memory / Net / Disk", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 266, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 54 + }, + "id": 136, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Inactive_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Inactive - Memory which has been less recently used. It is more eligible to be reclaimed for other purposes", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Active_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Active - Memory that has been used more recently and usually not reclaimed unless absolutely necessary", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Active / Inactive", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*CommitLimit - *./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 54 + }, + "id": 135, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Committed_AS_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Committed_AS - Amount of memory presently allocated on the system", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_CommitLimit_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "CommitLimit - Amount of memory currently available to be allocated on the system", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Committed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 191, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Inactive_file_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Inactive_file - File-backed memory on inactive LRU list", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Inactive_anon_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Inactive_anon - Anonymous and swap cache on inactive LRU list, including tmpfs (shmem)", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Active_file_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Active_file - File-backed memory on active LRU list", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Active_anon_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Active_anon - Anonymous and swap cache on active least-recently-used (LRU) list, including tmpfs", + "refId": "D", + "step": 240 + } + ], + "title": "Memory Active / Inactive Detail", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 64 + }, + "id": 130, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Writeback_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Writeback - Memory which is actively being written back to disk", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_WritebackTmp_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "WritebackTmp - Memory used by FUSE for temporary writeback buffers", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Dirty_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Dirty - Memory which is waiting to get written back to the disk", + "refId": "C", + "step": 240 + } + ], + "title": "Memory Writeback and Dirty", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 74 + }, + "id": 138, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Mapped_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Mapped - Used memory in mapped pages files which have been mapped, such as libraries", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Shmem_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Shmem - Used shared memory (shared between several processes, thus including RAM disks)", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_ShmemHugePages_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_ShmemPmdMapped_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "ShmemPmdMapped - Amount of shared (shmem/tmpfs) memory backed by huge pages", + "refId": "D", + "step": 240 + } + ], + "title": "Memory Shared and Mapped", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 74 + }, + "id": 131, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_SUnreclaim_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SUnreclaim - Part of Slab, that cannot be reclaimed on memory pressure", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SReclaimable - Part of Slab, that might be reclaimed, such as caches", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Slab", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 84 + }, + "id": 70, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_VmallocChunk_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "VmallocChunk - Largest contiguous block of vmalloc area which is free", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_VmallocTotal_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "VmallocTotal - Total size of vmalloc memory area", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_VmallocUsed_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "VmallocUsed - Amount of vmalloc area which is used", + "refId": "C", + "step": 240 + } + ], + "title": "Memory Vmalloc", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 84 + }, + "id": 159, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Bounce_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Bounce - Memory used for block device bounce buffers", + "refId": "A", + "step": 240 + } + ], + "title": "Memory Bounce", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Inactive *./" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 94 + }, + "id": 129, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_AnonHugePages_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "AnonHugePages - Memory in anonymous huge pages", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_AnonPages_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "AnonPages - Memory in user pages not backed by files", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Anonymous", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 94 + }, + "id": 160, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_KernelStack_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "KernelStack - Kernel memory stack. This is not reclaimable", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Percpu_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "PerCPU - Per CPU memory allocated dynamically by loadable modules", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Kernel / CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "pages", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 104 + }, + "id": 140, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_HugePages_Free{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "HugePages_Free - Huge pages in the pool that are not yet allocated", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_HugePages_Rsvd{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "HugePages_Rsvd - Huge pages for which a commitment to allocate from the pool has been made, but no allocation has yet been made", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_HugePages_Surp{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "HugePages_Surp - Huge pages in the pool above the value in /proc/sys/vm/nr_hugepages", + "refId": "C", + "step": 240 + } + ], + "title": "Memory HugePages Counter", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 104 + }, + "id": 71, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_HugePages_Total{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "HugePages - Total size of the pool of huge pages", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Hugepagesize - Huge Page size", + "refId": "B", + "step": 240 + } + ], + "title": "Memory HugePages Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 114 + }, + "id": 128, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_DirectMap1G_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "DirectMap1G - Amount of pages mapped as this size", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_DirectMap2M_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "DirectMap2M - Amount of pages mapped as this size", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_DirectMap4k_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "DirectMap4K - Amount of pages mapped as this size", + "refId": "C", + "step": 240 + } + ], + "title": "Memory DirectMap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 114 + }, + "id": 137, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Unevictable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Unevictable - Amount of unevictable memory that can't be swapped out for a variety of reasons", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_Mlocked_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "MLocked - Size of pages locked to memory using the mlock() system call", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Unevictable and MLocked", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 124 + }, + "id": 132, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_memory_NFS_Unstable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "NFS Unstable - Memory in NFS pages sent to the server, but not yet committed to the storage", + "refId": "A", + "step": 240 + } + ], + "title": "Memory NFS", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Memory Meminfo", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 267, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "pages out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 176, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_vmstat_pgpgin{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pagesin - Page in operations", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_vmstat_pgpgout{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pagesout - Page out operations", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Pages In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "pages out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 22, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_vmstat_pswpin{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pswpin - Pages swapped in", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_vmstat_pswpout{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pswpout - Pages swapped out", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Pages Swap In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "faults", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pgfault - Page major and minor fault operations" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "normal" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 175, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pgfault - Page major and minor fault operations", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pgmajfault - Major page fault operations", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pgminfault - Minor page fault operations", + "refId": "C", + "step": 240 + } + ], + "title": "Memory Page Faults", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 307, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_vmstat_oom_kill{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "oom killer invocations ", + "refId": "A", + "step": 240 + } + ], + "title": "OOM Killer", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Memory Vmstat", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 293, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Variation*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 260, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_timex_estimated_error_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Estimated error in seconds", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_timex_offset_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Time offset in between local system and reference clock", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_timex_maxerror_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Maximum error in seconds", + "refId": "C", + "step": 240 + } + ], + "title": "Time Synchronized Drift", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 291, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_timex_loop_time_constant{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Phase-locked loop time adjust", + "refId": "A", + "step": 240 + } + ], + "title": "Time PLL Adjust", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Variation*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 66 + }, + "id": 168, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_timex_sync_status{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Is clock synchronized to a reliable server (1 = yes, 0 = no)", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_timex_frequency_adjustment_ratio{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Local clock frequency adjustment", + "refId": "B", + "step": 240 + } + ], + "title": "Time Synchronized Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 66 + }, + "id": 294, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_timex_tick_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Seconds between clock ticks", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_timex_tai_offset_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "International Atomic Time (TAI) offset", + "refId": "B", + "step": 240 + } + ], + "title": "Time Misc", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "System Timesync", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 312, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 62, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_procs_blocked{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Processes blocked waiting for I/O to complete", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_procs_running{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Processes in runnable state", + "refId": "B", + "step": 240 + } + ], + "title": "Processes Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 315, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_processes_state{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ state }}", + "refId": "A", + "step": 240 + } + ], + "title": "Processes State", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "forks / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 53 + }, + "id": 148, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_forks_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Processes forks second", + "refId": "A", + "step": 240 + } + ], + "title": "Processes Forks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max.*/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 53 + }, + "id": 149, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Processes virtual memory size in bytes", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "process_resident_memory_max_bytes{instance=\"$node\",job=\"$job\"}", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Maximum amount of virtual memory available in bytes", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Processes virtual memory size in bytes", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(process_virtual_memory_max_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Maximum amount of virtual memory available in bytes", + "refId": "D", + "step": 240 + } + ], + "title": "Processes Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "PIDs limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 63 + }, + "id": 313, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_processes_pids{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Number of PIDs", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_processes_max_processes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "PIDs limit", + "refId": "B", + "step": 240 + } + ], + "title": "PIDs Number and Limit", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*waiting.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 63 + }, + "id": 305, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }} - seconds spent running a process", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }} - seconds spent by processing waiting for this CPU", + "refId": "B", + "step": 240 + } + ], + "title": "Process schedule stats Running / Waiting", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Threads limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 73 + }, + "id": 314, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_processes_threads{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Allocated threads", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_processes_max_threads{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Threads limit", + "refId": "B", + "step": 240 + } + ], + "title": "Threads Number and Limit", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "System Processes", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 269, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 8, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_context_switches_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Context switches", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_intr_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Interrupts", + "refId": "B", + "step": 240 + } + ], + "title": "Context Switches / Interrupts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 7, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_load1{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Load 1m", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_load5{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Load 5m", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_load15{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Load 15m", + "refId": "C", + "step": 240 + } + ], + "title": "System Load", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "hertz" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Max" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 10 + }, + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": false, + "viz": false + } + }, + { + "id": "custom.fillBelowTo", + "value": "Min" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Min" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": false, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 321, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "node_cpu_scaling_frequency_hertz{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }}", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "avg(node_cpu_scaling_frequency_max_hertz{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Max", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "avg(node_cpu_scaling_frequency_min_hertz{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Min", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "CPU Frequency Scaling", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "https://docs.kernel.org/accounting/psi.html", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Memory some" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Memory full" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "I/O some" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "I/O full" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 322, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(node_pressure_cpu_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "CPU some", + "range": true, + "refId": "CPU some", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(node_pressure_memory_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Memory some", + "range": true, + "refId": "Memory some", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(node_pressure_memory_stalled_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Memory full", + "range": true, + "refId": "Memory full", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(node_pressure_io_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "I/O some", + "range": true, + "refId": "I/O some", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(node_pressure_io_stalled_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "I/O full", + "range": true, + "refId": "I/O full", + "step": 240 + } + ], + "title": "Pressure Stall Information", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Critical*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 46 + }, + "id": 259, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_interrupts_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ type }} - {{ info }}", + "refId": "A", + "step": 240 + } + ], + "title": "Interrupts Detail", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 46 + }, + "id": 306, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_schedstat_timeslices_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }}", + "refId": "A", + "step": 240 + } + ], + "title": "Schedule timeslices executed by each cpu", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 151, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_entropy_available_bits{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Entropy available to random number generators", + "refId": "A", + "step": 240 + } + ], + "title": "Entropy", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 308, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(process_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Time spent", + "refId": "A", + "step": 240 + } + ], + "title": "CPU time spent in user and system contexts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 66 + }, + "id": 64, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "process_max_fds{instance=\"$node\",job=\"$job\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Maximum open file descriptors", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "process_open_fds{instance=\"$node\",job=\"$job\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Open file descriptors", + "refId": "B", + "step": 240 + } + ], + "title": "File Descriptors", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "System Misc", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 304, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "temperature", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "celsius" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Critical*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 158, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_hwmon_temp_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }} temp", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_hwmon_temp_crit_alarm_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }} Critical Alarm", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_hwmon_temp_crit_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }} Critical", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_hwmon_temp_crit_hyst_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }} Critical Historical", + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_hwmon_temp_max_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }} Max", + "refId": "E", + "step": 240 + } + ], + "title": "Hardware temperature monitor", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 300, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_cooling_device_cur_state{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Current {{ name }} in {{ type }}", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_cooling_device_max_state{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Max {{ name }} in {{ type }}", + "refId": "B", + "step": 240 + } + ], + "title": "Throttle cooling device", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 69 + }, + "id": 302, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_power_supply_online{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ power_supply }} online", + "refId": "A", + "step": 240 + } + ], + "title": "Power supply", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Hardware Misc", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 296, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 46 + }, + "id": 297, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_systemd_socket_accepted_connections_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ name }} Connections", + "refId": "A", + "step": 240 + } + ], + "title": "Systemd Sockets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Failed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF9830", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Deactivating" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFCB7D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Activating" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C8F2C2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 46 + }, + "id": 298, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"activating\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Activating", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"active\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Active", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"deactivating\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Deactivating", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"failed\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Failed", + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"inactive\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Inactive", + "refId": "E", + "step": 240 + } + ], + "title": "Systemd Units State", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Systemd", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 270, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The number (after merges) of I/O requests completed per second for the device", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "IO read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 9, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "intervalFactor": 4, + "legendFormat": "{{device}} - Reads completed", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} - Writes completed", + "refId": "B", + "step": 240 + } + ], + "title": "Disk IOps Completed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The number of bytes read from or written to the device per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 33, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "{{device}} - Read bytes", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Written bytes", + "refId": "B", + "step": 240 + } + ], + "title": "Disk R/W Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The average time for requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "time. read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 57 + }, + "id": 37, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_read_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}} - Read wait time avg", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_write_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Write wait time avg", + "refId": "B", + "step": 240 + } + ], + "title": "Disk Average Wait Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The average queue length of the requests that were issued to the device", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "aqu-sz", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 57 + }, + "id": 35, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_io_time_weighted_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}}", + "refId": "A", + "step": 240 + } + ], + "title": "Average Queue Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The number of read and write requests merged per second that were queued to the device", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "I/Os", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 133, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_reads_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} - Read merged", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_writes_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} - Write merged", + "refId": "B", + "step": 240 + } + ], + "title": "Disk R/W Merged", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of elapsed time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100% for devices serving requests serially. But for devices serving requests in parallel, such as RAID arrays and modern SSDs, this number does not reflect their performance limits.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "%util", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 36, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}} - IO", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_discard_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}} - discard", + "refId": "B", + "step": 240 + } + ], + "title": "Time Spent Doing I/Os", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The number of outstanding requests at the instant the sample was taken. Incremented as requests are given to appropriate struct request_queue and decremented as they finish.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Outstanding req.", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 77 + }, + "id": 34, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_disk_io_now{instance=\"$node\",job=\"$job\"}", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}} - IO now", + "refId": "A", + "step": 240 + } + ], + "title": "Instantaneous Queue Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "IOs", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 77 + }, + "id": 301, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_discards_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}} - Discards completed", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_disk_discards_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Discards merged", + "refId": "B", + "step": 240 + } + ], + "title": "Disk IOps Discards completed / merged", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Storage Disk", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 271, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 62 + }, + "id": 43, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - Available", + "metric": "", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_filesystem_free_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - Free", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - Size", + "refId": "C", + "step": 240 + } + ], + "title": "Filesystem space available", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "file nodes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 62 + }, + "id": 41, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_filesystem_files_free{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - Free file nodes", + "refId": "A", + "step": 240 + } + ], + "title": "File Nodes Free", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "files", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 72 + }, + "id": 28, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_filefd_maximum{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Max open files", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_filefd_allocated{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Open files", + "refId": "B", + "step": 240 + } + ], + "title": "File Descriptor", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "file Nodes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 72 + }, + "id": 219, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_filesystem_files{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - File nodes total", + "refId": "A", + "step": 240 + } + ], + "title": "File Nodes Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "/ ReadOnly" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 82 + }, + "id": 44, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_filesystem_readonly{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - ReadOnly", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_filesystem_device_error{instance=\"$node\",job=\"$job\",device!~'rootfs',fstype!~'tmpfs'}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - Device error", + "refId": "B", + "step": 240 + } + ], + "title": "Filesystem in ReadOnly / Error", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Storage Filesystem", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 272, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "receive_packets_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "receive_packets_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "transmit_packets_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "transmit_packets_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 60, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_receive_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_transmit_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Transmit", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic by Packets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 142, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_receive_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive errors", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_transmit_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Rransmit errors", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 57 + }, + "id": 143, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_receive_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive drop", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_transmit_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Transmit drop", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Drop", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 57 + }, + "id": 141, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_receive_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive compressed", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_transmit_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Transmit compressed", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Compressed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 146, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_receive_multicast_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive multicast", + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic Multicast", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 144, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_receive_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive fifo", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_transmit_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Transmit fifo", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Fifo", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 77 + }, + "id": 145, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_receive_frame_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive frame", + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic Frame", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 77 + }, + "id": 231, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_transmit_carrier_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Statistic transmit_carrier", + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic Carrier", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 87 + }, + "id": 232, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_network_transmit_colls_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Transmit colls", + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic Colls", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "entries", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "NF conntrack limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 87 + }, + "id": 61, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_nf_conntrack_entries{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "NF conntrack entries", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_nf_conntrack_entries_limit{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "NF conntrack limit", + "refId": "B", + "step": 240 + } + ], + "title": "NF Contrack", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Entries", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 97 + }, + "id": 230, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_arp_entries{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ device }} - ARP entries", + "refId": "A", + "step": 240 + } + ], + "title": "ARP Entries", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 97 + }, + "id": 288, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_network_mtu_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ device }} - Bytes", + "refId": "A", + "step": 240 + } + ], + "title": "MTU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 107 + }, + "id": 280, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_network_speed_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ device }} - Speed", + "refId": "A", + "step": 240 + } + ], + "title": "Speed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 107 + }, + "id": 289, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_network_transmit_queue_length{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ device }} - Interface transmit queue length", + "refId": "A", + "step": 240 + } + ], + "title": "Queue Length", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packetes drop (-) / process (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Dropped.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 117 + }, + "id": 290, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_softnet_processed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{cpu}} - Processed", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_softnet_dropped_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{cpu}} - Dropped", + "refId": "B", + "step": 240 + } + ], + "title": "Softnet Packets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 117 + }, + "id": 310, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_softnet_times_squeezed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{cpu}} - Squeezed", + "refId": "A", + "step": 240 + } + ], + "title": "Softnet Out of Quota", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 127 + }, + "id": 309, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_network_up{operstate=\"up\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{interface}} - Operational state UP", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_network_carrier{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{device}} - Physical link state", + "refId": "B" + } + ], + "title": "Network Operational Status", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Network Traffic", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 273, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 63, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_TCP_alloc{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP_alloc - Allocated sockets", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_TCP_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP_inuse - Tcp sockets currently in use", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_TCP_mem{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP_mem - Used memory for tcp", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_TCP_orphan{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP_orphan - Orphan sockets", + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_TCP_tw{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP_tw - Sockets waiting close", + "refId": "E", + "step": 240 + } + ], + "title": "Sockstat TCP", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 124, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_UDPLITE_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDPLITE_inuse - Udplite sockets currently in use", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_UDP_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP_inuse - Udp sockets currently in use", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_UDP_mem{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP_mem - Used memory for udp", + "refId": "C", + "step": 240 + } + ], + "title": "Sockstat UDP", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 125, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_FRAG_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "FRAG_inuse - Frag sockets currently in use", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_RAW_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "RAW_inuse - Raw sockets currently in use", + "refId": "C", + "step": 240 + } + ], + "title": "Sockstat FRAG / RAW", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 220, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_TCP_mem_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "mem_bytes - TCP sockets in that state", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_UDP_mem_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "mem_bytes - UDP sockets in that state", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_FRAG_memory{instance=\"$node\",job=\"$job\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "FRAG_memory - Used memory for frag", + "refId": "C" + } + ], + "title": "Sockstat Memory Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "sockets", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 68 + }, + "id": 126, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_sockstat_sockets_used{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Sockets_used - Sockets currently in use", + "refId": "A", + "step": 240 + } + ], + "title": "Sockstat Used", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Network Sockstat", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 274, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "octets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 221, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_IpExt_InOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "InOctets - Received octets", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_IpExt_OutOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "OutOctets - Sent octets", + "refId": "B", + "step": 240 + } + ], + "title": "Netstat IP In / Out Octets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "datagrams", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 81, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Ip_Forwarding{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Forwarding - IP forwarding", + "refId": "A", + "step": 240 + } + ], + "title": "Netstat IP Forwarding", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "messages out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 115, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Icmp_InMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "InMsgs - Messages which the entity received. Note that this counter includes all those counted by icmpInErrors", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Icmp_OutMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "OutMsgs - Messages which this entity attempted to send. Note that this counter includes all those counted by icmpOutErrors", + "refId": "B", + "step": 240 + } + ], + "title": "ICMP In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "messages out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 50, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Icmp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "InErrors - Messages which the entity received but determined as having ICMP-specific errors (bad ICMP checksums, bad length, etc.)", + "refId": "A", + "step": 240 + } + ], + "title": "ICMP Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "datagrams out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Snd.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 69 + }, + "id": 55, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Udp_InDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "InDatagrams - Datagrams received", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Udp_OutDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "OutDatagrams - Datagrams sent", + "refId": "B", + "step": 240 + } + ], + "title": "UDP In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "datagrams", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 69 + }, + "id": 109, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Udp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "InErrors - UDP Datagrams that could not be delivered to an application", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Udp_NoPorts{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "NoPorts - UDP Datagrams received on a port with no listener", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_UdpLite_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "InErrors Lite - UDPLite Datagrams that could not be delivered to an application", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Udp_RcvbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "RcvbufErrors - UDP buffer errors received", + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Udp_SndbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "SndbufErrors - UDP buffer errors send", + "refId": "E", + "step": 240 + } + ], + "title": "UDP Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "datagrams out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Snd.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 79 + }, + "id": 299, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Tcp_InSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "InSegs - Segments received, including those received in error. This count includes segments received on currently established connections", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Tcp_OutSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "OutSegs - Segments sent, including those on current connections but excluding those containing only retransmitted octets", + "refId": "B", + "step": 240 + } + ], + "title": "TCP In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 79 + }, + "id": 104, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_TcpExt_ListenOverflows{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "ListenOverflows - Times the listen queue of a socket overflowed", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_TcpExt_ListenDrops{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "ListenDrops - SYNs to LISTEN sockets ignored", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_TcpExt_TCPSynRetrans{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCPSynRetrans - SYN-SYN/ACK retransmits to break down retransmissions in SYN, fast/timeout retransmits", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Tcp_RetransSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "RetransSegs - Segments retransmitted - that is, the number of TCP segments transmitted containing one or more previously transmitted octets", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Tcp_InErrs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "InErrs - Segments received in error (e.g., bad TCP checksums)", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Tcp_OutRsts{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "OutRsts - Segments sent with RST flag", + "refId": "F" + } + ], + "title": "TCP Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "connections", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*MaxConn *./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 89 + }, + "id": 85, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_netstat_Tcp_CurrEstab{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "CurrEstab - TCP connections for which the current state is either ESTABLISHED or CLOSE- WAIT", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_netstat_Tcp_MaxConn{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "MaxConn - Limit on the total number of TCP connections the entity can support (Dynamic is \"-1\")", + "refId": "B", + "step": 240 + } + ], + "title": "TCP Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Sent.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 89 + }, + "id": 91, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_TcpExt_SyncookiesFailed{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "SyncookiesFailed - Invalid SYN cookies received", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_TcpExt_SyncookiesRecv{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "SyncookiesRecv - SYN cookies received", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_TcpExt_SyncookiesSent{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "SyncookiesSent - SYN cookies sent", + "refId": "C", + "step": 240 + } + ], + "title": "TCP SynCookie", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "connections", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 99 + }, + "id": 82, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Tcp_ActiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "ActiveOpens - TCP connections that have made a direct transition to the SYN-SENT state from the CLOSED state", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(node_netstat_Tcp_PassiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "PassiveOpens - TCP connections that have made a direct transition to the SYN-RCVD state from the LISTEN state", + "refId": "B", + "step": 240 + } + ], + "title": "TCP Direct Transition", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Enable with --collector.tcpstat argument on node-exporter", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "connections", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 99 + }, + "id": 320, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"established\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "established - TCP sockets in established state", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"fin_wait2\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "fin_wait2 - TCP sockets in fin_wait2 state", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"listen\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "listen - TCP sockets in listen state", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"time_wait\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "time_wait - TCP sockets in time_wait state", + "range": true, + "refId": "D", + "step": 240 + } + ], + "title": "TCP Stat", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Network Netstat", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 279, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 66 + }, + "id": 40, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_scrape_collector_duration_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{collector}} - Scrape duration", + "refId": "A", + "step": 240 + } + ], + "title": "Node Exporter Scrape Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*error.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + }, + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 66 + }, + "id": 157, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_scrape_collector_success{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{collector}} - Scrape success", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "node_textfile_scrape_error{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{collector}} - Scrape textfile error (1 = true)", + "refId": "B", + "step": 240 + } + ], + "title": "Node Exporter Scrape", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "refId": "A" + } + ], + "title": "Node Exporter", + "type": "row" + } + ], + "refresh": "", + "revision": 1, + "schemaVersion": 38, + "tags": [ + "linux" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "node", + "value": "node" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Job", + "multi": false, + "name": "job", + "options": [], + "query": { + "query": "label_values(node_uname_info, job)", + "refId": "Prometheus-job-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(node_uname_info{job=\"$job\"}, instance)", + "hide": 0, + "includeAll": false, + "label": "Host", + "multi": false, + "name": "node", + "options": [], + "query": { + "query": "label_values(node_uname_info{job=\"$job\"}, instance)", + "refId": "Prometheus-node-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", + "value": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+" + }, + "hide": 2, + "includeAll": false, + "multi": false, + "name": "diskdevices", + "options": [ + { + "selected": true, + "text": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", + "value": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+" + } + ], + "query": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Node Exporter Full", + "uid": "rYdddlPWk", + "version": 2, + "weekStart": "" +} diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/nodejs-app-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/nodejs-app-dashboard.json new file mode 100644 index 00000000..5a95578a --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/nodejs-app-dashboard.json @@ -0,0 +1,943 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "node.js prometheus client basic metrics", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 11159, + "graphTooltip": 0, + "id": 15, + "links": [], + "liveNow": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 10, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(process_cpu_user_seconds_total{instance=~\"$instance\"}[2m]) * 100", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "User CPU - {{instance}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(process_cpu_system_seconds_total{instance=~\"$instance\"}[2m]) * 100", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Sys CPU - {{instance}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Process CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 9, + "x": 10, + "y": 0 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_eventloop_lag_seconds{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Event Loop Lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 19, + "y": 0 + }, + "id": 2, + "interval": "", + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum(nodejs_version_info{instance=~\"$instance\"}) by (version)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{version}}", + "refId": "A" + } + ], + "title": "Node.js Version", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#F2495C", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 19, + "y": 3 + }, + "id": 4, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum(changes(process_start_time_seconds{instance=~\"$instance\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Process Restart Times", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 16, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "process_resident_memory_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Process Memory - {{instance}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_heap_size_total_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Heap Total - {{instance}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_heap_size_used_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Heap Used - {{instance}}", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_external_memory_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "External Memory - {{instance}}", + "refId": "D" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Process Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_active_handles_total{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Active Handler - {{instance}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_active_requests_total{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Active Request - {{instance}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Active Handlers/Requests Total", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_heap_space_size_total_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Heap Total - {{instance}} - {{space}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Heap Total Detail", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 14 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_heap_space_size_used_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Heap Used - {{instance}} - {{space}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Heap Used Detail", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 14 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "nodejs_heap_space_size_available_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Heap Used - {{instance}} - {{space}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Heap Available Detail", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": "10s", + "schemaVersion": 38, + "tags": [ + "nodejs" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(nodejs_version_info, instance)", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [], + "query": "label_values(nodejs_version_info, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "NodeJS Application Dashboard", + "uid": "PTSqcpJWk", + "version": 3, + "weekStart": "" +} diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/postgres-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/postgres-dashboard.json new file mode 100644 index 00000000..18e641e6 --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/postgres-dashboard.json @@ -0,0 +1,2963 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "This dashboard works with postgres_exporter for prometheus", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 9628, + "graphTooltip": 0, + "id": 15, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 34, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "General Counters", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 36, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_static{release=\"$release\", instance=\"$instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{short_version}}", + "refId": "A" + } + ], + "title": "Version", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "start time of the process", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "dateTimeFromNow" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 28, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_postmaster_start_time_seconds{release=\"$release\", instance=\"$instance\"} * 1000", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Start Time", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 10, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "SUM(pg_stat_database_tup_fetched{datname=~\"$datname\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 4 + } + ], + "title": "Current fetch data", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 11, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "SUM(pg_stat_database_tup_inserted{release=\"$release\", datname=~\"$datname\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 4 + } + ], + "title": "Current insert data", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 12, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "SUM(pg_stat_database_tup_updated{datname=~\"$datname\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 4 + } + ], + "title": "Current update data", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 38, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_settings_max_connections{release=\"$release\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Max Connections", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 32, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "Settings", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 4 + }, + "id": 40, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_settings_shared_buffers_bytes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Shared Buffers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 4 + }, + "id": 42, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_settings_effective_cache_size_bytes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Effective Cache", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 6, + "y": 4 + }, + "id": 44, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_settings_maintenance_work_mem_bytes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Maintenance Work Mem", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 9, + "y": 4 + }, + "id": 46, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_settings_work_mem_bytes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Work Mem", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 12, + "y": 4 + }, + "id": 48, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_settings_max_wal_size_bytes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Max WAL Size", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 15, + "y": 4 + }, + "id": 50, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_settings_random_page_cost{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Random Page Cost", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 18, + "y": 4 + }, + "id": 52, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_settings_seq_page_cost", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Seq Page Cost", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 20, + "y": 4 + }, + "id": 54, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_settings_max_worker_processes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Max Worker Processes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 22, + "y": 4 + }, + "id": 56, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_settings_max_parallel_workers{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Max Parallel Workers", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 30, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "Database Stats", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 1, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 3, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_stat_activity_count{datname=~\"$datname\", instance=~\"$instance\", state=\"active\"} !=0", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{datname}}, s: {{state}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Active sessions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 8 + }, + "hiddenSeries": false, + "id": 60, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(pg_stat_database_xact_commit{instance=\"$instance\", datname=~\"$datname\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{datname}} commits", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(pg_stat_database_xact_rollback{instance=\"$instance\", datname=~\"$datname\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{datname}} rollbacks", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Transactions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 8 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_stat_database_tup_updated{datname=~\"$datname\", instance=~\"$instance\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Update data", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 15 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_stat_database_tup_fetched{datname=~\"$datname\", instance=~\"$instance\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Fetch data (SELECT)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 15 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_stat_database_tup_inserted{datname=~\"$datname\", instance=~\"$instance\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Insert data", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 0, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 15 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_locks_count{datname=~\"$datname\", instance=~\"$instance\", mode=~\"$mode\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}},{{mode}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Lock tables", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 22 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "total", + "sortDesc": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_stat_database_tup_returned{datname=~\"$datname\", instance=~\"$instance\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Return data", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 22 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_stat_activity_count{datname=~\"$datname\", instance=~\"$instance\", state=~\"idle|idle in transaction|idle in transaction (aborted)\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}}, s: {{state}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Idle sessions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 22 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_stat_database_tup_deleted{datname=~\"$datname\", instance=~\"$instance\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{datname}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Delete data", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 2, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 29 + }, + "hiddenSeries": false, + "id": 62, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "pg_stat_database_blks_hit{instance=\"$instance\", datname=~\"$datname\"} / (pg_stat_database_blks_read{instance=\"$instance\", datname=~\"$datname\"} + pg_stat_database_blks_hit{instance=\"$instance\", datname=~\"$datname\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ datname }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Cache Hit Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 4, + "format": "percentunit", + "label": "", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 29 + }, + "hiddenSeries": false, + "id": 64, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(pg_stat_bgwriter_buffers_backend{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "buffers_backend", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(pg_stat_bgwriter_buffers_alloc{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "buffers_alloc", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "backend_fsync", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(pg_stat_bgwriter_buffers_checkpoint{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "buffers_checkpoint", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(pg_stat_bgwriter_buffers_clean{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "buffers_clean", + "refId": "E" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Buffers (bgwriter)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "decimals": 0, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 29 + }, + "hiddenSeries": false, + "id": 66, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(pg_stat_database_conflicts{instance=\"$instance\", datname=~\"$datname\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{datname}} conflicts", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(pg_stat_database_deadlocks{instance=\"$instance\", datname=~\"$datname\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{datname}} deadlocks", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Conflicts/Deadlocks", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total amount of data written to temporary files by queries in this database. All temporary files are counted, regardless of why the temporary file was created, and regardless of the log_temp_files setting.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 36 + }, + "hiddenSeries": false, + "id": 68, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(pg_stat_database_temp_bytes{instance=\"$instance\", datname=~\"$datname\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{datname}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Temp File (Bytes)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 16, + "x": 8, + "y": 36 + }, + "hiddenSeries": false, + "id": 70, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(pg_stat_bgwriter_checkpoint_write_time{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "write_time - Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk.", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "irate(pg_stat_bgwriter_checkpoint_sync_time{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "sync_time - Total amount of time that has been spent in the portion of checkpoint processing where files are synchronized to disk.", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Checkpoint Stats", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": "10s", + "schemaVersion": 38, + "tags": [ + "postgres", + "db", + "stats" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "auto": true, + "auto_count": 200, + "auto_min": "1s", + "current": { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + "hide": 0, + "label": "Interval", + "name": "interval", + "options": [ + { + "selected": true, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "1s", + "value": "1s" + }, + { + "selected": false, + "text": "5s", + "value": "5s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + } + ], + "query": "1s,5s,1m,5m,1h,6h,1d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "query_result(pg_exporter_last_scrape_duration_seconds)", + "refresh": 2, + "regex": "/.*kubernetes_namespace=\"([^\"]+).*/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Release", + "multi": false, + "name": "release", + "options": [], + "query": "query_result(pg_exporter_last_scrape_duration_seconds{kubernetes_namespace=\"$namespace\"})", + "refresh": 2, + "regex": "/.*release=\"([^\"]+)/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": "query_result(pg_up{release=\"$release\"})", + "refresh": 1, + "regex": "/.*instance=\"([^\"]+).*/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Database", + "multi": true, + "name": "datname", + "options": [], + "query": "label_values(datname)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Lock table", + "multi": true, + "name": "mode", + "options": [], + "query": "label_values({mode=~\"accessexclusivelock|accesssharelock|exclusivelock|rowexclusivelock|rowsharelock|sharelock|sharerowexclusivelock|shareupdateexclusivelock\"}, mode)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "PostgreSQL Database", + "uid": "000000039", + "version": 1, + "weekStart": "" +} diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/prometheus-blackbox-exporter.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/prometheus-blackbox-exporter.json new file mode 100644 index 00000000..638c4d5b --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/prometheus-blackbox-exporter.json @@ -0,0 +1,1097 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Prometheus Blackbox Exporter Overview", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 7587, + "graphTooltip": 0, + "id": 29, + "links": [], + "liveNow": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 138, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "probe_duration_seconds{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Global Probe Duration", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 15, + "panels": [], + "repeat": "target", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "$target status", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "text": "DOWN" + }, + "1": { + "text": "UP" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "#299c46", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 9 + }, + "id": 2, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "probe_success{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Status", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 10, + "x": 4, + "y": 9 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "probe_http_duration_seconds{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ phase }}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "HTTP Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 10, + "x": 14, + "y": 9 + }, + "hiddenSeries": false, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "probe_duration_seconds{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "seconds", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Probe Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "0": { + "text": "N/A" + }, + "1": { + "text": "YES" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 201 + }, + { + "color": "#d44a3a", + "value": 399 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 11 + }, + "id": 20, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "probe_http_status_code{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "HTTP Status Code", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 13 + }, + "id": 27, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "probe_http_version{instance=~\"$target\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "HTTP Version", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "text": "NO" + }, + "1": { + "text": "YES" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 + }, + { + "color": "#299c46", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 15 + }, + "id": 18, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "probe_http_ssl{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "SSL", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "0": { + "text": "NO" + }, + "1": { + "text": "YES" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 + }, + { + "color": "#299c46", + "value": 1209600 + } + ] + }, + "unit": "dtdurations" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 10, + "x": 4, + "y": 15 + }, + "id": 19, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "probe_ssl_earliest_cert_expiry{instance=~\"$target\"} - time()", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "SSL Expiry", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 5, + "x": 14, + "y": 15 + }, + "id": 23, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "avg(probe_duration_seconds{instance=~\"$target\"})", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Average Probe Duration", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 5, + "x": 19, + "y": 15 + }, + "id": 24, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "avg(probe_dns_lookup_time_seconds{instance=~\"$target\"})", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Average DNS Lookup", + "type": "stat" + } + ], + "refresh": "10s", + "schemaVersion": 38, + "tags": [ + "blackbox", + "prometheus" + ], + "templating": { + "list": [ + { + "auto": true, + "auto_count": 10, + "auto_min": "10s", + "current": { + "selected": false, + "text": "10s", + "value": "10s" + }, + "hide": 0, + "label": "Interval", + "name": "interval", + "options": [ + { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "5s", + "value": "5s" + }, + { + "selected": true, + "text": "10s", + "value": "10s" + }, + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "5s,10s,30s,1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "target", + "options": [], + "query": "label_values(probe_success, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Prometheus Blackbox Exporter", + "uid": "xtkCtBkiz", + "version": 2, + "weekStart": "" +} diff --git a/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json b/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json new file mode 100644 index 00000000..fe5fd244 --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json @@ -0,0 +1,2187 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [ + { + "asDropdown": false, + "icon": "dashboard", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Blackbox", + "tooltip": "", + "type": "link", + "url": "/d/xtkCtBkiz/prometheus-blackbox-exporter" + }, + { + "asDropdown": false, + "icon": "dashboard", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "PG", + "tooltip": "", + "type": "link", + "url": "/d/000000039/postgresql-database" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 25, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_indexed\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Latest indexed block", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head block number of the upstream endpoint", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 11, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_upstream_block_number{job=~\"$job\", instance=~\"$watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Upstream head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Chain head from an external public endpoint", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 12, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"$target_chain\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "External head ($target_chain)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-green", + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-green", + "value": null + }, + { + "color": "red", + "value": 35 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "last_block_process_duration_seconds{job=~\"$job\", instance=~\"$watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Last block process duration (s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "orange", + "index": 2, + "text": "LAZY" + }, + "1": { + "color": "semi-dark-green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"server\", field=\"is_active\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher kind", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "latest_upstream_block_number{job=~\"$job\", instance=~\"$watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_upstream", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_upstream_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 8 + }, + { + "color": "red", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 4 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 25 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "latest_block_number{job=\"chain_heads\", chain=\"$target_chain\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "diff_external", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "latest_block_number", + "operator": "-", + "right": "sync_status_block_number" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Watcher sync status block numbers", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 4, + "y": 6 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_processed_block_number{job=~\"$job\", instance=~\"$watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "latest_processed", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_canonical\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"intial_indexed\"}", + "hide": false, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "C" + } + ], + "title": "Sync status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-green", + "index": 1, + "text": "REAL TIME" + }, + "1": { + "color": "orange", + "index": 2, + "text": "HISTORICAL" + } + }, + "type": "value" + }, + { + "options": { + "match": "null+nan", + "result": { + "color": "semi-dark-red", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2.5, + "w": 4, + "x": 0, + "y": 6.5 + }, + "id": 16, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "is_syncing_historical_blocks{job=~\"$job\", instance=~\"$watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Watcher sync mode", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "last_block_num_events_total{job=~\"$job\", instance=~\"$watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number of events in the last block", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of entries in the block_progress and event table", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "dark-yellow", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 9 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "block_progress_total{job=~\"$job\", instance=~\"$watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "block_progress", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "event_total{job=~\"$job\", instance=~\"$watcher\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "event", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "DB indexed count ", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 9 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "database_size_bytes{job=~\"$job\", instance=~\"$watcher\", type=\"watcher\"} / 1024 / 1024", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "database_size_bytes{job=~\"$job\", instance=~\"$watcher\", type=\"job-queue\"} / 1024 / 1024", + "hide": false, + "instant": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "B" + } + ], + "title": "DB size (MB)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=~\"$job\", instance=~\"$watcher\", state=~\"active|retry|failed|cancelled|expired\", name=\"block-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num block-processing jobs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of pgboss jobs in the db", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "pgboss_jobs_total{job=~\"$job\", instance=~\"$watcher\", state=~\"active|retry|failed|cancelled|expired\", name=\"event-processing\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num event-processing jobs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 18 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"upstream\", field=\"is_using_rpc_client\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"upstream\", field=\"is_fevm\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"server\", field=\"rpc_supports_block_hash\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"upstream\", field=\"filter_logs_by_addresses\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"upstream\", field=\"filter_logs_by_topics\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + } + ], + "title": "Watcher config (upstream eth server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "watcher_config_info" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 18 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"server\", field=\"is_subgraph_watcher\"}", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"server\", field=\"max_events_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"server\", field=\"clear_entities_cache_interval\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"server\", field=\"max_simultaneous_requests\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"server\", field=\"max_request_queue_limit\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "F" + } + ], + "title": "Watcher config (server)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "is_subgraph_watcher" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "#757575", + "index": 0, + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 18 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^(?!Time$).*/", + "values": false + }, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"jobqueue\", field=\"num_events_in_batch\"}", + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"jobqueue\", field=\"block_delay_seconds\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"jobqueue\", field=\"use_block_ranges\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"jobqueue\", field=\"historical_logs_block_range\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "watcher_config_info{job=~\"$job\", instance=~\"$watcher\", category=\"jobqueue\", field=\"historical_max_fetch_ahead\"}", + "hide": false, + "instant": false, + "legendFormat": "{{field}}", + "range": true, + "refId": "E" + } + ], + "title": "Watcher config (job queue)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "boolean", + "targetField": "use_block_ranges" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of subgraph entities loaded in event processing", + "hide": true, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 23 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "event_processing_load_entity_total{job=~\"$job\", instance=~\"$watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num subgraph entities loaded", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of cache hits for subgraph entites in event processing ", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 23 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "event_processing_load_entity_cache_hit_total{job=~\"$job\", instance=~\"$watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num subgraph entities cache hits", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of subgraph entities currently present in pruned region of cache", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 23 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "cached_pruned_entities_total{job=~\"$job\", instance=~\"$watcher\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num subgraph entities cached (pruned region)", + "type": "timeseries" + } + ], + "repeat": "watcher", + "repeatDirection": "h", + "title": "$watcher", + "type": "row" + } + ], + "refresh": "10s", + "schemaVersion": 38, + "tags": [ + "watcher" + ], + "templating": { + "list": [ + { + "current": { + "isNone": true, + "selected": false, + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(watcher_config_info,job)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "job", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(watcher_config_info,job)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "isNone": true, + "selected": false, + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values({job=\"$job\"},instance)", + "hide": 0, + "includeAll": true, + "label": "", + "multi": false, + "name": "watcher", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({job=\"$job\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values({instance=~\"$watcher\"},chain)", + "hide": 2, + "includeAll": false, + "multi": false, + "name": "target_chain", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({instance=~\"$watcher\"},chain)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Watchers", + "version": 1, + "weekStart": "" +} diff --git a/stack_orchestrator/data/config/monitoring/grafana/provisioning/alerting/contactpoints.yml b/stack_orchestrator/data/config/monitoring/grafana/provisioning/alerting/contactpoints.yml new file mode 100644 index 00000000..5e924946 --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/provisioning/alerting/contactpoints.yml @@ -0,0 +1,14 @@ +# https://www.clever-cloud.com/blog/features/2021/12/03/slack-alerts-for-grafana/ + +apiVersion: 1 + +contactPoints: + - orgId: 1 + name: SlackNotifier + receivers: + - uid: a71b06e3-58b6-41fe-af65-fbbb29653951 + type: slack + settings: + # Slack hook URL (see https://api.slack.com/messaging/webhooks) + url: + disableResolveMessage: false diff --git a/stack_orchestrator/data/config/monitoring/grafana/provisioning/alerting/policies.yml b/stack_orchestrator/data/config/monitoring/grafana/provisioning/alerting/policies.yml new file mode 100644 index 00000000..b1beb31d --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/provisioning/alerting/policies.yml @@ -0,0 +1,15 @@ +# https://grafana.com/docs/grafana/latest/alerting/alerting-rules/create-notification-policy/ + +apiVersion: 1 + +policies: + - orgId: 1 + receiver: grafana-default-email + group_by: + - grafana_folder + - alertname + routes: + - receiver: SlackNotifier + object_matchers: + # Add matchers below + # - ['grafana_folder', '=', 'MyAlerts'] diff --git a/stack_orchestrator/data/config/monitoring/grafana/provisioning/dashboards/dashboards.yml b/stack_orchestrator/data/config/monitoring/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 00000000..9166cbff --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +providers: + - name: dashboards + type: file + updateIntervalSeconds: 10 + allowUiUpdates: true + options: + path: /etc/grafana/dashboards + foldersFromFilesStructure: true diff --git a/stack_orchestrator/data/config/monitoring/grafana/provisioning/datasources/prometheus.yml b/stack_orchestrator/data/config/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 00000000..b502d2e0 --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,16 @@ +apiVersion: 1 + +datasources: + - id: 1 + orgId: 1 + name: Prometheus + type: prometheus + typeName: Prometheus + typeLogoUrl: public/app/plugins/datasource/prometheus/img/prometheus_logo.svg + access: proxy + url: http://prometheus:9090 + isDefault: true + jsonData: + httpMethod: POST + version: 1 + editable: true diff --git a/stack_orchestrator/data/config/monitoring/postgres-exporter.yml b/stack_orchestrator/data/config/monitoring/postgres-exporter.yml new file mode 100644 index 00000000..72b8ffa1 --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/postgres-exporter.yml @@ -0,0 +1,8 @@ +auth_modules: + foo: + type: userpass + userpass: + username: username + password: password + options: + sslmode: disable diff --git a/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml b/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml new file mode 100644 index 00000000..2d6a44da --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml @@ -0,0 +1,67 @@ +global: + scrape_interval: 10s + evaluation_interval: 15s + +rule_files: + # - "first.rules" + # - "second.rules" + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['localhost:9090'] + + - job_name: node + static_configs: + # Add node-exporter targets to be monitored below + # - targets: ['example-host:9100'] + # labels: + # instance: 'my-host' + + - job_name: 'blackbox' + scrape_interval: 10s + metrics_path: /probe + params: + module: [http_2xx] + static_configs: + # Add URLs to be monitored below + - targets: + # - https://github.com + relabel_configs: + - source_labels: [__address__] + regex: (.*)(:80)? + target_label: __param_target + - source_labels: [__param_target] + regex: (.*) + target_label: instance + replacement: ${1} + - source_labels: [] + regex: .* + target_label: __address__ + replacement: blackbox:9115 + + - job_name: chain_heads + scrape_interval: 10s + metrics_path: /metrics + scheme: http + static_configs: + - targets: ['chain-head-exporter:5000'] + + - job_name: 'postgres' + scrape_interval: 30s + scrape_timeout: 30s + static_configs: + # Add DB targets below + # - targets: [example-server:5432] + # labels: + # instance: 'example-label' + metrics_path: /probe + params: + auth_module: [foo] + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: postgres-exporter:9187 diff --git a/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml b/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml new file mode 100644 index 00000000..9df9472f --- /dev/null +++ b/stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml @@ -0,0 +1,933 @@ +# https://grafana.com/docs/grafana/latest/alerting/alerting-rules/create-grafana-managed-rule/ + +apiVersion: 1 + +groups: + - orgId: 1 + name: watcher + folder: WatcherAlerts + interval: 30s + rules: + # Azimuth + - uid: azimuth_diff_external + title: azimuth_watcher_head_tracking + condition: condition + data: + - refId: diff + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + disableTextWrap: false + editorMode: code + expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"} + fullMetaSearch: false + includeNullMetadata: true + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: diff + useBackend: false + - refId: latest_external + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: latest_block_number{chain="ethereum"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_external + - refId: latest_indexed + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_indexed + - refId: condition + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: ${diff} >= 16 + intervalMs: 1000 + maxDataPoints: 43200 + refId: condition + type: math + noDataState: Alerting + execErrState: Alerting + for: 15m + annotations: + summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} + isPaused: false + - uid: censures_diff_external + title: censures_watcher_head_tracking + condition: condition + data: + - refId: diff + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + disableTextWrap: false + editorMode: code + expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"} + fullMetaSearch: false + includeNullMetadata: true + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: diff + useBackend: false + - refId: latest_external + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: latest_block_number{chain="ethereum"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_external + - refId: latest_indexed + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_indexed + - refId: condition + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: ${diff} >= 16 + intervalMs: 1000 + maxDataPoints: 43200 + refId: condition + type: math + noDataState: Alerting + execErrState: Alerting + for: 15m + annotations: + summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} + isPaused: false + - uid: claims_diff_external + title: claims_watcher_head_tracking + condition: condition + data: + - refId: diff + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + disableTextWrap: false + editorMode: code + expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"} + fullMetaSearch: false + includeNullMetadata: true + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: diff + useBackend: false + - refId: latest_external + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: latest_block_number{chain="ethereum"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_external + - refId: latest_indexed + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_indexed + - refId: condition + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: ${diff} >= 16 + intervalMs: 1000 + maxDataPoints: 43200 + refId: condition + type: math + noDataState: Alerting + execErrState: Alerting + for: 15m + annotations: + summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} + isPaused: false + - uid: conditional_star_release_diff_external + title: conditional_star_release_watcher_head_tracking + condition: condition + data: + - refId: diff + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + disableTextWrap: false + editorMode: code + expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"} + fullMetaSearch: false + includeNullMetadata: true + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: diff + useBackend: false + - refId: latest_external + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: latest_block_number{chain="ethereum"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_external + - refId: latest_indexed + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_indexed + - refId: condition + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: ${diff} >= 16 + intervalMs: 1000 + maxDataPoints: 43200 + refId: condition + type: math + noDataState: Alerting + execErrState: Alerting + for: 15m + annotations: + summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} + isPaused: false + - uid: delegated_sending_diff_external + title: delegated_sending_watcher_head_tracking + condition: condition + data: + - refId: diff + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + disableTextWrap: false + editorMode: code + expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"} + fullMetaSearch: false + includeNullMetadata: true + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: diff + useBackend: false + - refId: latest_external + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: latest_block_number{chain="ethereum"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_external + - refId: latest_indexed + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_indexed + - refId: condition + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: ${diff} >= 16 + intervalMs: 1000 + maxDataPoints: 43200 + refId: condition + type: math + noDataState: Alerting + execErrState: Alerting + for: 15m + annotations: + summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} + isPaused: false + - uid: ecliptic_diff_external + title: ecliptic_watcher_head_tracking + condition: condition + data: + - refId: diff + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + disableTextWrap: false + editorMode: code + expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"} + fullMetaSearch: false + includeNullMetadata: true + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: diff + useBackend: false + - refId: latest_external + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: latest_block_number{chain="ethereum"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_external + - refId: latest_indexed + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_indexed + - refId: condition + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: ${diff} >= 16 + intervalMs: 1000 + maxDataPoints: 43200 + refId: condition + type: math + noDataState: Alerting + execErrState: Alerting + for: 15m + annotations: + summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} + isPaused: false + - uid: linear_star_release_diff_external + title: linear_star_release_watcher_head_tracking + condition: condition + data: + - refId: diff + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + disableTextWrap: false + editorMode: code + expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="linear_star_release", kind="latest_indexed"} + fullMetaSearch: false + includeNullMetadata: true + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: diff + useBackend: false + - refId: latest_external + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: latest_block_number{chain="ethereum"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_external + - refId: latest_indexed + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_indexed + - refId: condition + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: ${diff} >= 16 + intervalMs: 1000 + maxDataPoints: 43200 + refId: condition + type: math + noDataState: Alerting + execErrState: Alerting + for: 15m + annotations: + summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} + isPaused: false + - uid: polls_diff_external + title: polls_watcher_head_tracking + condition: condition + data: + - refId: diff + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + disableTextWrap: false + editorMode: code + expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"} + fullMetaSearch: false + includeNullMetadata: true + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: diff + useBackend: false + - refId: latest_external + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: latest_block_number{chain="ethereum"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_external + - refId: latest_indexed + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_indexed + - refId: condition + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: ${diff} >= 16 + intervalMs: 1000 + maxDataPoints: 43200 + refId: condition + type: math + noDataState: Alerting + execErrState: Alerting + for: 15m + annotations: + summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} + isPaused: false + + # Sushi + - uid: sushiswap_diff_external + title: sushiswap_watcher_head_tracking + condition: condition + data: + - refId: diff + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + disableTextWrap: false + editorMode: code + expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"} + fullMetaSearch: false + includeNullMetadata: true + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: diff + useBackend: false + - refId: latest_external + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: latest_block_number{chain="filecoin"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_external + - refId: latest_indexed + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_indexed + - refId: condition + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: ${diff} >= 16 + intervalMs: 1000 + maxDataPoints: 43200 + refId: condition + type: math + noDataState: Alerting + execErrState: Alerting + for: 15m + annotations: + summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} + isPaused: false + - uid: merkl_sushiswap_diff_external + title: merkl_sushiswap_watcher_head_tracking + condition: condition + data: + - refId: diff + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + disableTextWrap: false + editorMode: code + expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"} + fullMetaSearch: false + includeNullMetadata: true + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: diff + useBackend: false + - refId: latest_external + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: latest_block_number{chain="filecoin"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_external + - refId: latest_indexed + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + datasource: + type: prometheus + uid: PBFA97CFB590B2093 + editorMode: code + expr: sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"} + hide: false + instant: true + legendFormat: __auto + range: false + refId: latest_indexed + - refId: condition + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: ${diff} >= 16 + intervalMs: 1000 + maxDataPoints: 43200 + refId: condition + type: math + noDataState: Alerting + execErrState: Alerting + for: 15m + annotations: + summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }} + isPaused: false diff --git a/stack_orchestrator/data/config/osmosis/.env.production b/stack_orchestrator/data/config/osmosis/.env.production new file mode 100644 index 00000000..2fd24c8e --- /dev/null +++ b/stack_orchestrator/data/config/osmosis/.env.production @@ -0,0 +1,3 @@ +NEXT_PUBLIC_COINGECKO_URL=https://api.coingecko.com + +NEXT_PUBLIC_ENABLE_FEATURES=true diff --git a/stack_orchestrator/data/config/osmosis/build-app-for-urbit.sh b/stack_orchestrator/data/config/osmosis/build-app-for-urbit.sh new file mode 100755 index 00000000..86e52178 --- /dev/null +++ b/stack_orchestrator/data/config/osmosis/build-app-for-urbit.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +# Check and exit if a deployment already exists (on restarts) +if [ -d /app-builds/osmosis/build ]; then + echo "Build already exists, remove volume to rebuild" + exit 0 +fi + +yarn build:static +./build-urbit.sh + +# Move build to app-builds +mkdir -p /app-builds/osmosis +cp -r ./out /app-builds/osmosis/build + +cp -r mar /app-builds/osmosis/ +cp desk.docket-0 /app-builds/osmosis/ diff --git a/stack_orchestrator/data/config/osmosis/build-app.sh b/stack_orchestrator/data/config/osmosis/build-app.sh new file mode 100755 index 00000000..9d13bbc3 --- /dev/null +++ b/stack_orchestrator/data/config/osmosis/build-app.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +# Check and exit if a deployment already exists (on restarts) +if [ -d /app-builds/osmosis/build ]; then + echo "Build already exists, remove volume to rebuild" + exit 0 +fi + +yarn build:static + +# Move build to app-builds +mkdir -p /app-builds/osmosis +cp -r ./out /app-builds/osmosis/build diff --git a/stack_orchestrator/data/config/osmosis/nginx/web-app.conf b/stack_orchestrator/data/config/osmosis/nginx/web-app.conf new file mode 100644 index 00000000..784df3b2 --- /dev/null +++ b/stack_orchestrator/data/config/osmosis/nginx/web-app.conf @@ -0,0 +1,16 @@ +server { + listen 80; + listen [::]:80; + server_name _; + + error_page 500 502 503 504 /50x.html; + + location / { + root /usr/share/nginx/osmosis/build; + index index.html index.htm index.nginx-debian.html; + + # First attempt to serve request as file, then as html, + # then as directory, then fall back to displaying a 404. + try_files $uri $uri.html $uri/ /index.html =404; + } +} diff --git a/stack_orchestrator/data/config/osmosis/urbit-files/desk.docket-0 b/stack_orchestrator/data/config/osmosis/urbit-files/desk.docket-0 new file mode 100644 index 00000000..232fe70f --- /dev/null +++ b/stack_orchestrator/data/config/osmosis/urbit-files/desk.docket-0 @@ -0,0 +1,10 @@ +:~ title+'Osmosis' + info+'Osmosis DEX Frontend, built and maintained by Laconic' + color+0xcd.75df + image+'https://altcoinsbox.com/wp-content/uploads/2023/03/osmosis-logo.png' + base+'osmosis' + glob-http+['REPLACE_WITH_GLOB_URL' REPLACE_WITH_GLOB_HASH] + version+[0 0 2] + website+'https://osmosis.zone/' + license+'MIT' +== diff --git a/stack_orchestrator/data/config/osmosis/urbit-files/mar/ico.hoon b/stack_orchestrator/data/config/osmosis/urbit-files/mar/ico.hoon new file mode 100644 index 00000000..e862b9b9 --- /dev/null +++ b/stack_orchestrator/data/config/osmosis/urbit-files/mar/ico.hoon @@ -0,0 +1,12 @@ +|_ dat=@ +++ grow + |% + ++ mime [/image/x-icon (as-octs:mimes:html dat)] + -- +++ grab + |% + ++ mime |=([p=mite q=octs] q.q) + ++ noun @ + -- +++ grad %mime +-- diff --git a/stack_orchestrator/data/config/osmosis/urbit-files/mar/jpg.hoon b/stack_orchestrator/data/config/osmosis/urbit-files/mar/jpg.hoon new file mode 100644 index 00000000..81a77d46 --- /dev/null +++ b/stack_orchestrator/data/config/osmosis/urbit-files/mar/jpg.hoon @@ -0,0 +1,12 @@ +|_ dat=@ +++ grow + |% + ++ mime [/image/jpeg (as-octs:mimes:html dat)] + -- +++ grab + |% + ++ mime |=([p=mite q=octs] q.q) + ++ noun @ + -- +++ grad %mime +-- diff --git a/stack_orchestrator/data/config/osmosis/urbit-files/mar/map.hoon b/stack_orchestrator/data/config/osmosis/urbit-files/mar/map.hoon new file mode 100644 index 00000000..e5d61eee --- /dev/null +++ b/stack_orchestrator/data/config/osmosis/urbit-files/mar/map.hoon @@ -0,0 +1,18 @@ +:: +:::: /hoon/map/mar + :: Mark for js source maps +/? 310 +:: +=, eyre +|_ mud=@ +++ grow + |% + ++ mime [/application/octet-stream (as-octs:mimes:html (@t mud))] + -- +++ grab + |% :: convert from + ++ mime |=([p=mite q=octs] (@t q.q)) + ++ noun cord :: clam from %noun + -- +++ grad %mime +-- diff --git a/stack_orchestrator/data/config/osmosis/urbit-files/mar/webp.hoon b/stack_orchestrator/data/config/osmosis/urbit-files/mar/webp.hoon new file mode 100644 index 00000000..1909d0dd --- /dev/null +++ b/stack_orchestrator/data/config/osmosis/urbit-files/mar/webp.hoon @@ -0,0 +1,12 @@ +|_ dat=@ +++ grow + |% + ++ mime [/image/webp (as-octs:mimes:html dat)] + -- +++ grab + |% + ++ mime |=([p=mite q=octs] q.q) + ++ noun @ + -- +++ grad %mime +-- diff --git a/stack_orchestrator/data/config/proxy-server/run.sh b/stack_orchestrator/data/config/proxy-server/run.sh new file mode 100755 index 00000000..9e8dc7f5 --- /dev/null +++ b/stack_orchestrator/data/config/proxy-server/run.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +if [ "$ENABLE_PROXY" = "true" ]; then + echo "Proxy server enabled" + yarn proxy +else + echo "Proxy server disabled, exiting" + exit 0 +fi diff --git a/stack_orchestrator/data/config/uniswap-interface/build-app.sh b/stack_orchestrator/data/config/uniswap-interface/build-app.sh new file mode 100755 index 00000000..81e306b6 --- /dev/null +++ b/stack_orchestrator/data/config/uniswap-interface/build-app.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +# Check and exit if a deployment already exists (on restarts) +if [ -d /app-builds/uniswap/build ]; then + echo "Build already exists, remove volume to rebuild" + exit 0 +fi + +yarn build + +# Copy over build and other files to app-builds for urbit deployment +mkdir -p /app-builds/uniswap +cp -r ./build /app-builds/uniswap/ + +cp -r mar /app-builds/uniswap/ +cp desk.docket-0 /app-builds/uniswap/ diff --git a/stack_orchestrator/data/config/uniswap-interface/urbit-files/desk.docket-0 b/stack_orchestrator/data/config/uniswap-interface/urbit-files/desk.docket-0 new file mode 100644 index 00000000..6256c41b --- /dev/null +++ b/stack_orchestrator/data/config/uniswap-interface/urbit-files/desk.docket-0 @@ -0,0 +1,10 @@ +:~ title+'Uniswap' + info+'Self-hosted uniswap frontend.' + color+0xcd.75df + image+'https://logowik.com/content/uploads/images/uniswap-uni7403.jpg' + base+'uniswap' + glob-http+['REPLACE_WITH_GLOB_URL' REPLACE_WITH_GLOB_HASH] + version+[0 0 1] + website+'https://uniswap.org/' + license+'MIT' +== diff --git a/stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/map.hoon b/stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/map.hoon new file mode 100644 index 00000000..e5d61eee --- /dev/null +++ b/stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/map.hoon @@ -0,0 +1,18 @@ +:: +:::: /hoon/map/mar + :: Mark for js source maps +/? 310 +:: +=, eyre +|_ mud=@ +++ grow + |% + ++ mime [/application/octet-stream (as-octs:mimes:html (@t mud))] + -- +++ grab + |% :: convert from + ++ mime |=([p=mite q=octs] (@t q.q)) + ++ noun cord :: clam from %noun + -- +++ grad %mime +-- diff --git a/stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/ttf.hoon b/stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/ttf.hoon new file mode 100644 index 00000000..2d29193e --- /dev/null +++ b/stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/ttf.hoon @@ -0,0 +1,12 @@ +|_ dat=octs +++ grow + |% + ++ mime [/font/ttf dat] + -- +++ grab + |% + ++ mime |=([=mite =octs] octs) + ++ noun octs + -- +++ grad %mime +-- diff --git a/stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/woff.hoon b/stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/woff.hoon new file mode 100644 index 00000000..2933aea7 --- /dev/null +++ b/stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/woff.hoon @@ -0,0 +1,12 @@ +|_ dat=octs +++ grow + |% + ++ mime [/font/woff dat] + -- +++ grab + |% + ++ mime |=([=mite =octs] octs) + ++ noun octs + -- +++ grad %mime +-- diff --git a/stack_orchestrator/data/config/urbit/deploy-app-to-remote-urbit.sh b/stack_orchestrator/data/config/urbit/deploy-app-to-remote-urbit.sh new file mode 100755 index 00000000..18019949 --- /dev/null +++ b/stack_orchestrator/data/config/urbit/deploy-app-to-remote-urbit.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# $1: Remote user host +# $2: App name (eg. uniswap) +# $3: Assets dir path (local) for app (eg. /home/user/myapp/urbit-files) +# $4: Remote Urbit ship's pier dir path (eg. /home/user/zod) +# $5: Glob file URL (eg. https://xyz.com/glob-0vabcd.glob) +# $6: Glob file hash (eg. 0vabcd) + +if [ "$#" -ne 6 ]; then + echo "Incorrect number of arguments" + echo "Usage: $0 " + exit 1 +fi + +remote_user_host="$1" +app_name=$2 +app_assets_folder=$3 +remote_pier_folder="$4" +glob_url="$5" +glob_hash="$6" + +installation_script="./install-urbit-app.sh" + +# Copy over the assets to remote machine in a tmp dir +remote_app_assets_folder=/tmp/urbit-app-assets/$app_name +ssh "$remote_user_host" "mkdir -p $remote_app_assets_folder" +scp -r $app_assets_folder/* $remote_user_host:$remote_app_assets_folder + +# Run the installation script +ssh "$remote_user_host" "bash -s $app_name $remote_app_assets_folder '${glob_url}' $glob_hash $remote_pier_folder" < "$installation_script" + +# Remove the tmp assets dir +ssh "$remote_user_host" "rm -rf $remote_app_assets_folder" diff --git a/stack_orchestrator/data/config/urbit/deploy-app.sh b/stack_orchestrator/data/config/urbit/deploy-app.sh new file mode 100755 index 00000000..bde6cd6f --- /dev/null +++ b/stack_orchestrator/data/config/urbit/deploy-app.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +if [ -z "$CERC_URBIT_APP" ]; then + echo "CERC_URBIT_APP not set, exiting" + exit 0 +fi + +echo "Creating Urbit application for ${CERC_URBIT_APP}" + +app_desk_dir=/urbit/zod/${CERC_URBIT_APP} +if [ -d ${app_desk_dir} ]; then + echo "Desk dir already exists for ${CERC_URBIT_APP}, skipping deployment..." + exit 0 +fi + +app_build=/app-builds/${CERC_URBIT_APP}/build +app_mark_files=/app-builds/${CERC_URBIT_APP}/mar +app_docket_file=/app-builds/${CERC_URBIT_APP}/desk.docket-0 + +echo "Reading app build from ${app_build}" +echo "Reading additional mark files from ${app_mark_files}" +echo "Reading docket file ${app_docket_file}" + +# Loop until the app's build appears +while [ ! -d ${app_build} ]; do + echo "${CERC_URBIT_APP} app build not found, retrying in 5s..." + sleep 5 +done +echo "Build found..." + +echo "Using IPFS endpoint ${CERC_IPFS_GLOB_HOST_ENDPOINT} for hosting the ${CERC_URBIT_APP} glob" +echo "Using IPFS server endpoint ${CERC_IPFS_SERVER_ENDPOINT} for reading ${CERC_URBIT_APP} glob" +ipfs_host_endpoint=${CERC_IPFS_GLOB_HOST_ENDPOINT} +ipfs_server_endpoint=${CERC_IPFS_SERVER_ENDPOINT} + +# Fire curl requests to perform operations on the ship +dojo () { + curl -s --data '{"source":{"dojo":"'"$1"'"},"sink":{"stdout":null}}' http://localhost:12321 +} + +hood () { + curl -s --data '{"source":{"dojo":"+hood/'"$1"'"},"sink":{"app":"hood"}}' http://localhost:12321 +} + +# Create / mount the app's desk +hood "merge %${CERC_URBIT_APP} our %landscape" +hood "mount %${CERC_URBIT_APP}" + +# Copy over build to desk data dir +cp -r ${app_build} ${app_desk_dir} + +# Copy over the additional mark files +cp ${app_mark_files}/* ${app_desk_dir}/mar/ + +rm "${app_desk_dir}/desk.bill" +rm "${app_desk_dir}/desk.ship" + +# Commit changes and create a glob +hood "commit %${CERC_URBIT_APP}" +dojo "-landscape!make-glob %${CERC_URBIT_APP} /build" + +glob_file=$(ls -1 -c zod/.urb/put | head -1) +echo "Created glob file: ${glob_file}" + +# Upload the glob file to IPFS +echo "Uploading glob file to ${ipfs_host_endpoint}" +upload_response=$(curl -X POST -F file=@./zod/.urb/put/${glob_file} ${ipfs_host_endpoint}/api/v0/add) +glob_cid=$(echo "$upload_response" | grep -o '"Hash":"[^"]*' | sed 's/"Hash":"//') + +glob_url="${ipfs_server_endpoint}/ipfs/${glob_cid}?filename=${glob_file}" +glob_hash=$(echo "$glob_file" | sed "s/glob-\([a-z0-9\.]*\).glob/\1/") + +echo "Glob file uploaded to IFPS:" +echo "{ cid: ${glob_cid}, filename: ${glob_file} }" +echo "{ url: ${glob_url}, hash: ${glob_hash} }" + +# Exit if the installation not required +if [ "$CERC_ENABLE_APP_INSTALL" = "false" ]; then + echo "CERC_ENABLE_APP_INSTALL set to false, skipping app installation" + exit 0 +fi + +# Curl and wait for the glob to be hosted +echo "Checking if glob file hosted at ${glob_url}" +while true; do + response=$(curl -sL -w "%{http_code}" -o /dev/null "$glob_url") + + if [ $response -eq 200 ]; then + echo "File found at $glob_url" + break # Exit the loop if the file is found + else + echo "File not found, retrying in a 5s..." + sleep 5 + fi +done + +# Replace the docket file for app +# Substitue the glob URL and hash +cp ${app_docket_file} ${app_desk_dir}/ +sed -i "s|REPLACE_WITH_GLOB_URL|${glob_url}|g; s|REPLACE_WITH_GLOB_HASH|${glob_hash}|g" ${app_desk_dir}/desk.docket-0 + +# Commit changes and install the app +hood "commit %${CERC_URBIT_APP}" +hood "install our %${CERC_URBIT_APP}" + +echo "${CERC_URBIT_APP} app installed" diff --git a/stack_orchestrator/data/config/urbit/install-urbit-app.sh b/stack_orchestrator/data/config/urbit/install-urbit-app.sh new file mode 100755 index 00000000..444ff45e --- /dev/null +++ b/stack_orchestrator/data/config/urbit/install-urbit-app.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# $1: App name (eg. uniswap) +# $2: Assets dir path (local) for app (eg. /home/user/myapp/urbit-files) +# $3: Glob file URL (eg. https://xyz.com/glob-0vabcd.glob) +# $4: Glob file hash (eg. 0vabcd) +# $5: Urbit ship's pier dir (default: ./zod) + +if [ "$#" -lt 4 ]; then + echo "Insufficient arguments" + echo "Usage: $0 [/path/to/remote/pier/folder]" + exit 1 +fi + +app_name=$1 +app_mark_files=$2/mar +app_docket_file=$2/desk.docket-0 +echo "Creating Urbit application for ${app_name}" +echo "Reading additional mark files from ${app_mark_files}" +echo "Reading docket file ${app_docket_file}" + +glob_url=$3 +glob_hash=$4 +echo "Using glob file from ${glob_url} with hash ${glob_hash}" + +# Default pier dir: ./zod +# Default desk dir: ./zod/ +pier_dir="${5:-./zod}" +app_desk_dir="${pier_dir}/${app_name}" +echo "Using ${app_desk_dir} as the ${app_name} desk dir path" + +# Fire curl requests to perform operations on the ship +dojo () { + curl -s --data '{"source":{"dojo":"'"$1"'"},"sink":{"stdout":null}}' http://localhost:12321 +} + +hood () { + curl -s --data '{"source":{"dojo":"+hood/'"$1"'"},"sink":{"app":"hood"}}' http://localhost:12321 +} + +# Create / mount the app's desk +hood "merge %${app_name} our %landscape" +hood "mount %${app_name}" + +# Copy over the additional mark files +cp ${app_mark_files}/* ${app_desk_dir}/mar/ + +rm "${app_desk_dir}/desk.bill" +rm "${app_desk_dir}/desk.ship" + +# Replace the docket file for app +# Substitue the glob URL and hash +cp ${app_docket_file} ${app_desk_dir}/ +sed -i "s|REPLACE_WITH_GLOB_URL|${glob_url}|g; s|REPLACE_WITH_GLOB_HASH|${glob_hash}|g" ${app_desk_dir}/desk.docket-0 + +# Commit changes and install the app +hood "commit %${app_name}" +hood "install our %${app_name}" + +echo "${app_name} app installed" diff --git a/stack_orchestrator/data/config/urbit/run-urbit-ship.sh b/stack_orchestrator/data/config/urbit/run-urbit-ship.sh new file mode 100755 index 00000000..0b219803 --- /dev/null +++ b/stack_orchestrator/data/config/urbit/run-urbit-ship.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +pier_dir="/urbit/zod" + +# TODO: Bootstrap fake ship on the first run + +# Run urbit ship in daemon mode +# Check if the directory exists +if [ -d "$pier_dir" ]; then + echo "Pier directory already exists, rebooting..." + /urbit/zod/.run -d +else + echo "Creating a new fake ship..." + urbit -d -F zod +fi diff --git a/stack_orchestrator/data/config/watcher-azimuth/start-job-runner.sh b/stack_orchestrator/data/config/watcher-azimuth/start-job-runner.sh new file mode 100755 index 00000000..4bcad74c --- /dev/null +++ b/stack_orchestrator/data/config/watcher-azimuth/start-job-runner.sh @@ -0,0 +1,28 @@ +#!/bin/sh +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi + +echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}" +echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}" +echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}" + +# Replace env variables in template TOML file +# Read in the config template TOML file and modify it +WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) +WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ + sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \ + s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \ + s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ") + +# Write the modified content to a new file +echo "$WATCHER_CONFIG" > environments/watcher-config.toml + +# Merge SO watcher config with existing config file +node merge-toml.js + +yarn watch:contract --address $CONTRACT_ADDRESS --kind $CONTRACT_NAME --checkpoint true --starting-block $STARTING_BLOCK + +echo 'yarn job-runner' +yarn job-runner diff --git a/stack_orchestrator/data/config/watcher-azimuth/start-server.sh b/stack_orchestrator/data/config/watcher-azimuth/start-server.sh index c84c58d0..fa334653 100755 --- a/stack_orchestrator/data/config/watcher-azimuth/start-server.sh +++ b/stack_orchestrator/data/config/watcher-azimuth/start-server.sh @@ -4,18 +4,17 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi -CERC_IPLD_ETH_RPC="${CERC_IPLD_ETH_RPC:-${DEFAULT_CERC_IPLD_ETH_RPC}}" -CERC_IPLD_ETH_GQL="${CERC_IPLD_ETH_GQL:-${DEFAULT_CERC_IPLD_ETH_GQL}}" - echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}" echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}" +echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}" # Replace env variables in template TOML file # Read in the config template TOML file and modify it WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \ - s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}| ") + s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \ + s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ") # Write the modified content to a new file echo "$WATCHER_CONFIG" > environments/watcher-config.toml diff --git a/stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml index 1a4616fc..2a91fedf 100644 --- a/stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml +++ b/stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml @@ -2,6 +2,9 @@ host = "0.0.0.0" maxSimultaneousRequests = -1 +[metrics] + host = "0.0.0.0" + [database] host = "watcher-db" port = 5432 @@ -12,3 +15,7 @@ [upstream.ethServer] gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL" rpcProviderEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_RPC" + +[jobQueue] + historicalLogsBlockRange = REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE + blockDelayInMilliSecs = 12000 diff --git a/stack_orchestrator/data/config/watcher-azimuth/watcher-params.env b/stack_orchestrator/data/config/watcher-azimuth/watcher-params.env deleted file mode 100644 index 8fcdc2d6..00000000 --- a/stack_orchestrator/data/config/watcher-azimuth/watcher-params.env +++ /dev/null @@ -1,5 +0,0 @@ -# Defaults - -# ipld-eth-server endpoints -DEFAULT_CERC_IPLD_ETH_RPC= -DEFAULT_CERC_IPLD_ETH_GQL= diff --git a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-job-runner.sh b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-job-runner.sh new file mode 100755 index 00000000..819b1096 --- /dev/null +++ b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-job-runner.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi +set -u + +echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}" + +# Read in the config template TOML file and modify it +WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) +WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ + sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ") + +# Write the modified content to a new file +echo "$WATCHER_CONFIG" > environments/local.toml + +echo "Running job-runner..." +DEBUG=vulcanize:* exec node --enable-source-maps dist/job-runner.js diff --git a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-server.sh b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-server.sh new file mode 100755 index 00000000..e2bbdaad --- /dev/null +++ b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-server.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi +set -u + +echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}" + +# Read in the config template TOML file and modify it +WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) +WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ + sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ") + +# Write the modified content to a new file +echo "$WATCHER_CONFIG" > environments/local.toml + +echo "Running server..." +DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js diff --git a/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml new file mode 100644 index 00000000..57494ce3 --- /dev/null +++ b/stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml @@ -0,0 +1,98 @@ +[server] + host = "0.0.0.0" + port = 3008 + kind = "active" + gqlPath = '/' + + # Checkpointing state. + checkpointing = true + + # Checkpoint interval in number of blocks. + checkpointInterval = 2000 + + # Enable state creation + # CAUTION: Disable only if state creation is not desired or can be filled subsequently + enableState = false + + subgraphPath = "./subgraph-build" + + # Interval to restart wasm instance periodically + wasmRestartBlocksInterval = 20 + + # Interval in number of blocks at which to clear entities cache. + clearEntitiesCacheInterval = 1000 + + # Max block range for which to return events in eventsInRange GQL query. + # Use -1 for skipping check on block range. + maxEventsBlockRange = 1000 + + # Flag to specify whether RPC endpoint supports block hash as block tag parameter + rpcSupportsBlockHashParam = false + + # GQL cache settings + [server.gqlCache] + enabled = true + + # Max in-memory cache size (in bytes) (default 8 MB) + # maxCacheSize + + # GQL cache-control max-age settings (in seconds) + maxAge = 15 + timeTravelMaxAge = 86400 # 1 day + +[metrics] + host = "0.0.0.0" + port = 9000 + [metrics.gql] + port = 9001 + +[database] + type = "postgres" + host = "merkl-sushiswap-v3-watcher-db" + port = 5432 + database = "merkl-sushiswap-v3-watcher" + username = "vdbm" + password = "password" + synchronize = true + logging = false + +[upstream] + [upstream.ethServer] + rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT" + + # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client) + rpcClient = true + + # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint + isFEVM = true + + # Boolean flag to filter event logs by contracts + filterLogsByAddresses = true + # Boolean flag to filter event logs by topics + filterLogsByTopics = false + + [upstream.cache] + name = "requests" + enabled = false + deleteOnStart = false + +[jobQueue] + dbConnectionString = "postgres://vdbm:password@merkl-sushiswap-v3-watcher-db/merkl-sushiswap-v3-watcher-job-queue" + maxCompletionLagInSecs = 300 + jobDelayInMilliSecs = 100 + eventsInBatch = 50 + subgraphEventsOrder = true + # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime + blockDelayInMilliSecs = 30000 + + # Boolean to switch between modes of processing events when starting the server. + # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them. + # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head). + useBlockRanges = true + + # Block range in which logs are fetched during historical blocks processing + historicalLogsBlockRange = 2000 + + # Max block range of historical processing after which it waits for completion of events processing + # If set to -1 historical processing does not wait for events processing and completes till latest canonical block + historicalMaxFetchAhead = 10000 diff --git a/stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh b/stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh new file mode 100755 index 00000000..819b1096 --- /dev/null +++ b/stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi +set -u + +echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}" + +# Read in the config template TOML file and modify it +WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) +WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ + sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ") + +# Write the modified content to a new file +echo "$WATCHER_CONFIG" > environments/local.toml + +echo "Running job-runner..." +DEBUG=vulcanize:* exec node --enable-source-maps dist/job-runner.js diff --git a/stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh b/stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh new file mode 100755 index 00000000..e2bbdaad --- /dev/null +++ b/stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +set -e +if [ -n "$CERC_SCRIPT_DEBUG" ]; then + set -x +fi +set -u + +echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}" + +# Read in the config template TOML file and modify it +WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml) +WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \ + sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ") + +# Write the modified content to a new file +echo "$WATCHER_CONFIG" > environments/local.toml + +echo "Running server..." +DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js diff --git a/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml b/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml new file mode 100644 index 00000000..24449d4f --- /dev/null +++ b/stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml @@ -0,0 +1,98 @@ +[server] + host = "0.0.0.0" + port = 3008 + kind = "active" + gqlPath = "/" + + # Checkpointing state. + checkpointing = true + + # Checkpoint interval in number of blocks. + checkpointInterval = 2000 + + # Enable state creation + # CAUTION: Disable only if state creation is not desired or can be filled subsequently + enableState = false + + subgraphPath = "./subgraph-build" + + # Interval to restart wasm instance periodically + wasmRestartBlocksInterval = 20 + + # Interval in number of blocks at which to clear entities cache. + clearEntitiesCacheInterval = 1000 + + # Max block range for which to return events in eventsInRange GQL query. + # Use -1 for skipping check on block range. + maxEventsBlockRange = 1000 + + # Flag to specify whether RPC endpoint supports block hash as block tag parameter + rpcSupportsBlockHashParam = false + + # GQL cache settings + [server.gqlCache] + enabled = true + + # Max in-memory cache size (in bytes) (default 8 MB) + # maxCacheSize + + # GQL cache-control max-age settings (in seconds) + maxAge = 15 + timeTravelMaxAge = 86400 # 1 day + +[metrics] + host = "0.0.0.0" + port = 9000 + [metrics.gql] + port = 9001 + +[database] + type = "postgres" + host = "sushiswap-v3-watcher-db" + port = 5432 + database = "sushiswap-v3-watcher" + username = "vdbm" + password = "password" + synchronize = true + logging = false + +[upstream] + [upstream.ethServer] + rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT" + + # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client) + rpcClient = true + + # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint + isFEVM = true + + # Boolean flag to filter event logs by contracts + filterLogsByAddresses = true + # Boolean flag to filter event logs by topics + filterLogsByTopics = true + + [upstream.cache] + name = "requests" + enabled = false + deleteOnStart = false + +[jobQueue] + dbConnectionString = "postgres://vdbm:password@sushiswap-v3-watcher-db/sushiswap-v3-watcher-job-queue" + maxCompletionLagInSecs = 300 + jobDelayInMilliSecs = 100 + eventsInBatch = 50 + subgraphEventsOrder = true + # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime + blockDelayInMilliSecs = 30000 + + # Boolean to switch between modes of processing events when starting the server. + # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them. + # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head). + useBlockRanges = true + + # Block range in which logs are fetched during historical blocks processing + historicalLogsBlockRange = 2000 + + # Max block range of historical processing after which it waits for completion of events processing + # If set to -1 historical processing does not wait for events processing and completes till latest canonical block + historicalMaxFetchAhead = 10000 diff --git a/stack_orchestrator/data/container-build/cerc-act-runner-task-executor/build.sh b/stack_orchestrator/data/container-build/cerc-act-runner-task-executor/build.sh index b625ed4b..939d6898 100755 --- a/stack_orchestrator/data/container-build/cerc-act-runner-task-executor/build.sh +++ b/stack_orchestrator/data/container-build/cerc-act-runner-task-executor/build.sh @@ -2,4 +2,6 @@ # Build a local version of the task executor for act-runner source ${CERC_CONTAINER_BASE_DIR}/build-base.sh SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -docker build -t cerc/act-runner-task-executor:local -f ${CERC_REPO_BASE_DIR}/hosting/act-runner/Dockerfile.task-executor ${build_command_args} ${SCRIPT_DIR} + +cd ${CERC_REPO_BASE_DIR}/hosting/act-runner +docker build -t cerc/act-runner-task-executor:local -f Dockerfile.task-executor ${build_command_args} . diff --git a/stack_orchestrator/data/container-build/cerc-laconic-console-host/config.yml b/stack_orchestrator/data/container-build/cerc-laconic-console-host/config.yml index d557ace5..6c310842 100644 --- a/stack_orchestrator/data/container-build/cerc-laconic-console-host/config.yml +++ b/stack_orchestrator/data/container-build/cerc-laconic-console-host/config.yml @@ -2,5 +2,5 @@ services: wns: - server: 'LACONIC_HOSTED_ENDPOINT:9473/api' - webui: 'LACONIC_HOSTED_ENDPOINT:9473/console' + server: 'LACONIC_HOSTED_ENDPOINT/api' + webui: 'LACONIC_HOSTED_ENDPOINT/console' diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-10.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-10.yml new file mode 100644 index 00000000..a467903e --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-10.yml @@ -0,0 +1,14 @@ +record: + type: ApplicationDeploymentRecord + version: 1.2.3 + name: name + description: description + application: application + url: url + dns: dns + request: request + meta: + foo: bar + tags: + - a + - b diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-11.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-11.yml new file mode 100644 index 00000000..3afbd64d --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-11.yml @@ -0,0 +1,12 @@ +record: + type: GeneralRecord + version: 1.2.3 + name: name + description: description + category: category + value: value + meta: + foo: bar + tags: + - a + - b diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-12.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-12.yml new file mode 100644 index 00000000..4c5f024c --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-12.yml @@ -0,0 +1,17 @@ +record: + type: ApplicationArtifact + version: 1.2.3 + name: name + description: description + application: appidgoeshere + content_type: content_type + os: os + cpu: cpu + uri: + - uri://a + - uri://b + meta: + foo: bar + tags: + - a + - b diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-7.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-7.yml new file mode 100644 index 00000000..1bf4ad46 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-7.yml @@ -0,0 +1,11 @@ +record: + type: DnsRecord + version: 0.0.1 + name: "foo" + resource_type: "A" + value: "bar" + meta: + foo: bar + tags: + - a + - b diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-8.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-8.yml new file mode 100644 index 00000000..95bd195a --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-8.yml @@ -0,0 +1,18 @@ +record: + type: ApplicationRecord + version: 0.0.1 + name: my-demo-app + description: "Description of my app" + homepage: http://my.demo.app + license: license + author: author + repository: + - "https://my.demo.repo" + repository_ref: "v0.1.0" + app_version: "0.1.0" + app_type: "webapp" + meta: + foo: bar + tags: + - a + - b diff --git a/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-9.yml b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-9.yml new file mode 100644 index 00000000..415e5c74 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-9.yml @@ -0,0 +1,17 @@ +record: + type: ApplicationDeploymentRequest + version: 1.2.3 + application: application + dns: dns + config: + env: + ENV_VAR_A: A + ENV_VAR_B: B + crn: + - crn://foo.bar + - crn://bar.baz + meta: + foo: bar + tags: + - a + - b diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile index 147cec29..8949c4e9 100644 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile @@ -1,6 +1,6 @@ # Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile # [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster -ARG VARIANT=18-bullseye +ARG VARIANT=20-bullseye-slim FROM node:${VARIANT} ARG USERNAME=node @@ -24,32 +24,21 @@ RUN \ && su ${USERNAME} -c "npm config -g set prefix ${NPM_GLOBAL}" \ # Install eslint && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \ + # Install semver + && su ${USERNAME} -c "umask 0002 && npm install -g semver" \ && npm cache clean --force > /dev/null 2>&1 # [Optional] Uncomment this section to install additional OS packages. RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ && apt-get -y install --no-install-recommends jq gettext-base -# [Optional] Uncomment if you want to install an additional version of node using nvm -# ARG EXTRA_NODE_VERSION=10 -# RUN su node -c "source /usr/local/share/nvm/nvm.sh && nvm install ${EXTRA_NODE_VERSION}" - -# We do this to get a yq binary from the published container, for the correct architecture we're building here -# COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq - -COPY /scripts /scripts - # [Optional] Uncomment if you want to install more global node modules # RUN su node -c "npm install -g " -# RUN mkdir -p /config -# COPY ./config.yml /config - -# Install simple web server for now (use nginx perhaps later) -# RUN yarn global add http-server - # Expose port for http EXPOSE 3000 +COPY /scripts /scripts + # Default command sleeps forever so docker doesn't kill it -CMD ["/scripts/start-serving-app.sh"] +ENTRYPOINT ["/scripts/start-serving-app.sh"] diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile.webapp b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile.webapp index f4b5d4d8..51664deb 100644 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile.webapp +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile.webapp @@ -1,4 +1,8 @@ FROM cerc/nextjs-base:local + +ARG CERC_NEXT_VERSION=keep +ARG CERC_BUILD_TOOL + WORKDIR /app COPY . . RUN rm -rf node_modules build .next* diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/build.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/build.sh index 3cf5f7f4..cca8d64b 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/build.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/build.sh @@ -11,3 +11,19 @@ CERC_CONTAINER_BUILD_DOCKERFILE=${CERC_CONTAINER_BUILD_DOCKERFILE:-$SCRIPT_DIR/D CERC_CONTAINER_BUILD_TAG=${CERC_CONTAINER_BUILD_TAG:-cerc/nextjs-base:local} docker build -t $CERC_CONTAINER_BUILD_TAG ${build_command_args} -f $CERC_CONTAINER_BUILD_DOCKERFILE $CERC_CONTAINER_BUILD_WORK_DIR + +if [ $? -eq 0 ] && [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/nextjs-base:local" ]; then + cat < /dev/null && pwd ) +CERC_MIN_NEXTVER=13.4.2 +CERC_NEXT_VERSION="${CERC_NEXT_VERSION:-keep}" +CERC_BUILD_TOOL="${CERC_BUILD_TOOL}" +if [ -z "$CERC_BUILD_TOOL" ]; then + if [ -f "yarn.lock" ]; then + CERC_BUILD_TOOL=yarn + else + CERC_BUILD_TOOL=npm + fi +fi + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) WORK_DIR="${1:-/app}" cd "${WORK_DIR}" || exit 1 -cp next.config.js next.config.dist +# If this file doesn't exist at all, we'll get errors below. +if [ ! -f "next.config.js" ]; then + touch next.config.js +fi + +if [ ! -f "next.config.dist" ]; then + cp next.config.js next.config.dist +fi + +which js-beautify >/dev/null +if [ $? -ne 0 ]; then + npm i -g js-beautify +fi -npm i -g js-beautify js-beautify next.config.dist > next.config.js +echo "" >> next.config.js -npm install - -CONFIG_LINES=$(wc -l next.config.js | awk '{ print $1 }') -MOD_EXPORTS_LINE=$(grep -n 'module.exports' next.config.js | cut -d':' -f1) - -head -$(( ${MOD_EXPORTS_LINE} - 1 )) next.config.js > next.config.js.1 - -cat > next.config.js.2 < next.config.js.0 < next.config.js.1 < next.config.js.3 +CONFIG_LINES=$(wc -l next.config.js | awk '{ print $1 }') +ENV_LINE=$(grep -n 'env:' next.config.js | cut -d':' -f1) +WEBPACK_CONF_LINE=$(egrep -n 'webpack:\s+\([^,]+,' next.config.js | cut -d':' -f1) +NEXT_SECTION_ADJUSTMENT=0 -cat > next.config.js.4 < { - config.plugins.push(new webpack.DefinePlugin(envMap)); - return config; - }, +if [ -n "$WEBPACK_CONF_LINE" ]; then + WEBPACK_CONF_VAR=$(egrep -n 'webpack:\s+\([^,]+,' next.config.js | cut -d',' -f1 | cut -d'(' -f2) + head -$(( ${WEBPACK_CONF_LINE} )) next.config.js > next.config.js.2 + cat > next.config.js.3 < next.config.js.2 + cat > next.config.js.3 < { + config.plugins.push(new webpack.DefinePlugin(envMap)); + return config; + }, +EOF + NEXT_SECTION_ADJUSTMENT=1 + NEXT_SECTION_LINE=$ENV_LINE +else + echo "WARNING: Cannot find location to insert environment variable map in next.config.js" 1>&2 + rm -f next.config.js.* + NEXT_SECTION_LINE=0 +fi -tail -$(( ${CONFIG_LINES} - ${MOD_EXPORTS_LINE} + 1 )) next.config.js | grep -v 'process\.env\.' > next.config.js.5 +tail -$(( ${CONFIG_LINES} - ${NEXT_SECTION_LINE} + ${NEXT_SECTION_ADJUSTMENT} )) next.config.js > next.config.js.5 -cat next.config.js.* | js-beautify > next.config.js +cat next.config.js.* | sed 's/^ *//g' | js-beautify | grep -v 'process\.\env\.' | js-beautify > next.config.js rm next.config.js.* "${SCRIPT_DIR}/find-env.sh" "$(pwd)" > .env-list.json -npm run build -rm .env-list.json \ No newline at end of file +if [ ! -f "package.dist" ]; then + cp package.json package.dist +fi + +cat package.dist | jq '.scripts.cerc_compile = "next experimental-compile"' | jq '.scripts.cerc_generate = "next experimental-generate"' > package.json + +CUR_NEXT_VERSION="`jq -r '.dependencies.next' package.json`" + +if [ "$CERC_NEXT_VERSION" != "keep" ] && [ "$CUR_NEXT_VERSION" != "$CERC_NEXT_VERSION" ]; then + echo "Changing 'next' version specifier from '$CUR_NEXT_VERSION' to '$CERC_NEXT_VERSION' (set with '--extra-build-args \"--build-arg CERC_NEXT_VERSION=$CERC_NEXT_VERSION\"')" + cat package.json | jq ".dependencies.next = \"$CERC_NEXT_VERSION\"" > package.json.$$ + mv package.json.$$ package.json +fi + +$CERC_BUILD_TOOL install || exit 1 + +CUR_NEXT_VERSION=`jq -r '.version' node_modules/next/package.json` + +semver -p -r ">=$CERC_MIN_NEXTVER" $CUR_NEXT_VERSION +if [ $? -ne 0 ]; then + cat <" + +############################################################################### + +EOF + cat package.json | jq ".dependencies.next = \"^$CERC_MIN_NEXTVER\"" > package.json.$$ + mv package.json.$$ package.json + $CERC_BUILD_TOOL install || exit 1 +fi + +$CERC_BUILD_TOOL run cerc_compile || exit 1 + +exit 0 diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/find-env.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/find-env.sh index 0c0e87c9..59cb3d49 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/find-env.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/find-env.sh @@ -20,5 +20,10 @@ for d in $(find . -maxdepth 1 -type d | grep -v '\./\.' | grep '/' | cut -d'/' - done done +NEXT_CONF="next.config.js next.config.dist" +for f in $NEXT_CONF; do + cat "$f" | tr -s '[:blank:]' '\n' | tr -s '[{},()]' '\n' | egrep -o 'process.env.[A-Za-z0-9_]+' >> $TMPF +done + cat $TMPF | sort -u | jq --raw-input . | jq --slurp . rm -f $TMPF diff --git a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh index abe72935..3692f2f6 100755 --- a/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh +++ b/stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh @@ -3,11 +3,59 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi + SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +CERC_MAX_GENERATE_TIME=${CERC_MAX_GENERATE_TIME:-60} +tpid="" + +ctrl_c() { + kill $tpid $(ps -ef | grep node | grep next | awk '{print $2}') 2>/dev/null +} + +trap ctrl_c INT + +CERC_BUILD_TOOL="${CERC_BUILD_TOOL}" +if [ -z "$CERC_BUILD_TOOL" ]; then + if [ -f "yarn.lock" ] && [ ! -f "package-lock.json" ]; then + CERC_BUILD_TOOL=yarn + else + CERC_BUILD_TOOL=npm + fi +fi CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/app}" cd "$CERC_WEBAPP_FILES_DIR" -rm -rf .next-r "$SCRIPT_DIR/apply-runtime-env.sh" "`pwd`" .next .next-r -npm start .next-r -p ${CERC_LISTEN_PORT:-3000} +mv .next .next.old +mv .next-r/.next . + +if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then + jq -e '.scripts.cerc_generate' package.json >/dev/null + if [ $? -eq 0 ]; then + npm run cerc_generate > gen.out 2>&1 & + tail -f gen.out & + tpid=$! + + count=0 + generate_done="false" + while [ $count -lt $CERC_MAX_GENERATE_TIME ] && [ "$generate_done" == "false" ]; do + sleep 1 + count=$((count + 1)) + grep 'rendered as static' gen.out > /dev/null + if [ $? -eq 0 ]; then + generate_done="true" + fi + done + + if [ $generate_done != "true" ]; then + echo "ERROR: 'npm run cerc_generate' not successful within CERC_MAX_GENERATE_TIME" 1>&2 + exit 1 + fi + + kill $tpid $(ps -ef | grep node | grep next | grep generate | awk '{print $2}') 2>/dev/null + tpid="" + fi +fi + +$CERC_BUILD_TOOL start . -p ${CERC_LISTEN_PORT:-3000} diff --git a/stack_orchestrator/data/container-build/cerc-optimism-contracts/Dockerfile b/stack_orchestrator/data/container-build/cerc-optimism-contracts/Dockerfile index ed9c4b22..2499df0a 100644 --- a/stack_orchestrator/data/container-build/cerc-optimism-contracts/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-optimism-contracts/Dockerfile @@ -17,6 +17,6 @@ WORKDIR /app COPY . . RUN echo "Building optimism" && \ - yarn && yarn build + pnpm install && pnpm build WORKDIR /app/packages/contracts-bedrock diff --git a/stack_orchestrator/data/container-build/cerc-optimism-op-batcher/Dockerfile b/stack_orchestrator/data/container-build/cerc-optimism-op-batcher/Dockerfile index 23d6b629..f52e75b9 100644 --- a/stack_orchestrator/data/container-build/cerc-optimism-op-batcher/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-optimism-op-batcher/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.19.0-alpine3.15 as builder +FROM golang:1.21.0-alpine3.18 as builder ARG VERSION=v0.0.0 @@ -9,7 +9,7 @@ COPY ./op-batcher /app/op-batcher COPY ./op-bindings /app/op-bindings COPY ./op-node /app/op-node COPY ./op-service /app/op-service -COPY ./op-signer /app/op-signer +#COPY ./op-signer /app/op-signer COPY ./go.mod /app/go.mod COPY ./go.sum /app/go.sum @@ -23,7 +23,7 @@ ARG TARGETOS TARGETARCH RUN make op-batcher VERSION="$VERSION" GOOS=$TARGETOS GOARCH=$TARGETARCH -FROM alpine:3.15 +FROM alpine:3.18 RUN apk add --no-cache jq bash diff --git a/stack_orchestrator/data/container-build/cerc-optimism-op-node/Dockerfile b/stack_orchestrator/data/container-build/cerc-optimism-op-node/Dockerfile index 17d273b6..ad63bb2c 100644 --- a/stack_orchestrator/data/container-build/cerc-optimism-op-node/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-optimism-op-node/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.19.0-alpine3.15 as builder +FROM golang:1.21.0-alpine3.18 as builder ARG VERSION=v0.0.0 @@ -21,7 +21,7 @@ ARG TARGETOS TARGETARCH RUN make op-node VERSION="$VERSION" GOOS=$TARGETOS GOARCH=$TARGETARCH -FROM alpine:3.15 +FROM alpine:3.18 RUN apk add --no-cache openssl jq diff --git a/stack_orchestrator/data/container-build/cerc-optimism-op-proposer/Dockerfile b/stack_orchestrator/data/container-build/cerc-optimism-op-proposer/Dockerfile index e91aa4bb..9032a7ff 100644 --- a/stack_orchestrator/data/container-build/cerc-optimism-op-proposer/Dockerfile +++ b/stack_orchestrator/data/container-build/cerc-optimism-op-proposer/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.19.0-alpine3.15 as builder +FROM golang:1.21.0-alpine3.18 as builder ARG VERSION=v0.0.0 @@ -9,7 +9,7 @@ COPY ./op-proposer /app/op-proposer COPY ./op-bindings /app/op-bindings COPY ./op-node /app/op-node COPY ./op-service /app/op-service -COPY ./op-signer /app/op-signer +#COPY ./op-signer /app/op-signer COPY ./go.mod /app/go.mod COPY ./go.sum /app/go.sum COPY ./.git /app/.git @@ -22,7 +22,7 @@ ARG TARGETOS TARGETARCH RUN make op-proposer VERSION="$VERSION" GOOS=$TARGETOS GOARCH=$TARGETARCH -FROM alpine:3.15 +FROM alpine:3.18 RUN apk add --no-cache jq bash diff --git a/stack_orchestrator/data/container-build/cerc-osmosis-front-end-urbit/build.sh b/stack_orchestrator/data/container-build/cerc-osmosis-front-end-urbit/build.sh new file mode 100755 index 00000000..9a47ada9 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-osmosis-front-end-urbit/build.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Build the osmosis front end image +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh +docker build -t cerc/osmosis-front-end-urbit:local -f ${CERC_REPO_BASE_DIR}/osmosis-frontend/docker/Dockerfile.static ${build_command_args} ${CERC_REPO_BASE_DIR}/osmosis-frontend diff --git a/stack_orchestrator/data/container-build/cerc-osmosis-front-end/build.sh b/stack_orchestrator/data/container-build/cerc-osmosis-front-end/build.sh new file mode 100755 index 00000000..c4a9f27e --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-osmosis-front-end/build.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Build the osmosis front end image +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh +docker build -t cerc/osmosis-front-end:local -f ${CERC_REPO_BASE_DIR}/osmosis-frontend/docker/Dockerfile.static ${build_command_args} ${CERC_REPO_BASE_DIR}/osmosis-frontend diff --git a/stack_orchestrator/data/container-build/cerc-uniswap-interface/Dockerfile b/stack_orchestrator/data/container-build/cerc-uniswap-interface/Dockerfile new file mode 100644 index 00000000..59804896 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-uniswap-interface/Dockerfile @@ -0,0 +1,10 @@ +FROM node:18.17.1-alpine3.18 + +RUN apk --update --no-cache add git make alpine-sdk bash + +WORKDIR /app + +COPY . . + +RUN echo "Building uniswap-interface" && \ + yarn diff --git a/stack_orchestrator/data/container-build/cerc-uniswap-interface/build.sh b/stack_orchestrator/data/container-build/cerc-uniswap-interface/build.sh new file mode 100755 index 00000000..af1971b5 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-uniswap-interface/build.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Build the uniswap-interface image +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh + +# See: https://stackoverflow.com/a/246128/1701505 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/uniswap-interface:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/uniswap-interface diff --git a/stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/Dockerfile b/stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/Dockerfile new file mode 100644 index 00000000..e09738ac --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/Dockerfile @@ -0,0 +1,10 @@ +FROM node:18.17.1-alpine3.18 + +RUN apk --update --no-cache add git python3 alpine-sdk bash curl jq + +WORKDIR /app + +COPY . . + +RUN echo "Installing dependencies and building merkl-sushiswap-v3-watcher-ts" && \ + yarn && yarn build diff --git a/stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/build.sh b/stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/build.sh new file mode 100755 index 00000000..b53ee621 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/build.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Build cerc/watcher-merkl-sushiswap-v3 + +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh + +# See: https://stackoverflow.com/a/246128/1701505 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/my-new-stack:local -f ${CERC_REPO_BASE_DIR}/my-new-stack/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/my-new-stack + +docker build -t cerc/watcher-merkl-sushiswap-v3:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/merkl-sushiswap-v3-watcher-ts diff --git a/stack_orchestrator/data/container-build/cerc-watcher-sushiswap-v3/Dockerfile b/stack_orchestrator/data/container-build/cerc-watcher-sushiswap-v3/Dockerfile new file mode 100644 index 00000000..ac6241c4 --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-watcher-sushiswap-v3/Dockerfile @@ -0,0 +1,10 @@ +FROM node:18.17.1-alpine3.18 + +RUN apk --update --no-cache add git python3 alpine-sdk bash curl jq + +WORKDIR /app + +COPY . . + +RUN echo "Installing dependencies and building sushiswap-v3-watcher-ts" && \ + yarn && yarn build diff --git a/stack_orchestrator/data/container-build/cerc-watcher-sushiswap-v3/build.sh b/stack_orchestrator/data/container-build/cerc-watcher-sushiswap-v3/build.sh new file mode 100755 index 00000000..4eb79eee --- /dev/null +++ b/stack_orchestrator/data/container-build/cerc-watcher-sushiswap-v3/build.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Build cerc/watcher-sushiswap-v3 + +source ${CERC_CONTAINER_BASE_DIR}/build-base.sh + +# See: https://stackoverflow.com/a/246128/1701505 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +docker build -t cerc/watcher-sushiswap-v3:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/sushiswap-v3-watcher-ts diff --git a/stack_orchestrator/data/container-image-list.txt b/stack_orchestrator/data/container-image-list.txt index 256f0a6f..fd295be5 100644 --- a/stack_orchestrator/data/container-image-list.txt +++ b/stack_orchestrator/data/container-image-list.txt @@ -57,3 +57,6 @@ cerc/nitro-contracts cerc/mobymask-snap cerc/ponder cerc/nitro-rpc-client +cerc/watcher-merkl-sushiswap-v3 +cerc/watcher-sushiswap-v3 +cerc/uniswap-interface diff --git a/stack_orchestrator/data/pod-list.txt b/stack_orchestrator/data/pod-list.txt index 4ba1bac0..9ad000c7 100644 --- a/stack_orchestrator/data/pod-list.txt +++ b/stack_orchestrator/data/pod-list.txt @@ -43,3 +43,5 @@ nitro-contracts mobymask-snap ponder ipld-eth-server-payments +merkl-sushiswap-v3 +sushiswap-v3 diff --git a/stack_orchestrator/data/repository-list.txt b/stack_orchestrator/data/repository-list.txt index ceaa910c..cddaccce 100644 --- a/stack_orchestrator/data/repository-list.txt +++ b/stack_orchestrator/data/repository-list.txt @@ -47,3 +47,6 @@ github.com/cerc-io/go-nitro github.com/cerc-io/ts-nitro github.com/cerc-io/mobymask-snap github.com/cerc-io/ponder +github.com/cerc-io/merkl-sushiswap-v3-watcher-ts +github.com/cerc-io/sushiswap-v3-watcher-ts +github.com/cerc-io/uniswap-interface diff --git a/stack_orchestrator/data/stacks/act-runner/stack.yml b/stack_orchestrator/data/stacks/act-runner/stack.yml index a236fccf..473bc01e 100644 --- a/stack_orchestrator/data/stacks/act-runner/stack.yml +++ b/stack_orchestrator/data/stacks/act-runner/stack.yml @@ -3,7 +3,7 @@ name: act-runner description: "Local act-runner" repos: - git.vdb.to/cerc-io/hosting - - gitea.com/gitea/act_runner + - gitea.com/telackey/act_runner@telackey/entrypoint containers: - cerc/act-runner - cerc/act-runner-task-executor diff --git a/stack_orchestrator/data/stacks/azimuth/README.md b/stack_orchestrator/data/stacks/azimuth/README.md index 67f42b75..21200369 100644 --- a/stack_orchestrator/data/stacks/azimuth/README.md +++ b/stack_orchestrator/data/stacks/azimuth/README.md @@ -9,19 +9,11 @@ Prerequisite: `ipld-eth-server` RPC and GQL endpoints Clone required repositories: ```bash -laconic-so --stack azimuth setup-repositories +laconic-so --stack azimuth setup-repositories --pull ``` NOTE: If the repository already exists and checked out to a different version, `setup-repositories` command will throw an error. -For getting around this, the `azimuth-watcher-ts` repository can be removed and then run the command. - -Checkout to the required versions and branches in repos - -```bash -# azimuth-watcher-ts -cd ~/cerc/azimuth-watcher-ts -git checkout v0.1.0 -``` +For getting around this, the `azimuth-watcher-ts` repository can be removed and then run the command again. Build the container images: @@ -31,42 +23,101 @@ laconic-so --stack azimuth build-containers This should create the required docker images in the local image registry. -### Configuration +## Create a deployment -* Create and update an env file to be used in the next step: +First, create a spec file for the deployment, which will map the stack's ports and volumes to the host: +```bash +laconic-so --stack azimuth deploy init --output azimuth-spec.yml +``` + +### Ports + +Edit `network` in spec file to map container ports to same ports in host + +```yaml +... +network: + ports: + watcher-db: + - 0.0.0.0:15432:5432 + azimuth-watcher-job-runner: + - 0.0.0.0:9000:9000 + azimuth-watcher-server: + - 0.0.0.0:3001:3001 + censures-watcher-job-runner: + - 0.0.0.0:9002:9002 + censures-watcher-server: + - 0.0.0.0:3002:3002 + claims-watcher-job-runner: + - 0.0.0.0:9004:9004 + claims-watcher-server: + - 0.0.0.0:3003:3003 + conditional-star-release-watcher-job-runner: + - 0.0.0.0:9006:9006 + conditional-star-release-watcher-server: + - 0.0.0.0:3004:3004 + delegated-sending-watcher-job-runner: + - 0.0.0.0:9008:9008 + delegated-sending-watcher-server: + - 0.0.0.0:3005:3005 + ecliptic-watcher-job-runner: + - 0.0.0.0:9010:9010 + ecliptic-watcher-server: + - 0.0.0.0:3006:3006 + linear-star-release-watcher-job-runner: + - 0.0.0.0:9012:9012 + linear-star-release-watcher-server: + - 0.0.0.0:3007:3007 + polls-watcher-job-runner: + - 0.0.0.0:9014:9014 + polls-watcher-server: + - 0.0.0.0:3008:3008 + gateway-server: + - 0.0.0.0:4000:4000 +... +``` + +### Data volumes +Container data volumes are bind-mounted to specified paths in the host filesystem. +The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory. The default mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`. + +--- + +Once you've made any needed changes to the spec file, create a deployment from it: +```bash +laconic-so --stack azimuth deploy create --spec-file azimuth-spec.yml --deployment-dir azimuth-deployment +``` + +## Set env variables + +Inside the deployment directory, open the file `config.env` and add variable to update RPC endpoint : ```bash - # External ipld-eth-server endpoints + # External RPC endpoints CERC_IPLD_ETH_RPC= - CERC_IPLD_ETH_GQL= ``` -* NOTE: If `ipld-eth-server` is running on the host machine, use `host.docker.internal` as the hostname to access host ports +* NOTE: If RPC endpoint is on the host machine, use `host.docker.internal` as the hostname to access the host port, or use the `ip a` command to find the IP address of the `docker0` interface (this will usually be something like `172.17.0.1` or `172.18.0.1`) -### Deploy the stack +## Start the stack -* Deploy the containers: - - ```bash - laconic-so --stack azimuth deploy-system --env-file up - ``` +Start the deployment: +```bash +laconic-so deployment --dir azimuth-deployment start +``` * List and check the health status of all the containers using `docker ps` and wait for them to be `healthy` ## Clean up -Stop all the services running in background: +To stop all azimuth services running in the background, while preserving chain data: ```bash -laconic-so --stack azimuth deploy-system down +laconic-so deployment --dir azimuth-deployment stop ``` -Clear volumes created by this stack: +To stop all azimuth services and also delete data: ```bash -# List all relevant volumes -docker volume ls -q --filter "name=.*watcher_db_data" - -# Remove all the listed volumes -docker volume rm $(docker volume ls -q --filter "name=.*watcher_db_data") +laconic-so deployment --dir azimuth-deployment stop --delete-volumes ``` diff --git a/stack_orchestrator/data/stacks/azimuth/stack.yml b/stack_orchestrator/data/stacks/azimuth/stack.yml index 47e0d058..374f2af0 100644 --- a/stack_orchestrator/data/stacks/azimuth/stack.yml +++ b/stack_orchestrator/data/stacks/azimuth/stack.yml @@ -1,7 +1,7 @@ version: "1.0" name: azimuth repos: - - github.com/cerc-io/azimuth-watcher-ts@v0.1.1 + - github.com/cerc-io/azimuth-watcher-ts@v0.1.3 containers: - cerc/watcher-azimuth pods: diff --git a/stack_orchestrator/data/stacks/fixturenet-optimism/README.md b/stack_orchestrator/data/stacks/fixturenet-optimism/README.md index 4d933f83..dd681aa5 100644 --- a/stack_orchestrator/data/stacks/fixturenet-optimism/README.md +++ b/stack_orchestrator/data/stacks/fixturenet-optimism/README.md @@ -12,6 +12,7 @@ Clone required repositories: laconic-so --stack fixturenet-optimism setup-repositories # If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories mentioned below and re-run the command +# The repositories are located in $HOME/cerc by default ``` Build the container images: @@ -39,20 +40,53 @@ This should create the required docker images in the local image registry: * `cerc/optimism-op-batcher` * `cerc/optimism-op-proposer` -## Deploy -Deploy the stack: +## Create a deployment +First, create a spec file for the deployment, which will map the stack's ports and volumes to the host: ```bash -laconic-so --stack fixturenet-optimism deploy up +laconic-so --stack fixturenet-optimism deploy init --map-ports-to-host any-fixed-random --output fixturenet-optimism-spec.yml ``` -The `fixturenet-optimism-contracts` service takes a while to complete running as it: -1. waits for the 'Merge' to happen on L1 -2. waits for a finalized block to exist on L1 (so that it can be taken as a starting block for roll ups) -3. deploys the L1 contracts -It may restart a few times after running into errors. +### Ports +It is usually necessary to expose certain container ports on one or more the host's addresses to allow incoming connections. +Any ports defined in the Docker compose file are exposed by default with random port assignments, bound to "any" interface (IP address 0.0.0.0), but the port mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`. +In addition, a stack-wide port mapping "recipe" can be applied at the time the +`laconic-so deploy init` command is run, by supplying the desired recipe with the `--map-ports-to-host` option. The following recipes are supported: +| Recipe | Host Port Mapping | +|--------|-------------------| +| any-variable-random | Bind to 0.0.0.0 using a random port assigned at start time (default) | +| localhost-same | Bind to 127.0.0.1 using the same port number as exposed by the containers | +| any-same | Bind to 0.0.0.0 using the same port number as exposed by the containers | +| localhost-fixed-random | Bind to 127.0.0.1 using a random port number selected at the time the command is run (not checked for already in use)| +| any-fixed-random | Bind to 0.0.0.0 using a random port number selected at the time the command is run (not checked for already in use) | + +For example, you may wish to use `any-fixed-random` to generate the initial mappings and then edit the spec file to set the `fixturenet-eth-geth-1` RPC to port 8545 and the `op-geth` RPC to port 9545 on the host. + +Or, you may wish to use `any-same` for the initial mappings -- in which case you'll have to edit the spec to file to ensure the various geth instances aren't all trying to publish to host ports 8545/8546 at once. + +### Data volumes +Container data volumes are bind-mounted to specified paths in the host filesystem. +The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory. The default mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`. + +--- +Once you've made any needed changes to the spec file, create a deployment from it: +```bash +laconic-so --stack fixturenet-optimism deploy create --spec-file fixturenet-optimism-spec.yml --deployment-dir fixturenet-optimism-deployment +``` + +## Start the stack +Start the deployment: +```bash +laconic-so deployment --dir fixturenet-optimism-deployment start +``` +1. The `fixturenet-eth` L1 chain will start up first and begin producing blocks. +2. The `fixturenet-optimism-contracts` service will configure and deploy the Optimism contracts to L1, exiting when complete. This may take several minutes; you can follow the progress by following the container's logs (see below). +3. The `op-node` and `op-geth` services will initialize themselves (if not already initialized) and start +4. The remaining services, `op-batcher` and `op-proposer` will start + +### Logs To list and monitor the running containers: ```bash @@ -65,22 +99,69 @@ docker ps docker logs -f ``` -## Clean up +## Example: bridge some ETH from L1 to L2 -Stop all services running in the background: +Send some ETH from the desired account to the `L1StandardBridgeProxy` contract on L1 to test bridging to L2. +We can use the testing account `0xe6CE22afe802CAf5fF7d3845cec8c736ecc8d61F` which is pre-funded and unlocked, and the `cerc/foundry:local` container to make use of the `cast` cli. + +1. Note the docker network the stack is running on: ```bash -laconic-so --stack fixturenet-optimism deploy down 30 +docker network ls +# The network name will be something like laconic-[some_hash]_default +``` +2. Set some variables: +```bash +L1_RPC=http://fixturenet-eth-geth-1:8545 +L2_RPC=http://op-geth:8545 +NETWORK= +DEPLOYMENT_CONTEXT= +ACCOUNT=0xe6CE22afe802CAf5fF7d3845cec8c736ecc8d61F ``` -Clear volumes created by this stack: +If you need to check the L1 chain-id, you can use: +```bash +docker run --rm --network $NETWORK cerc/foundry:local "cast chain-id --rpc-url $L1_RPC" +``` + +3. Check the account starting balance on L2 (it should be 0): +```bash +docker run --rm --network $NETWORK cerc/foundry:local "cast balance $ACCOUNT --rpc-url $L2_RPC" +# 0 +``` + +4. Read the bridge contract address from the L1 deployment records in the `op-node` container: +```bash +# get the container id for op-node +NODE_CONTAINER=$(docker ps --filter "name=op-node" -q) +BRIDGE=$(docker exec $NODE_CONTAINER cat /l1-deployment/$DEPLOYMENT_CONTEXT/L1StandardBridgeProxy.json | jq -r .address) +``` + +5. Use cast to send some ETH to the bridge contract: +```bash +docker run --rm --network $NETWORK cerc/foundry:local "cast send --from $ACCOUNT --value 1ether $BRIDGE --rpc-url $L1_RPC" +``` + +6. Allow a couple minutes for the bridge to complete + +7. Check the L2 balance again (it should show the bridged funds): +```bash +docker run --rm --network $NETWORK cerc/foundry:local "cast balance $ACCOUNT --rpc-url $L2_RPC" +# 1000000000000000000 +``` + +## Clean up + +To stop all services running in the background, while preserving chain data: ```bash -# List all relevant volumes -docker volume ls -q --filter "name=.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data" +laconic-so deployment --dir fixturenet-optimism-deployment stop +``` -# Remove all the listed volumes -docker volume rm $(docker volume ls -q --filter "name=.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data") +To stop all services and also delete chain data: + +```bash +laconic-so deployment --dir fixturenet-optimism-deployment stop --delete-volumes ``` ## Troubleshooting diff --git a/stack_orchestrator/data/stacks/fixturenet-optimism/deploy/commands.py b/stack_orchestrator/data/stacks/fixturenet-optimism/deploy/commands.py new file mode 100644 index 00000000..fa757cf5 --- /dev/null +++ b/stack_orchestrator/data/stacks/fixturenet-optimism/deploy/commands.py @@ -0,0 +1,39 @@ +# Copyright © 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from stack_orchestrator.deploy.deployment_context import DeploymentContext +from ruamel.yaml import YAML + + +def create(context: DeploymentContext, extra_args): + # Slightly modify the base fixturenet-eth compose file to replace the startup script for fixturenet-eth-geth-1 + # We need to start geth with the flag to allow non eip-155 compliant transactions in order to publish the + # deterministic-deployment-proxy contract, which itself is a prereq for Optimism contract deployment + fixturenet_eth_compose_file = context.deployment_dir.joinpath('compose', 'docker-compose-fixturenet-eth.yml') + + with open(fixturenet_eth_compose_file, 'r') as yaml_file: + yaml = YAML() + yaml_data = yaml.load(yaml_file) + + new_script = '../config/fixturenet-optimism/run-geth.sh:/opt/testnet/run.sh' + + if new_script not in yaml_data['services']['fixturenet-eth-geth-1']['volumes']: + yaml_data['services']['fixturenet-eth-geth-1']['volumes'].append(new_script) + + with open(fixturenet_eth_compose_file, 'w') as yaml_file: + yaml = YAML() + yaml.dump(yaml_data, yaml_file) + + return None diff --git a/stack_orchestrator/data/stacks/fixturenet-optimism/l2-only.md b/stack_orchestrator/data/stacks/fixturenet-optimism/l2-only.md index 4e9daf43..4299ca8d 100644 --- a/stack_orchestrator/data/stacks/fixturenet-optimism/l2-only.md +++ b/stack_orchestrator/data/stacks/fixturenet-optimism/l2-only.md @@ -1,4 +1,4 @@ -# fixturenet-optimism +# fixturenet-optimism (L2-only) Instructions to setup and deploy L2 fixturenet using [Optimism](https://stack.optimism.io) @@ -28,9 +28,43 @@ This should create the required docker images in the local image registry: * `cerc/optimism-op-batcher` * `cerc/optimism-op-proposer` -## Deploy +## Create a deployment -Create and update an env file to be used in the next step ([defaults](../../config/fixturenet-optimism/l1-params.env)): +First, create a spec file for the deployment, which will map the stack's ports and volumes to the host: +```bash +laconic-so --stack fixturenet-optimism deploy init --map-ports-to-host any-fixed-random --output fixturenet-optimism-spec.yml +``` +### Ports +It is usually necessary to expose certain container ports on one or more the host's addresses to allow incoming connections. +Any ports defined in the Docker compose file are exposed by default with random port assignments, bound to "any" interface (IP address 0.0.0.0), but the port mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`. + +In addition, a stack-wide port mapping "recipe" can be applied at the time the +`laconic-so deploy init` command is run, by supplying the desired recipe with the `--map-ports-to-host` option. The following recipes are supported: +| Recipe | Host Port Mapping | +|--------|-------------------| +| any-variable-random | Bind to 0.0.0.0 using a random port assigned at start time (default) | +| localhost-same | Bind to 127.0.0.1 using the same port number as exposed by the containers | +| any-same | Bind to 0.0.0.0 using the same port number as exposed by the containers | +| localhost-fixed-random | Bind to 127.0.0.1 using a random port number selected at the time the command is run (not checked for already in use)| +| any-fixed-random | Bind to 0.0.0.0 using a random port number selected at the time the command is run (not checked for already in use) | + +For example, you may wish to use `any-fixed-random` to generate the initial mappings and then edit the spec file to set the `op-geth` RPC to an easy to remember port like 8545 or 9545 on the host. + +### Data volumes +Container data volumes are bind-mounted to specified paths in the host filesystem. +The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory. The default mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`. + +--- +Once you've made any needed changes to the spec file, create a deployment from it: +```bash +laconic-so --stack fixturenet-optimism deploy create --spec-file fixturenet-optimism-spec.yml --deployment-dir fixturenet-optimism-deployment +``` + +Finally, open the `stack.yml` file inside your deployment directory and, under the `pods:` section, remove (or comment out) the entry for `fixturenet-eth`. This will prevent the deployment from trying to spin up a new L1 chain when starting the stack. + +## Set chain env variables + +Inside the deployment directory, open the file `config.env` and add the following variables to point the stack at your L1 rpc and provide account credentials ([defaults](../../config/fixturenet-optimism/l1-params.env)): ```bash # External L1 endpoint @@ -45,30 +79,29 @@ Create and update an env file to be used in the next step ([defaults](../../conf CERC_L1_ACCOUNTS_CSV_URL= # OR - # Specify the required account credentials + # Specify the required account credentials for the Admin account + # Other generated accounts will be funded from this account, so it should contain ~20 Eth CERC_L1_ADDRESS= CERC_L1_PRIV_KEY= - CERC_L1_ADDRESS_2= - CERC_L1_PRIV_KEY_2= ``` -* NOTE: If L1 is running on the host machine, use `host.docker.internal` as the hostname to access the host port - -Deploy the stack: +* NOTE: If L1 is running on the host machine, use `host.docker.internal` as the hostname to access the host port, or use the `ip a` command to find the IP address of the `docker0` interface (this will usually be something like `172.17.0.1` or `172.18.0.1`) +## Start the stack +Start the deployment: ```bash -laconic-so --stack fixturenet-optimism deploy --include fixturenet-optimism --env-file up +laconic-so deployment --dir fixturenet-optimism-deployment start ``` +1. The stack will check for a response from the L1 endpoint specified in your env file. +2. The `fixturenet-optimism-contracts` service will configure and deploy the Optimism contracts to L1, exiting when complete. This may take several minutes; you can follow the progress by following the container's logs (see below). +3. The `op-node` and `op-geth` services will initialize themselves (if not already initialized) and start +4. The remaining services, `op-batcher` and `op-proposer` will start -The `fixturenet-optimism-contracts` service may take a while (`~15 mins`) to complete running as it: -1. waits for the 'Merge' to happen on L1 -2. waits for a finalized block to exist on L1 (so that it can be taken as a starting block for roll ups) -3. deploys the L1 contracts - -To list down and monitor the running containers: +### Logs +To list and monitor the running containers: ```bash -laconic-so --stack fixturenet-optimism deploy --include fixturenet-optimism ps +laconic-so --stack fixturenet-optimism deploy ps # With status docker ps @@ -79,20 +112,16 @@ docker logs -f ## Clean up -Stop all services running in the background: +To stop all L2 services running in the background, while preserving chain data: ```bash -laconic-so --stack fixturenet-optimism deploy --include fixturenet-optimism down 30 +laconic-so deployment --dir fixturenet-optimism-deployment stop ``` -Clear volumes created by this stack: +To stop all L2 services and also delete chain data: ```bash -# List all relevant volumes -docker volume ls -q --filter "name=.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data" - -# Remove all the listed volumes -docker volume rm $(docker volume ls -q --filter "name=.*l1_deployment|.*l2_accounts|.*l2_config|.*l2_geth_data") +laconic-so deployment --dir fixturenet-optimism-deployment stop --delete-volumes ``` ## Troubleshooting diff --git a/stack_orchestrator/data/stacks/fixturenet-optimism/stack.yml b/stack_orchestrator/data/stacks/fixturenet-optimism/stack.yml index 75c7620b..bca34b16 100644 --- a/stack_orchestrator/data/stacks/fixturenet-optimism/stack.yml +++ b/stack_orchestrator/data/stacks/fixturenet-optimism/stack.yml @@ -5,8 +5,8 @@ repos: - git.vdb.to/cerc-io/go-ethereum@v1.11.6-statediff-v5 - git.vdb.to/cerc-io/lighthouse - github.com/dboreham/foundry - - github.com/ethereum-optimism/optimism@v1.0.4 - - github.com/ethereum-optimism/op-geth@v1.101105.2 + - github.com/ethereum-optimism/optimism@op-node/v1.3.0 + - github.com/ethereum-optimism/op-geth@v1.101304.0 containers: - cerc/go-ethereum - cerc/lighthouse diff --git a/stack_orchestrator/data/stacks/graph-node/README.md b/stack_orchestrator/data/stacks/graph-node/README.md index 0527efc0..df3ae1eb 100644 --- a/stack_orchestrator/data/stacks/graph-node/README.md +++ b/stack_orchestrator/data/stacks/graph-node/README.md @@ -59,7 +59,7 @@ ports: Create deployment: ```bash -laconic-so deploy create --spec-file graph-node-spec.yml --deployment-dir graph-node-deployment +laconic-so --stack graph-node deploy create --spec-file graph-node-spec.yml --deployment-dir graph-node-deployment ``` ## Start the stack diff --git a/stack_orchestrator/data/stacks/merkl-sushiswap-v3/README.md b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/README.md new file mode 100644 index 00000000..4284c2ad --- /dev/null +++ b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/README.md @@ -0,0 +1,81 @@ +# Merkl SushiSwap v3 Watcher + +## Setup + +Clone required repositories: + +```bash +laconic-so --stack merkl-sushiswap-v3 setup-repositories --git-ssh --pull +``` + +Build the container images: + +```bash +laconic-so --stack merkl-sushiswap-v3 build-containers +``` + +## Deploy + +### Configuration + +Create and update an env file to be used in the next step: + + ```bash + # External Filecoin (ETH RPC) endpoint to point the watcher + CERC_ETH_RPC_ENDPOINT= + ``` + +### Deploy the stack + +```bash +laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 --env-file up +``` + +* To list down and monitor the running containers: + + ```bash + laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 ps + + # With status + docker ps -a + + # Check logs for a container + docker logs -f + ``` + +* Open the GQL playground at http://localhost:3007/graphql + + ```graphql + { + _meta { + block { + number + timestamp + } + hasIndexingErrors + } + + factories { + id + poolCount + } + } + ``` + +## Clean up + +Stop all the services running in background: + +```bash +laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 down +``` + +Clear volumes created by this stack: + +```bash +# List all relevant volumes +docker volume ls -q --filter "name=merkl_sushiswap_v3" + +# Remove all the listed volumes +docker volume rm $(docker volume ls -q --filter "name=merkl_sushiswap_v3") +``` diff --git a/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml new file mode 100644 index 00000000..847df33c --- /dev/null +++ b/stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml @@ -0,0 +1,9 @@ +version: "1.0" +name: merkl-sushiswap-v3 +description: "SushiSwap v3 watcher stack" +repos: + - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.6 +containers: + - cerc/watcher-merkl-sushiswap-v3 +pods: + - watcher-merkl-sushiswap-v3 diff --git a/stack_orchestrator/data/stacks/monitoring/README.md b/stack_orchestrator/data/stacks/monitoring/README.md new file mode 100644 index 00000000..74b1bff8 --- /dev/null +++ b/stack_orchestrator/data/stacks/monitoring/README.md @@ -0,0 +1,180 @@ +# monitoring + +* Instructions to setup and run a Prometheus server and Grafana dashboard +* Comes with the following built-in exporters / dashboards: + * Chain Head Exporter - for tracking chain heads given external ETH RPC endpoints + * Watchers dashboard + * [Prometheus Blackbox](https://grafana.com/grafana/dashboards/7587-prometheus-blackbox-exporter/) - for tracking HTTP endpoints + * [NodeJS Application Dashboard](https://grafana.com/grafana/dashboards/11159-nodejs-application-dashboard/) - for default NodeJS metrics + * [PostgreSQL Database](https://grafana.com/grafana/dashboards/9628-postgresql-database/) - for monitoring Postgres dbs + * [Node Exporter Full](https://grafana.com/grafana/dashboards/1860-node-exporter-full/) - for monitoring system metrics +* See [monitoring-watchers.md](./monitoring-watchers.md) for an example usage of the stack with pre-configured dashboards for watchers + +## Setup + +Clone required repositories: + +```bash +laconic-so --stack monitoring setup-repositories --git-ssh --pull +``` + +Build the container images: + +```bash +laconic-so --stack monitoring build-containers +``` + +## Create a deployment + +First, create a spec file for the deployment, which will map the stack's ports and volumes to the host: + +```bash +laconic-so --stack monitoring deploy init --output monitoring-spec.yml +``` + +### Ports + +Edit `network` in spec file to map container ports to same ports in host: + +``` +... +network: + ports: + prometheus: + - '9090:9090' + grafana: + - '3000:3000' +... +``` + +### Data volumes + +Container data volumes are bind-mounted to specified paths in the host filesystem. +The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory. The default mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`. + +--- + +Once you've made any needed changes to the spec file, create a deployment from it: + +```bash +laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --deployment-dir monitoring-deployment +``` + +## Configure + +### Prometheus Config + +* Add desired scrape configs to prometheus config file (`monitoring-deployment/config/monitoring/prometheus/prometheus.yml`) in the deployment folder; for example: + + ```yml + ... + - job_name: + metrics_path: /metrics/path + scheme: http + static_configs: + - targets: [':'] + ``` + +* Node exporter: update the `node` job to add any node-exporter targets to be monitored: + + ```yml + ... + - job_name: 'node' + ... + static_configs: + # Add node-exporter targets to be monitored below + - targets: [example-host:9100] + labels: + instance: 'my-host' + ``` + +* Blackbox (in-stack exporter): update the `blackbox` job to add any endpoints to be monitored on the Blackbox dashboard: + + ```yml + ... + - job_name: 'blackbox' + ... + static_configs: + # Add URLs to be monitored below + - targets: + - + - + ``` + +* Postgres (in-stack exporter): + * Update the `postgres` job to add Postgres db targets to be monitored: + + ```yml + ... + - job_name: 'postgres' + ... + static_configs: + # Add DB targets below + - targets: [example-server:5432] + labels: + instance: 'example-db' + ``` + * Add database credentials to be used in `auth_modules` in the postgres-exporter config file (`monitoring-deployment/config/monitoring/postgres-exporter.yml`) + +Note: Use `host.docker.internal` as host to access ports on the host machine + +### Grafana Config + +Place the dashboard json files in grafana dashboards config directory (`monitoring-deployment/config/monitoring/grafana/dashboards`) in the deployment folder + +### Env + +Set the following env variables in the deployment env config file (`monitoring-deployment/config.env`): + + ```bash + # For chain-head exporter + + # External ETH RPC endpoint (ethereum) + # (Optional, default: https://mainnet.infura.io/v3) + CERC_ETH_RPC_ENDPOINT= + + # Infura key to be used + # (Optional, used with ETH_RPC_ENDPOINT if provided) + CERC_INFURA_KEY= + + # External ETH RPC endpoint (filecoin) + # (Optional, default: https://api.node.glif.io/rpc/v1) + CERC_FIL_RPC_ENDPOINT= + + # Grafana server host URL (used in various links in alerts, etc.) + # (Optional, default: http://localhost:3000) + GF_SERVER_ROOT_URL= + ``` + +## Start the stack + +Start the deployment: + +```bash +laconic-so deployment --dir monitoring-deployment start +``` + +* List and check the health status of all the containers using `docker ps` and wait for them to be `healthy` + +* Grafana should now be visible at http://localhost:3000 with configured dashboards + +## Clean up + +To stop monitoring services running in the background, while preserving data: + +```bash +# Only stop the docker containers +laconic-so deployment --dir monitoring-deployment stop + +# Run 'start' to restart the deployment +``` + +To stop monitoring services and also delete data: + +```bash +# Stop the docker containers +laconic-so deployment --dir monitoring-deployment stop --delete-volumes + +# Remove deployment directory (deployment will have to be recreated for a re-run) +rm -rf monitoring-deployment +``` diff --git a/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md new file mode 100644 index 00000000..bce2d84b --- /dev/null +++ b/stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md @@ -0,0 +1,170 @@ +# Monitoring Watchers + +Instructions to setup and run monitoring stack with configured Watchers dashboard + +## Create a deployment + +After completing [setup](./README.md#setup), create a spec file for the deployment, which will map the stack's ports and volumes to the host: + +```bash +laconic-so --stack monitoring deploy init --output monitoring-watchers-spec.yml +``` + +### Ports + +Edit `network` in spec file to map container ports to same ports in host: + +``` +... +network: + ports: + prometheus: + - '9090:9090' + grafana: + - '3000:3000' +... +``` + +--- + +Once you've made any needed changes to the spec file, create a deployment from it: + +```bash +laconic-so --stack monitoring deploy create --spec-file monitoring-watchers-spec.yml --deployment-dir monitoring-watchers-deployment +``` + +## Configure + +### Prometheus scrape config + +Add the following scrape configs to prometheus config file (`monitoring-watchers-deployment/config/monitoring/prometheus/prometheus.yml`) in the deployment folder: + + ```yml + ... + - job_name: 'blackbox' + ... + static_configs: + - targets: + - + ... + - job_name: azimuth + scrape_interval: 10s + metrics_path: /metrics + scheme: http + static_configs: + - targets: ['AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_PORT'] + labels: + instance: 'azimuth' + chain: 'ethereum' + - targets: ['CENSURES_WATCHER_HOST:CENSURES_WATCHER_PORT'] + labels: + instance: 'censures' + chain: 'ethereum' + - targets: ['CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_PORT'] + labels: + instance: 'claims' + chain: 'ethereum' + - targets: ['CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_PORT'] + labels: + instance: 'conditional_star_release' + chain: 'ethereum' + - targets: ['DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_PORT'] + labels: + instance: 'delegated_sending' + chain: 'ethereum' + - targets: ['ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_PORT'] + labels: + instance: 'ecliptic' + chain: 'ethereum' + - targets: ['LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_PORT'] + labels: + instance: 'linear_star_release' + chain: 'ethereum' + - targets: ['POLLS_WATCHER_HOST:POLLS_WATCHER_PORT'] + labels: + instance: 'polls' + chain: 'ethereum' + + - job_name: sushi + scrape_interval: 20s + metrics_path: /metrics + scheme: http + static_configs: + - targets: ['SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_PORT'] + labels: + instance: 'sushiswap' + chain: 'filecoin' + - targets: ['MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_PORT'] + labels: + instance: 'merkl_sushiswap' + chain: 'filecoin' + ``` + +Add scrape config as done above for any additional watcher to add it to the Watchers dashboard. + +### Grafana alerts config + +Place the pre-configured watcher alerts rules in Grafana provisioning directory: + + ```bash + cp monitoring-watchers-deployment/config/monitoring/watcher-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/ + ``` + +Update the alerting contact points config (`monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/contactpoints.yml`) with desired contact points + +Add corresponding routes to the notification policies config (`monitoring-watchers-deployment/monitoring/grafana/provisioning/alerting/policies.yaml`) with appropriate object-matchers: + + ```yml + ... + routes: + - receiver: SlackNotifier + object_matchers: + # Add matchers below + - ['grafana_folder', '=', 'WatcherAlerts'] + ``` + +### Env + +Set the following env variables in the deployment env config file (`monitoring-watchers-deployment/config.env`): + + ```bash + # Infura key to be used + CERC_INFURA_KEY= + + # Grafana server host URL to be used + # (Optional, default: http://localhost:3000) + GF_SERVER_ROOT_URL= + ``` + +## Start the stack + +Start the deployment: + +```bash +laconic-so deployment --dir monitoring-watchers-deployment start +``` + +* List and check the health status of all the containers using `docker ps` and wait for them to be `healthy` + +* Grafana should now be visible at http://localhost:3000 with configured dashboards + +## Clean up + +To stop monitoring services running in the background, while preserving data: + +```bash +# Only stop the docker containers +laconic-so deployment --dir monitoring-watchers-deployment stop + +# Run 'start' to restart the deployment +``` + +To stop monitoring services and also delete data: + +```bash +# Stop the docker containers +laconic-so deployment --dir monitoring-watchers-deployment stop --delete-volumes + +# Remove deployment directory (deployment will have to be recreated for a re-run) +rm -rf monitoring-watchers-deployment +``` diff --git a/stack_orchestrator/data/stacks/monitoring/stack.yml b/stack_orchestrator/data/stacks/monitoring/stack.yml new file mode 100644 index 00000000..058ca971 --- /dev/null +++ b/stack_orchestrator/data/stacks/monitoring/stack.yml @@ -0,0 +1,9 @@ +version: "0.1" +name: monitoring +repos: + - github.com/cerc-io/watcher-ts@v0.2.79 +containers: + - cerc/watcher-ts +pods: + - prom-server + - grafana diff --git a/stack_orchestrator/data/stacks/node-exporter/README.md b/stack_orchestrator/data/stacks/node-exporter/README.md new file mode 100644 index 00000000..30376af9 --- /dev/null +++ b/stack_orchestrator/data/stacks/node-exporter/README.md @@ -0,0 +1,17 @@ +# node-exporter + +## Start the stack + +```bash +laconic-so --stack node-exporter deploy up +``` + +* The host node's metrics can be accessed at `http://localhost:9100/metrics` + +## Clean up + +Stop the node-exporter running in background: + +```bash +laconic-so --stack node-exporter deploy down +``` diff --git a/stack_orchestrator/data/stacks/node-exporter/stack.yml b/stack_orchestrator/data/stacks/node-exporter/stack.yml new file mode 100644 index 00000000..a9b3b7b3 --- /dev/null +++ b/stack_orchestrator/data/stacks/node-exporter/stack.yml @@ -0,0 +1,6 @@ +version: "0.1" +name: node-exporter +repos: +containers: +pods: + - node-exporter diff --git a/stack_orchestrator/data/stacks/osmosis-urbit-app/README.md b/stack_orchestrator/data/stacks/osmosis-urbit-app/README.md new file mode 100644 index 00000000..59da7411 --- /dev/null +++ b/stack_orchestrator/data/stacks/osmosis-urbit-app/README.md @@ -0,0 +1,158 @@ +# Self-hosted Osmosis Front End On Urbit + +Instructions to build and deploy: +- Urbit +- An ipfs node +- The osmosis front end + +## Setup + +Clone required repositories: + +```bash +laconic-so --stack osmosis-urbit-app setup-repositories --pull + +# If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories and re-run the command +``` + +Build the container images: + +```bash +laconic-so --stack osmosis-urbit-app build-containers +``` + +## Create a deployment + +First, create a spec file for the deployment, which will map the stack's ports and volumes to the host: + +```bash +laconic-so --stack osmosis-urbit-app deploy init --output osmosis-urbit-app-spec.yml +``` + +### Ports + +Edit `network` in spec file to map container ports to same ports in host: + +```yml +... +network: + ports: + urbit-fake-ship: + - '8080:80' + proxy-server: + - '4000:4000' + ipfs: + - '8081:8080' + - '5001:5001' +``` + +Note: Skip the `ipfs` ports if need to use an externally running IPFS node + +### Data volumes + +Container data volumes are bind-mounted to specified paths in the host filesystem. +The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory. The default mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`. + +--- + +Once you've made any needed changes to the spec file, create a deployment from it: + +```bash +laconic-so --stack osmosis-urbit-app deploy create --spec-file osmosis-urbit-app-spec.yml --deployment-dir osmosis-urbit-app-deployment +``` + +## Set env variables + +Inside the deployment directory, open the file `config.env` and set the following env variables: + +(Note: Use the following config as is if running the stack locally) + + ```bash + # App to be installed (Do not change) + CERC_URBIT_APP=osmosis + + # Osmosis API base URL + # Set this to proxy server endpoint for osmosis app + # (Eg. http://localhost:4000 - in case stack is being run locally with proxy enabled) + # (Eg. https://abc.xyz.com - in case https://abc.xyz.com is pointed to the proxy endpoint) + CERC_WEB_API_BASE_URL=http://localhost:4000 + + # Optional + + # Whether to enable app installation on Urbit + # (just builds and uploads the glob file if disabled) (Default: true) + CERC_ENABLE_APP_INSTALL= + + # Whether to run the proxy server + # (Disable only if proxy not required to be run) (Default: true) + CERC_ENABLE_PROXY= + + # Proxy server configuration + # Used only if proxy is enabled + + # Upstream API URL + CERC_PROXY_UPSTREAM=https://app.osmosis.zone + + # IPFS configuration + + # IFPS endpoint to host the glob file on + # (Default: http://ipfs:5001 pointing to in-stack IPFS node) + CERC_IPFS_GLOB_HOST_ENDPOINT= + + # IFPS endpoint to fetch the glob file from + # (Default: http://ipfs:8080 pointing to in-stack IPFS node) + CERC_IPFS_SERVER_ENDPOINT= + ``` + +## Start the stack + +Start the deployment: + +```bash +laconic-so deployment --dir osmosis-urbit-app-deployment start +``` + +* List and check the health status of all the containers using `docker ps` and wait for them to be `healthy` + +* Run the following to get login password for Urbit web interface: + + ```bash + laconic-so deployment --dir osmosis-urbit-app-deployment exec urbit-fake-ship "curl -s --data '{\"source\":{\"dojo\":\"+code\"},\"sink\":{\"stdout\":null}}' http://localhost:12321" + + # Expected output: "\n"% + ``` + +* Open the Urbit web UI at http://localhost:8080 and use the `PASSWORD` from previous step to login + +* The osmosis app is not available when starting stack for the first time. Check `urbit-fake-ship` logs to see that app has installed: + + ```bash + laconic-so deployment --dir osmosis-urbit-app-deployment logs -f + + # Expected output: + # laconic-3ccf7ee79bdae874-urbit-fake-ship-1 | docket: fetching %http glob for %osmosis desk + # laconic-3ccf7ee79bdae874-urbit-fake-ship-1 | ">="">="osmosis app installed + ``` + +* The osmosis app will be now visible at http://localhost:8080 + +## Clean up + +To stop all osmosis-urbit-app services running in the background, while preserving data: + +```bash +# Only stop the docker containers +laconic-so deployment --dir osmosis-urbit-app-deployment stop + +# Run 'start' to restart the deployment +``` + +To stop all osmosis-urbit-app services and also delete data: + +```bash +# Stop the docker containers +laconic-so deployment --dir osmosis-urbit-app-deployment stop --delete-volumes + +# Remove deployment directory (deployment will have to be recreated for a re-run) +rm -r osmosis-urbit-app-deployment +``` diff --git a/stack_orchestrator/data/stacks/osmosis-urbit-app/stack.yml b/stack_orchestrator/data/stacks/osmosis-urbit-app/stack.yml new file mode 100644 index 00000000..2cf3e9fd --- /dev/null +++ b/stack_orchestrator/data/stacks/osmosis-urbit-app/stack.yml @@ -0,0 +1,13 @@ +version: "0.1" +name: osmosis-urbit-app +repos: + - github.com/cerc-io/osmosis-frontend@laconic + - github.com/cerc-io/watcher-ts@v0.2.78 +containers: + - cerc/osmosis-front-end-urbit + - cerc/watcher-ts +pods: + - osmosis-front-end-urbit + - proxy-server + - fixturenet-urbit + - kubo diff --git a/stack_orchestrator/data/stacks/osmosis/README.md b/stack_orchestrator/data/stacks/osmosis/README.md new file mode 100644 index 00000000..54cda3da --- /dev/null +++ b/stack_orchestrator/data/stacks/osmosis/README.md @@ -0,0 +1,151 @@ +# self-hosted osmosis + +Instructions to build and deploy: +- Self-hosted gitea +- An ipfs node +- The osmosis front end +- A laconicd chain + +## Setup + +Clone required repositories: + +```bash +laconic-so --stack osmosis setup-repositories --pull + +# If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories and re-run the command +``` + +Build the container images: + +```bash +# TODO: support image for the gitea package registry +# laconic-so --stack build-support build-containers + +laconic-so --stack osmosis build-containers +``` + +## Create a deployment + +First, create a spec file for the deployment, which will map the stack's ports and volumes to the host: +```bash +laconic-so --stack osmosis deploy init --output osmosis-spec.yml +``` + +### Ports + +Edit `network` in spec file to map container ports to same ports in host + +``` +... +network: + ports: + proxy-server: + - '4000:4000' + nginx: + - '3000:80' +``` + +### Data volumes + +Container data volumes are bind-mounted to specified paths in the host filesystem. +The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory. The default mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`. + +--- + +Once you've made any needed changes to the spec file, create a deployment from it: +```bash +laconic-so --stack osmosis deploy create --spec-file osmosis-spec.yml --deployment-dir osmosis-deployment +``` + +## Set env variables + +Inside the deployment directory, open the file `config.env` and set the following env variables: + + (Note: Following config can be used as is if the stack is being run locally) + + ```bash + # Osmosis API base URL + # Set this to proxy server endpoint for osmosis app + # (Eg. http://localhost:4000 - in case stack is being run locally with proxy enabled) + # (Eg. https://abc.xyz.com - in case https://abc.xyz.com is pointed to the proxy endpoint) + CERC_WEB_API_BASE_URL=http://localhost:4000 + + # Optional + + # Whether to run the proxy server + # (Disable only if proxy not required to be run) (Default: true) + CERC_ENABLE_PROXY= + + # Proxy server configuration + # Used only if proxy is enabled + + # Upstream API URL + CERC_PROXY_UPSTREAM=https://app.osmosis.zone + ``` + +## Start the stack + +Start the deployment: + +```bash +laconic-so deployment --dir osmosis-deployment start +``` + +* The `osmosis-front-end` container will take some time to run to completion as it builds the front-end app with given configuration; same can be tracked in the logs: + + ```bash + laconic-so deployment --dir osmosis-deployment logs -f osmosis-front-end + ``` + +* List and check the health status of all the containers using `docker ps` and wait for them to be `healthy` + +* The web app can be accessed at http://localhost:3000 + +## Laconic registry + +Setup a test chain: + +```bash +export CERC_NPM_REGISTRY_URL=https://git.vdb.to/api/packages/cerc-io/npm/ + +laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd,git.vdb.to/cerc-io/laconic-sdk,git.vdb.to/cerc-io/laconic-registry-cli,git.vdb.to/cerc-io/laconic-console + +laconic-so --stack fixturenet-laconic-loaded build-containers + +export LACONIC_HOSTED_ENDPOINT=http:// + +laconic-so --stack fixturenet-laconic-loaded deploy up +``` + +then `docker exec` into the `laconicd` container and either export the private key or create a new one and send funds to it. Use that private key for `LACONIC_HOTWALLET_KEY`. + +## Clean up + +To stop all osmosis services running in the background, while preserving data: + +```bash +# Only stop the docker containers +laconic-so deployment --dir osmosis-deployment stop + +# Run 'start' to restart the deployment +``` + +To stop all osmosis services and also delete data: + +```bash +# Stop the docker containers +laconic-so deployment --dir osmosis-deployment stop --delete-volumes + +# Remove deployment directory (deployment will have to be recreated for a re-run) +rm -r osmosis-deployment +``` + +To stop stack running Laconic registry + +```bash +laconic-so --stack fixturenet-laconic-loaded deploy down + +# Delete volumes +laconic-so --stack fixturenet-laconic-loaded deploy down --delete-volumes +``` diff --git a/stack_orchestrator/data/stacks/osmosis/stack.yml b/stack_orchestrator/data/stacks/osmosis/stack.yml new file mode 100644 index 00000000..221fbb60 --- /dev/null +++ b/stack_orchestrator/data/stacks/osmosis/stack.yml @@ -0,0 +1,30 @@ +version: "0.1" +name: osmosis +repos: + # For gitea + # - git.vdb.to/cerc-io/hosting@names-for-so + # - gitea.com/gitea/act_runner + # For osmosis frontend + - github.com/cerc-io/watcher-ts@v0.2.78 + - github.com/cerc-io/osmosis-frontend@laconic # TODO: Use release +containers: + # - cerc/act-runner + # - cerc/act-runner-task-executor + - cerc/watcher-ts + - cerc/osmosis-front-end +pods: + # - kubo + - proxy-server + - osmosis-front-end + # TODO: Fix pod config for deployment command + # TODO: mirroring all of osmosis repos: https://git.vdb.to/cerc-io/hosting/pulls/42 + # - name: gitea + # repository: cerc-io/hosting + # path: gitea + # pre_start_command: "run-this-first.sh" + # post_start_command: "initialize-gitea.sh" + # - name: act-runner + # repository: cerc-io/hosting + # path: act-runner + # pre_start_command: "pre_start.sh" + # post_start_command: "post_start.sh" diff --git a/stack_orchestrator/data/stacks/package-registry/stack.yml b/stack_orchestrator/data/stacks/package-registry/stack.yml index f6367ab1..e4b06a8c 100644 --- a/stack_orchestrator/data/stacks/package-registry/stack.yml +++ b/stack_orchestrator/data/stacks/package-registry/stack.yml @@ -3,7 +3,7 @@ name: package-registry description: "Local Package Registry" repos: - git.vdb.to/cerc-io/hosting - - gitea.com/gitea/act_runner + - gitea.com/telackey/act_runner@telackey/entrypoint containers: - cerc/act-runner - cerc/act-runner-task-executor diff --git a/stack_orchestrator/data/stacks/proxy-server/README.md b/stack_orchestrator/data/stacks/proxy-server/README.md new file mode 100644 index 00000000..294b01a9 --- /dev/null +++ b/stack_orchestrator/data/stacks/proxy-server/README.md @@ -0,0 +1,79 @@ +# Proxy Server + +Instructions to setup and deploy a HTTP proxy server + +## Setup + +Clone required repository: + +```bash +laconic-so --stack proxy-server setup-repositories --pull + +# If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories mentioned below and re-run the command +``` + +Build the container image: + +```bash +laconic-so --stack proxy-server build-containers +``` + +## Create a deployment + +* First, create a spec file for the deployment, which will allow mapping the stack's ports and volumes to the host: + + ```bash + laconic-so --stack proxy-server deploy init --output proxy-server-spec.yml + ``` + +* Edit `network` in spec file to map container ports to same ports in host: + + ```yml + ... + network: + ports: + proxy-server: + - '4000:4000' + ... + ``` + +* Once you've made any needed changes to the spec file, create a deployment from it: + + ```bash + laconic-so --stack proxy-server deploy create --spec-file proxy-server-spec.yml --deployment-dir proxy-server-deployment + ``` + +* Inside the deployment directory, open the file `config.env` and set the following env variables: + + ```bash + # Whether to run the proxy server (Optional) (Default: true) + CERC_ENABLE_PROXY= + + # Upstream endpoint + # (Eg. https://api.example.org) + CERC_PROXY_UPSTREAM= + + # Origin header to be used (Optional) + # (Eg. https://app.example.org) + CERC_PROXY_ORIGIN_HEADER= + ``` + +## Start the stack + +Start the deployment: + +```bash +laconic-so deployment --dir proxy-server-deployment start +``` + +* List and check the health status of the container using `docker ps` + +* The proxy server will now be listening at http://localhost:4000 + +## Clean up + +To stop the service running in background: + +```bash +laconic-so deployment --dir proxy-server-deployment stop +``` diff --git a/stack_orchestrator/data/stacks/proxy-server/stack.yml b/stack_orchestrator/data/stacks/proxy-server/stack.yml new file mode 100644 index 00000000..313a7f91 --- /dev/null +++ b/stack_orchestrator/data/stacks/proxy-server/stack.yml @@ -0,0 +1,8 @@ +version: "0.1" +name: proxy-server +repos: + - github.com/cerc-io/watcher-ts@v0.2.78 +containers: + - cerc/watcher-ts +pods: + - proxy-server diff --git a/stack_orchestrator/data/stacks/sushiswap-v3/README.md b/stack_orchestrator/data/stacks/sushiswap-v3/README.md new file mode 100644 index 00000000..7116a6d9 --- /dev/null +++ b/stack_orchestrator/data/stacks/sushiswap-v3/README.md @@ -0,0 +1,62 @@ +# SushiSwap v3 Watcher + +## Setup + +Clone required repositories: + +```bash +laconic-so --stack sushiswap-v3 setup-repositories --git-ssh --pull +``` + +Build the container images: + +```bash +laconic-so --stack sushiswap-v3 build-containers +``` + +## Deploy + +### Configuration + +Create and update an env file to be used in the next step: + + ```bash + # External Filecoin (ETH RPC) endpoint to point the watcher + CERC_ETH_RPC_ENDPOINT= + ``` + +### Deploy the stack + +```bash +laconic-so --stack sushiswap-v3 deploy --cluster sushiswap_v3 --env-file up +``` + +* To list down and monitor the running containers: + + ```bash + laconic-so --stack sushiswap-v3 deploy --cluster sushiswap_v3 ps + + # With status + docker ps -a + + # Check logs for a container + docker logs -f + ``` + +## Clean up + +Stop all the services running in background: + +```bash +laconic-so --stack sushiswap-v3 deploy --cluster sushiswap_v3 down +``` + +Clear volumes created by this stack: + +```bash +# List all relevant volumes +docker volume ls -q --filter "name=sushiswap_v3" + +# Remove all the listed volumes +docker volume rm $(docker volume ls -q --filter "name=sushiswap_v3") +``` diff --git a/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml b/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml new file mode 100644 index 00000000..12103d62 --- /dev/null +++ b/stack_orchestrator/data/stacks/sushiswap-v3/stack.yml @@ -0,0 +1,9 @@ +version: "1.0" +name: sushiswap-v3 +description: "SushiSwap v3 watcher stack" +repos: + - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.6 +containers: + - cerc/watcher-sushiswap-v3 +pods: + - watcher-sushiswap-v3 diff --git a/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md b/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md new file mode 100644 index 00000000..cdbb7f6a --- /dev/null +++ b/stack_orchestrator/data/stacks/uniswap-urbit-app/README.md @@ -0,0 +1,168 @@ +# Self-hosted Uniswap Frontend + +Instructions to setup and deploy Uniswap app on Urbit + +Build and deploy: + +- Urbit +- Uniswap app + +## Setup + +Clone required repositories: + +```bash +laconic-so --stack uniswap-urbit-app setup-repositories --pull + +# If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories and re-run the command +``` + +Build the container images: + +```bash +laconic-so --stack uniswap-urbit-app build-containers +``` + +## Create a deployment + +First, create a spec file for the deployment, which will map the stack's ports and volumes to the host: + +```bash +laconic-so --stack uniswap-urbit-app deploy init --output uniswap-urbit-app-spec.yml +``` + +### Ports + +Edit `network` in spec file to map container ports to same ports in host: + +```yml +... +network: + ports: + urbit-fake-ship: + - '8080:80' + proxy-server: + - '4000:4000' + ipfs: + - '8081:8080' + - '5001:5001' +... +``` + +Note: Skip the `ipfs` ports if need to use an externally running IPFS node + +### Data volumes + +Container data volumes are bind-mounted to specified paths in the host filesystem. +The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory. The default mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`. + +--- + +Once you've made any needed changes to the spec file, create a deployment from it: + +```bash +laconic-so --stack uniswap-urbit-app deploy create --spec-file uniswap-urbit-app-spec.yml --deployment-dir uniswap-urbit-app-deployment +``` + +## Set env variables + +Inside the deployment directory, open the file `config.env` and set the following env variables: + + ```bash + # App to be installed (Do not change) + CERC_URBIT_APP=uniswap + + # External RPC endpoints + # https://docs.infura.io/getting-started#2-create-an-api-key + CERC_INFURA_KEY= + + # Uniswap API GQL Endpoint + # Set this to GQL proxy server endpoint for uniswap app + # (Eg. http://localhost:4000/v1/graphql - in case stack is being run locally with proxy enabled) + # (Eg. https://abc.xyz.com/v1/graphql - in case https://abc.xyz.com is pointed to the proxy endpoint) + CERC_UNISWAP_GQL=http://localhost:4000/v1/graphql + + # Optional + + # Whether to enable app installation on Urbit + # (just builds and uploads the glob file if disabled) (Default: true) + CERC_ENABLE_APP_INSTALL= + + # Whether to run the proxy GQL server + # (disable only if proxy not required to be run) (Default: true) + CERC_ENABLE_PROXY= + + # Proxy server configuration + # Used only if proxy is enabled + + # Upstream API URL + # (Eg. https://api.example.org) + CERC_PROXY_UPSTREAM=https://api.uniswap.org + + # Origin header to be used in the proxy + # (Eg. https://app.example.org) + CERC_PROXY_ORIGIN_HEADER=https://app.uniswap.org + + # IPFS configuration + + # IFPS endpoint to host the glob file on + # (Default: http://ipfs:5001 pointing to in-stack IPFS node) + CERC_IPFS_GLOB_HOST_ENDPOINT= + + # IFPS endpoint to fetch the glob file from + # (Default: http://ipfs:8080 pointing to in-stack IPFS node) + CERC_IPFS_SERVER_ENDPOINT= + ``` + +## Start the stack + +Start the deployment: + +```bash +laconic-so deployment --dir uniswap-urbit-app-deployment start +``` + +* List and check the health status of all the containers using `docker ps` and wait for them to be `healthy` + +* Run the following to get login password for Urbit web interface: + + ```bash + laconic-so deployment --dir uniswap-urbit-app-deployment exec urbit-fake-ship "curl -s --data '{\"source\":{\"dojo\":\"+code\"},\"sink\":{\"stdout\":null}}' http://localhost:12321" + + # Expected output: "\n"% + ``` + +* Open the Urbit web UI at http://localhost:8080 and use the `PASSWORD` from previous step to login + +* The uniswap app is not available when starting stack for the first time. Check `urbit-fake-ship` logs to see that app has installed: + + ```bash + laconic-so deployment --dir uniswap-urbit-app-deployment logs -f + + # Expected output: + # laconic-3ccf7ee79bdae874-urbit-fake-ship-1 | docket: fetching %http glob for %uniswap desk + # laconic-3ccf7ee79bdae874-urbit-fake-ship-1 | ">="">="uniswap app installed + ``` + +* The uniswap app will be now visible at http://localhost:8080 + +## Clean up + +To stop all uniswap-urbit-app services running in the background, while preserving data: + +```bash +# Only stop the docker containers +laconic-so deployment --dir uniswap-urbit-app-deployment stop + +# Run 'start' to restart the deployment +``` + +To stop all uniswap-urbit-app services and also delete data: + +```bash +# Stop the docker containers +laconic-so deployment --dir uniswap-urbit-app-deployment stop --delete-volumes + +# Remove deployment directory (deployment will have to be recreated for a re-run) +rm -r uniswap-urbit-app-deployment +``` diff --git a/stack_orchestrator/data/stacks/uniswap-urbit-app/stack.yml b/stack_orchestrator/data/stacks/uniswap-urbit-app/stack.yml new file mode 100644 index 00000000..1e5ad3f5 --- /dev/null +++ b/stack_orchestrator/data/stacks/uniswap-urbit-app/stack.yml @@ -0,0 +1,13 @@ +version: "0.1" +name: uniswap-urbit-app +repos: + - github.com/cerc-io/uniswap-interface@laconic # TODO: Use release + - github.com/cerc-io/watcher-ts@v0.2.78 +containers: + - cerc/uniswap-interface + - cerc/watcher-ts +pods: + - uniswap-interface + - proxy-server + - fixturenet-urbit + - kubo diff --git a/stack_orchestrator/data/stacks/webapp-template/README.md b/stack_orchestrator/data/stacks/webapp-template/README.md new file mode 100644 index 00000000..4441e475 --- /dev/null +++ b/stack_orchestrator/data/stacks/webapp-template/README.md @@ -0,0 +1 @@ +# Template stack for webapp deployments diff --git a/stack_orchestrator/data/stacks/webapp-template/stack.yml b/stack_orchestrator/data/stacks/webapp-template/stack.yml new file mode 100644 index 00000000..d574e764 --- /dev/null +++ b/stack_orchestrator/data/stacks/webapp-template/stack.yml @@ -0,0 +1,7 @@ +version: "1.0" +name: test +description: "Webapp deployment stack" +containers: + - cerc/webapp-template-container +pods: + - webapp-template diff --git a/stack_orchestrator/deploy/compose/deploy_docker.py b/stack_orchestrator/deploy/compose/deploy_docker.py index 79ab1482..b2622820 100644 --- a/stack_orchestrator/deploy/compose/deploy_docker.py +++ b/stack_orchestrator/deploy/compose/deploy_docker.py @@ -16,14 +16,17 @@ from pathlib import Path from python_on_whales import DockerClient, DockerException from stack_orchestrator.deploy.deployer import Deployer, DeployerException, DeployerConfigGenerator +from stack_orchestrator.deploy.deployment_context import DeploymentContext class DockerDeployer(Deployer): name: str = "compose" + type: str - def __init__(self, deployment_dir, compose_files, compose_project_name, compose_env_file) -> None: + def __init__(self, type, deployment_context: DeploymentContext, compose_files, compose_project_name, compose_env_file) -> None: self.docker = DockerClient(compose_files=compose_files, compose_project_name=compose_project_name, compose_env_file=compose_env_file) + self.type = type def up(self, detach, services): try: @@ -37,6 +40,19 @@ class DockerDeployer(Deployer): except DockerException as e: raise DeployerException(e) + def update(self): + try: + return self.docker.compose.restart() + except DockerException as e: + raise DeployerException(e) + + def status(self): + try: + for p in self.docker.compose.ps(): + print(f"{p.name}\t{p.state.status}") + except DockerException as e: + raise DeployerException(e) + def ps(self): try: return self.docker.compose.ps() @@ -61,17 +77,17 @@ class DockerDeployer(Deployer): except DockerException as e: raise DeployerException(e) - def run(self, image, command, user, volumes, entrypoint=None): + def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, ports=[], detach=False): try: - return self.docker.run(image=image, command=command, user=user, volumes=volumes, entrypoint=entrypoint) + return self.docker.run(image=image, command=command, user=user, volumes=volumes, + entrypoint=entrypoint, envs=env, detach=detach, publish=ports, publish_all=len(ports) == 0) except DockerException as e: raise DeployerException(e) class DockerDeployerConfigGenerator(DeployerConfigGenerator): - config_file_name: str = "kind-config.yml" - def __init__(self) -> None: + def __init__(self, type: str) -> None: super().__init__() # Nothing needed at present for the docker deployer diff --git a/stack_orchestrator/deploy/deploy.py b/stack_orchestrator/deploy/deploy.py index 1c467067..18d27a21 100644 --- a/stack_orchestrator/deploy/deploy.py +++ b/stack_orchestrator/deploy/deploy.py @@ -24,6 +24,8 @@ from importlib import resources import subprocess import click from pathlib import Path +from stack_orchestrator import constants +from stack_orchestrator.opts import opts from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, global_options2, get_dev_root_path from stack_orchestrator.deploy.deployer import Deployer, DeployerException from stack_orchestrator.deploy.deployer_factory import getDeployer @@ -39,7 +41,7 @@ from stack_orchestrator.deploy.deployment_create import setup as deployment_setu @click.option("--exclude", help="don\'t start these components") @click.option("--env-file", help="env file to be used") @click.option("--cluster", help="specify a non-default cluster name") -@click.option("--deploy-to", help="cluster system to deploy to (compose or k8s)") +@click.option("--deploy-to", help="cluster system to deploy to (compose or k8s or k8s-kind)") @click.pass_context def command(ctx, include, exclude, env_file, cluster, deploy_to): '''deploy a stack''' @@ -62,11 +64,19 @@ def command(ctx, include, exclude, env_file, cluster, deploy_to): def create_deploy_context( - global_context, deployment_context: DeploymentContext, stack, include, exclude, cluster, env_file, deployer): + global_context, + deployment_context: DeploymentContext, + stack, + include, + exclude, + cluster, + env_file, + deploy_to) -> DeployCommandContext: + # Extract the cluster name from the deployment, if we have one + if deployment_context and cluster is None: + cluster = deployment_context.get_cluster_id() cluster_context = _make_cluster_context(global_context, stack, include, exclude, cluster, env_file) - deployment_dir = deployment_context.deployment_dir if deployment_context else None - # See: https://gabrieldemarmiesse.github.io/python-on-whales/sub-commands/compose/ - deployer = getDeployer(deployer, deployment_dir, compose_files=cluster_context.compose_files, + deployer = getDeployer(deploy_to, deployment_context, compose_files=cluster_context.compose_files, compose_project_name=cluster_context.cluster, compose_env_file=cluster_context.env_file) return DeployCommandContext(stack, cluster_context, deployer) @@ -102,6 +112,22 @@ def down_operation(ctx, delete_volumes, extra_args_list): ctx.obj.deployer.down(timeout=timeout_arg, volumes=delete_volumes) +def status_operation(ctx): + global_context = ctx.parent.parent.obj + if not global_context.dry_run: + if global_context.verbose: + print("Running compose status") + ctx.obj.deployer.status() + + +def update_operation(ctx): + global_context = ctx.parent.parent.obj + if not global_context.dry_run: + if global_context.verbose: + print("Running compose update") + ctx.obj.deployer.update() + + def ps_operation(ctx): global_context = ctx.parent.parent.obj if not global_context.dry_run: @@ -155,7 +181,7 @@ def exec_operation(ctx, extra_args): if global_context.verbose: print(f"Running compose exec {service_name} {command_to_exec}") try: - ctx.obj.deployer.execute(service_name, command_to_exec, envs=container_exec_env) + ctx.obj.deployer.execute(service_name, command_to_exec, envs=container_exec_env, tty=True) except DeployerException: print("container command returned error exit status") @@ -248,6 +274,22 @@ def _make_runtime_env(ctx): return container_exec_env +def _make_default_cluster_name(deployment, compose_dir, stack, include, exclude): + # Create default unique, stable cluster name from confile file path and stack name if provided + if deployment: + path = os.path.realpath(os.path.abspath(compose_dir)) + else: + path = "internal" + unique_cluster_descriptor = f"{path},{stack},{include},{exclude}" + if opts.o.debug: + print(f"pre-hash descriptor: {unique_cluster_descriptor}") + hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest()[:16] + cluster = f"{constants.cluster_name_prefix}{hash}" + if opts.o.debug: + print(f"Using cluster name: {cluster}") + return cluster + + # stack has to be either PathLike pointing to a stack yml file, or a string with the name of a known stack def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file): @@ -265,16 +307,9 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file): compose_dir = Path(__file__).absolute().parent.parent.joinpath("data", "compose") if cluster is None: - # Create default unique, stable cluster name from confile file path and stack name if provided - # TODO: change this to the config file path - path = os.path.realpath(sys.argv[0]) - unique_cluster_descriptor = f"{path},{stack},{include},{exclude}" - if ctx.debug: - print(f"pre-hash descriptor: {unique_cluster_descriptor}") - hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest() - cluster = f"laconic-{hash}" - if ctx.verbose: - print(f"Using cluster name: {cluster}") + cluster = _make_default_cluster_name(deployment, compose_dir, stack, include, exclude) + else: + _make_default_cluster_name(deployment, compose_dir, stack, include, exclude) # See: https://stackoverflow.com/a/20885799/1701505 from stack_orchestrator import data diff --git a/stack_orchestrator/deploy/deploy_util.py b/stack_orchestrator/deploy/deploy_util.py index 9829490d..8b812d3a 100644 --- a/stack_orchestrator/deploy/deploy_util.py +++ b/stack_orchestrator/deploy/deploy_util.py @@ -14,9 +14,10 @@ # along with this program. If not, see . import os -from typing import List +from typing import List, Any from stack_orchestrator.deploy.deploy_types import DeployCommandContext, VolumeMapping from stack_orchestrator.util import get_parsed_stack_config, get_yaml, get_compose_file_dir, get_pod_list +from stack_orchestrator.opts import opts def _container_image_from_service(stack: str, service: str): @@ -37,6 +38,33 @@ def _container_image_from_service(stack: str, service: str): return image_name +def parsed_pod_files_map_from_file_names(pod_files): + parsed_pod_yaml_map : Any = {} + for pod_file in pod_files: + with open(pod_file, "r") as pod_file_descriptor: + parsed_pod_file = get_yaml().load(pod_file_descriptor) + parsed_pod_yaml_map[pod_file] = parsed_pod_file + if opts.o.debug: + print(f"parsed_pod_yaml_map: {parsed_pod_yaml_map}") + return parsed_pod_yaml_map + + +def images_for_deployment(pod_files: List[str]): + image_set = set() + parsed_pod_yaml_map = parsed_pod_files_map_from_file_names(pod_files) + # Find the set of images in the pods + for pod_name in parsed_pod_yaml_map: + pod = parsed_pod_yaml_map[pod_name] + services = pod["services"] + for service_name in services: + service_info = services[service_name] + image = service_info["image"] + image_set.add(image) + if opts.o.debug: + print(f"image_set: {image_set}") + return image_set + + def _volumes_to_docker(mounts: List[VolumeMapping]): # Example from doc: [("/", "/host"), ("/etc/hosts", "/etc/hosts", "rw")] result = [] diff --git a/stack_orchestrator/deploy/deployer.py b/stack_orchestrator/deploy/deployer.py index 68b0088a..2df784a2 100644 --- a/stack_orchestrator/deploy/deployer.py +++ b/stack_orchestrator/deploy/deployer.py @@ -27,10 +27,18 @@ class Deployer(ABC): def down(self, timeout, volumes): pass + @abstractmethod + def update(self): + pass + @abstractmethod def ps(self): pass + @abstractmethod + def status(self): + pass + @abstractmethod def port(self, service, private_port): pass @@ -44,7 +52,7 @@ class Deployer(ABC): pass @abstractmethod - def run(self, image, command, user, volumes, entrypoint): + def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, ports=[], detach=False): pass diff --git a/stack_orchestrator/deploy/deployer_factory.py b/stack_orchestrator/deploy/deployer_factory.py index 5d515418..959c1b7a 100644 --- a/stack_orchestrator/deploy/deployer_factory.py +++ b/stack_orchestrator/deploy/deployer_factory.py @@ -13,23 +13,24 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from stack_orchestrator import constants from stack_orchestrator.deploy.k8s.deploy_k8s import K8sDeployer, K8sDeployerConfigGenerator from stack_orchestrator.deploy.compose.deploy_docker import DockerDeployer, DockerDeployerConfigGenerator def getDeployerConfigGenerator(type: str): if type == "compose" or type is None: - return DockerDeployerConfigGenerator() - elif type == "k8s": - return K8sDeployerConfigGenerator() + return DockerDeployerConfigGenerator(type) + elif type == constants.k8s_deploy_type or type == constants.k8s_kind_deploy_type: + return K8sDeployerConfigGenerator(type) else: print(f"ERROR: deploy-to {type} is not valid") -def getDeployer(type: str, deployment_dir, compose_files, compose_project_name, compose_env_file): +def getDeployer(type: str, deployment_context, compose_files, compose_project_name, compose_env_file): if type == "compose" or type is None: - return DockerDeployer(deployment_dir, compose_files, compose_project_name, compose_env_file) - elif type == "k8s": - return K8sDeployer(deployment_dir, compose_files, compose_project_name, compose_env_file) + return DockerDeployer(type, deployment_context, compose_files, compose_project_name, compose_env_file) + elif type == type == constants.k8s_deploy_type or type == constants.k8s_kind_deploy_type: + return K8sDeployer(type, deployment_context, compose_files, compose_project_name, compose_env_file) else: print(f"ERROR: deploy-to {type} is not valid") diff --git a/stack_orchestrator/deploy/deployment.py b/stack_orchestrator/deploy/deployment.py index e22d7dcc..f364121f 100644 --- a/stack_orchestrator/deploy/deployment.py +++ b/stack_orchestrator/deploy/deployment.py @@ -16,8 +16,11 @@ import click from pathlib import Path import sys -from stack_orchestrator.deploy.deploy import up_operation, down_operation, ps_operation, port_operation -from stack_orchestrator.deploy.deploy import exec_operation, logs_operation, create_deploy_context +from stack_orchestrator import constants +from stack_orchestrator.deploy.images import push_images_operation +from stack_orchestrator.deploy.deploy import up_operation, down_operation, ps_operation, port_operation, status_operation +from stack_orchestrator.deploy.deploy import exec_operation, logs_operation, create_deploy_context, update_operation +from stack_orchestrator.deploy.deploy_types import DeployCommandContext from stack_orchestrator.deploy.deployment_context import DeploymentContext @@ -45,13 +48,17 @@ def command(ctx, dir): ctx.obj = deployment_context -def make_deploy_context(ctx): +def make_deploy_context(ctx) -> DeployCommandContext: context: DeploymentContext = ctx.obj stack_file_path = context.get_stack_file() env_file = context.get_env_file() - cluster_name = context.get_cluster_name() + cluster_name = context.get_cluster_id() + if constants.deploy_to_key in context.spec.obj: + deployment_type = context.spec.obj[constants.deploy_to_key] + else: + deployment_type = constants.compose_deploy_type return create_deploy_context(ctx.parent.parent.obj, context, stack_file_path, None, None, cluster_name, env_file, - context.spec.obj["deploy-to"]) + deployment_type) @command.command() @@ -104,6 +111,14 @@ def ps(ctx): ps_operation(ctx) +@command.command() +@click.pass_context +def push_images(ctx): + deploy_command_context: DeployCommandContext = make_deploy_context(ctx) + deployment_context: DeploymentContext = ctx.obj + push_images_operation(deploy_command_context, deployment_context) + + @command.command() @click.argument('extra_args', nargs=-1) # help: command: port @click.pass_context @@ -132,4 +147,12 @@ def logs(ctx, tail, follow, extra_args): @command.command() @click.pass_context def status(ctx): - print(f"Context: {ctx.parent.obj}") + ctx.obj = make_deploy_context(ctx) + status_operation(ctx) + + +@command.command() +@click.pass_context +def update(ctx): + ctx.obj = make_deploy_context(ctx) + update_operation(ctx) diff --git a/stack_orchestrator/deploy/deployment_context.py b/stack_orchestrator/deploy/deployment_context.py index cd731394..27e32812 100644 --- a/stack_orchestrator/deploy/deployment_context.py +++ b/stack_orchestrator/deploy/deployment_context.py @@ -14,29 +14,39 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import hashlib +import os from pathlib import Path +from stack_orchestrator import constants +from stack_orchestrator.util import get_yaml from stack_orchestrator.deploy.stack import Stack from stack_orchestrator.deploy.spec import Spec class DeploymentContext: deployment_dir: Path + id: str spec: Spec stack: Stack def get_stack_file(self): - return self.deployment_dir.joinpath("stack.yml") + return self.deployment_dir.joinpath(constants.stack_file_name) def get_spec_file(self): - return self.deployment_dir.joinpath("spec.yml") + return self.deployment_dir.joinpath(constants.spec_file_name) def get_env_file(self): - return self.deployment_dir.joinpath("config.env") + return self.deployment_dir.joinpath(constants.config_file_name) - # TODO: implement me - def get_cluster_name(self): - return None + def get_deployment_file(self): + return self.deployment_dir.joinpath(constants.deployment_file_name) + + def get_compose_dir(self): + return self.deployment_dir.joinpath(constants.compose_dir_name) + + def get_cluster_id(self): + return self.id def init(self, dir): self.deployment_dir = dir @@ -44,3 +54,16 @@ class DeploymentContext: self.spec.init_from_file(self.get_spec_file()) self.stack = Stack(self.spec.obj["stack"]) self.stack.init_from_file(self.get_stack_file()) + deployment_file_path = self.get_deployment_file() + if deployment_file_path.exists(): + with deployment_file_path: + obj = get_yaml().load(open(deployment_file_path, "r")) + self.id = obj[constants.cluster_id_key] + # Handle the case of a legacy deployment with no file + # Code below is intended to match the output from _make_default_cluster_name() + # TODO: remove when we no longer need to support legacy deployments + else: + path = os.path.realpath(os.path.abspath(self.get_compose_dir())) + unique_cluster_descriptor = f"{path},{self.get_stack_file()},None,None" + hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest()[:16] + self.id = f"{constants.cluster_name_prefix}{hash}" diff --git a/stack_orchestrator/deploy/deployment_create.py b/stack_orchestrator/deploy/deployment_create.py index c00c0dc6..138fc1bb 100644 --- a/stack_orchestrator/deploy/deployment_create.py +++ b/stack_orchestrator/deploy/deployment_create.py @@ -20,17 +20,20 @@ from pathlib import Path from typing import List import random from shutil import copy, copyfile, copytree +from secrets import token_hex import sys +from stack_orchestrator import constants +from stack_orchestrator.opts import opts from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config, global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts, - get_pod_script_paths, get_plugin_code_paths) + get_pod_script_paths, get_plugin_code_paths, error_exit, env_var_map_from_file) from stack_orchestrator.deploy.deploy_types import LaconicStackSetupCommand from stack_orchestrator.deploy.deployer_factory import getDeployerConfigGenerator from stack_orchestrator.deploy.deployment_context import DeploymentContext def _make_default_deployment_dir(): - return "deployment-001" + return Path("deployment-001") def _get_ports(stack): @@ -102,8 +105,8 @@ def _fixup_pod_file(pod, spec, compose_dir): } pod["volumes"][volume] = new_volume_spec # Fix up ports - if "ports" in spec: - spec_ports = spec["ports"] + if "network" in spec and "ports" in spec["network"]: + spec_ports = spec["network"]["ports"] for container_name, container_ports in spec_ports.items(): if container_name in pod["services"]: pod["services"][container_name]["ports"] = container_ports @@ -242,37 +245,76 @@ def _parse_config_variables(variable_values: str): variable_name = variable_value_pair[0] variable_value = variable_value_pair[1] result_values[variable_name] = variable_value - result = {"config": result_values} + result = result_values return result @click.command() @click.option("--config", help="Provide config variables for the deployment") +@click.option("--config-file", help="Provide config variables in a file for the deployment") +@click.option("--kube-config", help="Provide a config file for a k8s deployment") +@click.option("--image-registry", help="Provide a container image registry url for this k8s cluster") @click.option("--output", required=True, help="Write yaml spec file here") @click.option("--map-ports-to-host", required=False, help="Map ports to the host as one of: any-variable-random (default), " "localhost-same, any-same, localhost-fixed-random, any-fixed-random") @click.pass_context -def init(ctx, config, output, map_ports_to_host): - yaml = get_yaml() +def init(ctx, config, config_file, kube_config, image_registry, output, map_ports_to_host): stack = global_options(ctx).stack - debug = global_options(ctx).debug - default_spec_file_content = call_stack_deploy_init(ctx.obj) - spec_file_content = {"stack": stack, "deploy-to": ctx.obj.deployer.name} + deployer_type = ctx.obj.deployer.type + deploy_command_context = ctx.obj + return init_operation( + deploy_command_context, + stack, deployer_type, + config, config_file, + kube_config, + image_registry, + output, + map_ports_to_host) + + +# The init command's implementation is in a separate function so that we can +# call it from other commands, bypassing the click decoration stuff +def init_operation(deploy_command_context, stack, deployer_type, config, + config_file, kube_config, image_registry, output, map_ports_to_host): + + default_spec_file_content = call_stack_deploy_init(deploy_command_context) + spec_file_content = {"stack": stack, constants.deploy_to_key: deployer_type} + if deployer_type == "k8s": + if kube_config is None: + error_exit("--kube-config must be supplied with --deploy-to k8s") + if image_registry is None: + error_exit("--image-registry must be supplied with --deploy-to k8s") + spec_file_content.update({constants.kube_config_key: kube_config}) + spec_file_content.update({constants.image_resigtry_key: image_registry}) + else: + # Check for --kube-config supplied for non-relevant deployer types + if kube_config is not None: + error_exit(f"--kube-config is not allowed with a {deployer_type} deployment") + if image_registry is not None: + error_exit(f"--image-registry is not allowed with a {deployer_type} deployment") if default_spec_file_content: spec_file_content.update(default_spec_file_content) config_variables = _parse_config_variables(config) + # Implement merge, since update() overwrites if config_variables: - # Implement merge, since update() overwrites orig_config = spec_file_content.get("config", {}) - new_config = config_variables["config"] + new_config = config_variables merged_config = {**new_config, **orig_config} spec_file_content.update({"config": merged_config}) - if debug: - print(f"Creating spec file for stack: {stack} with content: {spec_file_content}") + if config_file: + config_file_path = Path(config_file) + if not config_file_path.exists(): + error_exit(f"config file: {config_file} does not exist") + config_file_variables = env_var_map_from_file(config_file_path) + if config_file_variables: + orig_config = spec_file_content.get("config", {}) + new_config = config_file_variables + merged_config = {**new_config, **orig_config} + spec_file_content.update({"config": merged_config}) ports = _get_mapped_ports(stack, map_ports_to_host) - spec_file_content["ports"] = ports + spec_file_content.update({"network": {"ports": ports}}) named_volumes = _get_named_volumes(stack) if named_volumes: @@ -281,8 +323,11 @@ def init(ctx, config, output, map_ports_to_host): volume_descriptors[named_volume] = f"./data/{named_volume}" spec_file_content["volumes"] = volume_descriptors + if opts.o.debug: + print(f"Creating spec file for stack: {stack} with content: {spec_file_content}") + with open(output, "w") as output_file: - yaml.dump(spec_file_content, output_file) + get_yaml().dump(spec_file_content, output_file) def _write_config_file(spec_file: Path, config_env_file: Path): @@ -296,12 +341,25 @@ def _write_config_file(spec_file: Path, config_env_file: Path): output_file.write(f"{variable_name}={variable_value}\n") +def _write_kube_config_file(external_path: Path, internal_path: Path): + if not external_path.exists(): + error_exit(f"Kube config file {external_path} does not exist") + copyfile(external_path, internal_path) + + def _copy_files_to_directory(file_paths: List[Path], directory: Path): for path in file_paths: # Using copy to preserve the execute bit copy(path, os.path.join(directory, os.path.basename(path))) +def _create_deployment_file(deployment_dir: Path): + deployment_file_path = deployment_dir.joinpath(constants.deployment_file_name) + cluster = f"{constants.cluster_name_prefix}{token_hex(8)}" + with open(deployment_file_path, "w") as output_file: + output_file.write(f"{constants.cluster_id_key}: {cluster}\n") + + @click.command() @click.option("--spec-file", required=True, help="Spec file to use to create this deployment") @click.option("--deployment-dir", help="Create deployment files in this directory") @@ -310,29 +368,42 @@ def _copy_files_to_directory(file_paths: List[Path], directory: Path): @click.option("--initial-peers", help="Initial set of persistent peers") @click.pass_context def create(ctx, spec_file, deployment_dir, network_dir, initial_peers): - # This function fails with a useful error message if the file doens't exist + deployment_command_context = ctx.obj + return create_operation(deployment_command_context, spec_file, deployment_dir, network_dir, initial_peers) + + +# The init command's implementation is in a separate function so that we can +# call it from other commands, bypassing the click decoration stuff +def create_operation(deployment_command_context, spec_file, deployment_dir, network_dir, initial_peers): parsed_spec = get_parsed_deployment_spec(spec_file) stack_name = parsed_spec["stack"] + deployment_type = parsed_spec[constants.deploy_to_key] stack_file = get_stack_file_path(stack_name) parsed_stack = get_parsed_stack_config(stack_name) - if global_options(ctx).debug: + if opts.o.debug: print(f"parsed spec: {parsed_spec}") if deployment_dir is None: - deployment_dir = _make_default_deployment_dir() - if os.path.exists(deployment_dir): - print(f"Error: {deployment_dir} already exists") - sys.exit(1) - os.mkdir(deployment_dir) + deployment_dir_path = _make_default_deployment_dir() + else: + deployment_dir_path = Path(deployment_dir) + if deployment_dir_path.exists(): + error_exit(f"{deployment_dir_path} already exists") + os.mkdir(deployment_dir_path) # Copy spec file and the stack file into the deployment dir - copyfile(spec_file, os.path.join(deployment_dir, "spec.yml")) - copyfile(stack_file, os.path.join(deployment_dir, os.path.basename(stack_file))) + copyfile(spec_file, deployment_dir_path.joinpath(constants.spec_file_name)) + copyfile(stack_file, deployment_dir_path.joinpath(os.path.basename(stack_file))) + _create_deployment_file(deployment_dir_path) # Copy any config varibles from the spec file into an env file suitable for compose - _write_config_file(spec_file, os.path.join(deployment_dir, "config.env")) + _write_config_file(spec_file, deployment_dir_path.joinpath(constants.config_file_name)) + # Copy any k8s config file into the deployment dir + if deployment_type == "k8s": + _write_kube_config_file(Path(parsed_spec[constants.kube_config_key]), + deployment_dir_path.joinpath(constants.kube_config_filename)) # Copy the pod files into the deployment dir, fixing up content pods = get_pod_list(parsed_stack) - destination_compose_dir = os.path.join(deployment_dir, "compose") + destination_compose_dir = deployment_dir_path.joinpath("compose") os.mkdir(destination_compose_dir) - destination_pods_dir = os.path.join(deployment_dir, "pods") + destination_pods_dir = deployment_dir_path.joinpath("pods") os.mkdir(destination_pods_dir) data_dir = Path(__file__).absolute().parent.parent.joinpath("data") yaml = get_yaml() @@ -340,12 +411,12 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers): pod_file_path = get_pod_file_path(parsed_stack, pod) parsed_pod_file = yaml.load(open(pod_file_path, "r")) extra_config_dirs = _find_extra_config_dirs(parsed_pod_file, pod) - destination_pod_dir = os.path.join(destination_pods_dir, pod) + destination_pod_dir = destination_pods_dir.joinpath(pod) os.mkdir(destination_pod_dir) - if global_options(ctx).debug: + if opts.o.debug: print(f"extra config dirs: {extra_config_dirs}") _fixup_pod_file(parsed_pod_file, parsed_spec, destination_compose_dir) - with open(os.path.join(destination_compose_dir, "docker-compose-%s.yml" % pod), "w") as output_file: + with open(destination_compose_dir.joinpath("docker-compose-%s.yml" % pod), "w") as output_file: yaml.dump(parsed_pod_file, output_file) # Copy the config files for the pod, if any config_dirs = {pod} @@ -353,28 +424,27 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers): for config_dir in config_dirs: source_config_dir = data_dir.joinpath("config", config_dir) if os.path.exists(source_config_dir): - destination_config_dir = os.path.join(deployment_dir, "config", config_dir) + destination_config_dir = deployment_dir_path.joinpath("config", config_dir) # If the same config dir appears in multiple pods, it may already have been copied if not os.path.exists(destination_config_dir): copytree(source_config_dir, destination_config_dir) # Copy the script files for the pod, if any if pod_has_scripts(parsed_stack, pod): - destination_script_dir = os.path.join(destination_pod_dir, "scripts") + destination_script_dir = destination_pod_dir.joinpath("scripts") os.mkdir(destination_script_dir) script_paths = get_pod_script_paths(parsed_stack, pod) _copy_files_to_directory(script_paths, destination_script_dir) # Delegate to the stack's Python code # The deploy create command doesn't require a --stack argument so we need to insert the # stack member here. - deployment_command_context = ctx.obj deployment_command_context.stack = stack_name deployment_context = DeploymentContext() - deployment_context.init(Path(deployment_dir)) + deployment_context.init(deployment_dir_path) # Call the deployer to generate any deployer-specific files (e.g. for kind) - deployer_config_generator = getDeployerConfigGenerator(parsed_spec["deploy-to"]) - # TODO: make deployment_dir a Path above - deployer_config_generator.generate(Path(deployment_dir)) - call_stack_deploy_create(deployment_context, [network_dir, initial_peers]) + deployer_config_generator = getDeployerConfigGenerator(deployment_type) + # TODO: make deployment_dir_path a Path above + deployer_config_generator.generate(deployment_dir_path) + call_stack_deploy_create(deployment_context, [network_dir, initial_peers, deployment_command_context]) # TODO: this code should be in the stack .py files but diff --git a/stack_orchestrator/deploy/images.py b/stack_orchestrator/deploy/images.py new file mode 100644 index 00000000..ddbb33f7 --- /dev/null +++ b/stack_orchestrator/deploy/images.py @@ -0,0 +1,62 @@ +# Copyright © 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from typing import Set + +from python_on_whales import DockerClient + +from stack_orchestrator import constants +from stack_orchestrator.opts import opts +from stack_orchestrator.deploy.deployment_context import DeploymentContext +from stack_orchestrator.deploy.deploy_types import DeployCommandContext +from stack_orchestrator.deploy.deploy_util import images_for_deployment + + +def _image_needs_pushed(image: str): + # TODO: this needs to be more intelligent + return image.endswith(":local") + + +def remote_tag_for_image(image: str, remote_repo_url: str): + # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy + (org, image_name_with_version) = image.split("/") + (image_name, image_version) = image_name_with_version.split(":") + if image_version == "local": + return f"{remote_repo_url}/{image_name}:deploy" + else: + return image + + +# TODO: needs lots of error handling +def push_images_operation(command_context: DeployCommandContext, deployment_context: DeploymentContext): + # Get the list of images for the stack + cluster_context = command_context.cluster_context + images: Set[str] = images_for_deployment(cluster_context.compose_files) + # Tag the images for the remote repo + remote_repo_url = deployment_context.spec.obj[constants.image_resigtry_key] + docker = DockerClient() + for image in images: + if _image_needs_pushed(image): + remote_tag = remote_tag_for_image(image, remote_repo_url) + if opts.o.verbose: + print(f"Tagging {image} to {remote_tag}") + docker.image.tag(image, remote_tag) + # Run docker push commands to upload + for image in images: + if _image_needs_pushed(image): + remote_tag = remote_tag_for_image(image, remote_repo_url) + if opts.o.verbose: + print(f"Pushing image {remote_tag}") + docker.image.push(remote_tag) diff --git a/stack_orchestrator/deploy/k8s/cluster_info.py b/stack_orchestrator/deploy/k8s/cluster_info.py index 9275db2b..85fd63a8 100644 --- a/stack_orchestrator/deploy/k8s/cluster_info.py +++ b/stack_orchestrator/deploy/k8s/cluster_info.py @@ -17,37 +17,116 @@ from kubernetes import client from typing import Any, List, Set from stack_orchestrator.opts import opts +from stack_orchestrator.util import env_var_map_from_file from stack_orchestrator.deploy.k8s.helpers import named_volumes_from_pod_files, volume_mounts_for_service, volumes_for_pod_files -from stack_orchestrator.deploy.k8s.helpers import parsed_pod_files_map_from_file_names, get_node_pv_mount_path -from stack_orchestrator.deploy.k8s.helpers import env_var_map_from_file, envs_from_environment_variables_map +from stack_orchestrator.deploy.k8s.helpers import get_node_pv_mount_path +from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variables_map +from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment from stack_orchestrator.deploy.deploy_types import DeployEnvVars +from stack_orchestrator.deploy.spec import Spec +from stack_orchestrator.deploy.images import remote_tag_for_image class ClusterInfo: - parsed_pod_yaml_map: Any = {} + parsed_pod_yaml_map: Any image_set: Set[str] = set() - app_name: str = "test-app" - deployment_name: str = "test-deployment" + app_name: str environment_variables: DeployEnvVars + spec: Spec def __init__(self) -> None: pass - def int(self, pod_files: List[str], compose_env_file): + def int(self, pod_files: List[str], compose_env_file, deployment_name, spec: Spec): self.parsed_pod_yaml_map = parsed_pod_files_map_from_file_names(pod_files) # Find the set of images in the pods + self.image_set = images_for_deployment(pod_files) + self.environment_variables = DeployEnvVars(env_var_map_from_file(compose_env_file)) + self.app_name = deployment_name + self.spec = spec + if (opts.o.debug): + print(f"Env vars: {self.environment_variables.map}") + + def get_ingress(self): + # No ingress for a deployment that has no http-proxy defined, for now + http_proxy_info_list = self.spec.get_http_proxy() + ingress = None + if http_proxy_info_list: + # TODO: handle multiple definitions + http_proxy_info = http_proxy_info_list[0] + if opts.o.debug: + print(f"http-proxy: {http_proxy_info}") + # TODO: good enough parsing for webapp deployment for now + host_name = http_proxy_info["host-name"] + rules = [] + tls = [client.V1IngressTLS( + hosts=[host_name], + secret_name=f"{self.app_name}-tls" + )] + paths = [] + for route in http_proxy_info["routes"]: + path = route["path"] + proxy_to = route["proxy-to"] + if opts.o.debug: + print(f"proxy config: {path} -> {proxy_to}") + # proxy_to has the form : + proxy_to_port = int(proxy_to.split(":")[1]) + paths.append(client.V1HTTPIngressPath( + path_type="Prefix", + path=path, + backend=client.V1IngressBackend( + service=client.V1IngressServiceBackend( + # TODO: this looks wrong + name=f"{self.app_name}-service", + # TODO: pull port number from the service + port=client.V1ServiceBackendPort(number=proxy_to_port) + ) + ) + )) + rules.append(client.V1IngressRule( + host=host_name, + http=client.V1HTTPIngressRuleValue( + paths=paths + ) + )) + spec = client.V1IngressSpec( + tls=tls, + rules=rules + ) + ingress = client.V1Ingress( + metadata=client.V1ObjectMeta( + name=f"{self.app_name}-ingress", + annotations={ + "kubernetes.io/ingress.class": "nginx", + "cert-manager.io/cluster-issuer": "letsencrypt-prod" + } + ), + spec=spec + ) + return ingress + + # TODO: suppoprt multiple services + def get_service(self): for pod_name in self.parsed_pod_yaml_map: pod = self.parsed_pod_yaml_map[pod_name] services = pod["services"] for service_name in services: service_info = services[service_name] - image = service_info["image"] - self.image_set.add(image) - if opts.o.debug: - print(f"image_set: {self.image_set}") - self.environment_variables = DeployEnvVars(env_var_map_from_file(compose_env_file)) - if (opts.o.debug): - print(f"Env vars: {self.environment_variables.map}") + port = int(service_info["ports"][0]) + if opts.o.debug: + print(f"service port: {port}") + service = client.V1Service( + metadata=client.V1ObjectMeta(name=f"{self.app_name}-service"), + spec=client.V1ServiceSpec( + type="ClusterIP", + ports=[client.V1ServicePort( + port=port, + target_port=port + )], + selector={"app": self.app_name} + ) + ) + return service def get_pvcs(self): result = [] @@ -89,8 +168,8 @@ class ClusterInfo: result.append(pv) return result - # to suit the deployment, and also annotate the container specs to point at said volumes - def get_deployment(self): + # TODO: put things like image pull policy into an object-scope struct + def get_deployment(self, image_pull_policy: str = None): containers = [] for pod_name in self.parsed_pod_yaml_map: pod = self.parsed_pod_yaml_map[pod_name] @@ -99,12 +178,20 @@ class ClusterInfo: container_name = service_name service_info = services[service_name] image = service_info["image"] + port = int(service_info["ports"][0]) + if opts.o.debug: + print(f"image: {image}") + print(f"service port: {port}") + # Re-write the image tag for remote deployment + image_to_use = remote_tag_for_image( + image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image volume_mounts = volume_mounts_for_service(self.parsed_pod_yaml_map, service_name) container = client.V1Container( name=container_name, - image=image, + image=image_to_use, + image_pull_policy=image_pull_policy, env=envs_from_environment_variables_map(self.environment_variables.map), - ports=[client.V1ContainerPort(container_port=80)], + ports=[client.V1ContainerPort(container_port=port)], volume_mounts=volume_mounts, resources=client.V1ResourceRequirements( requests={"cpu": "100m", "memory": "200Mi"}, @@ -113,9 +200,10 @@ class ClusterInfo: ) containers.append(container) volumes = volumes_for_pod_files(self.parsed_pod_yaml_map) + image_pull_secrets = [client.V1LocalObjectReference(name="laconic-registry")] template = client.V1PodTemplateSpec( metadata=client.V1ObjectMeta(labels={"app": self.app_name}), - spec=client.V1PodSpec(containers=containers, volumes=volumes), + spec=client.V1PodSpec(containers=containers, image_pull_secrets=image_pull_secrets, volumes=volumes), ) spec = client.V1DeploymentSpec( replicas=1, template=template, selector={ @@ -125,7 +213,7 @@ class ClusterInfo: deployment = client.V1Deployment( api_version="apps/v1", kind="Deployment", - metadata=client.V1ObjectMeta(name=self.deployment_name), + metadata=client.V1ObjectMeta(name=f"{self.app_name}-deployment"), spec=spec, ) return deployment diff --git a/stack_orchestrator/deploy/k8s/deploy_k8s.py b/stack_orchestrator/deploy/k8s/deploy_k8s.py index bc256b6b..0a339fe9 100644 --- a/stack_orchestrator/deploy/k8s/deploy_k8s.py +++ b/stack_orchestrator/deploy/k8s/deploy_k8s.py @@ -13,47 +13,83 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from datetime import datetime, timezone + from pathlib import Path from kubernetes import client, config +from stack_orchestrator import constants from stack_orchestrator.deploy.deployer import Deployer, DeployerConfigGenerator from stack_orchestrator.deploy.k8s.helpers import create_cluster, destroy_cluster, load_images_into_kind from stack_orchestrator.deploy.k8s.helpers import pods_in_deployment, log_stream_from_string, generate_kind_config from stack_orchestrator.deploy.k8s.cluster_info import ClusterInfo from stack_orchestrator.opts import opts +from stack_orchestrator.deploy.deployment_context import DeploymentContext +from stack_orchestrator.util import error_exit + + +class AttrDict(dict): + def __init__(self, *args, **kwargs): + super(AttrDict, self).__init__(*args, **kwargs) + self.__dict__ = self + + +def _check_delete_exception(e: client.exceptions.ApiException): + if e.status == 404: + if opts.o.debug: + print("Failed to delete object, continuing") + else: + error_exit(f"k8s api error: {e}") class K8sDeployer(Deployer): name: str = "k8s" + type: str core_api: client.CoreV1Api apps_api: client.AppsV1Api + networking_api: client.NetworkingV1Api k8s_namespace: str = "default" kind_cluster_name: str - cluster_info : ClusterInfo + cluster_info: ClusterInfo deployment_dir: Path + deployment_context: DeploymentContext - def __init__(self, deployment_dir, compose_files, compose_project_name, compose_env_file) -> None: + def __init__(self, type, deployment_context: DeploymentContext, compose_files, compose_project_name, compose_env_file) -> None: + self.type = type + # TODO: workaround pending refactoring above to cope with being created with a null deployment_context + if deployment_context is None: + return + self.deployment_dir = deployment_context.deployment_dir + self.deployment_context = deployment_context + self.kind_cluster_name = compose_project_name + self.cluster_info = ClusterInfo() + self.cluster_info.int(compose_files, compose_env_file, compose_project_name, deployment_context.spec) if (opts.o.debug): - print(f"Deployment dir: {deployment_dir}") + print(f"Deployment dir: {deployment_context.deployment_dir}") print(f"Compose files: {compose_files}") print(f"Project name: {compose_project_name}") print(f"Env file: {compose_env_file}") - self.deployment_dir = deployment_dir - self.kind_cluster_name = compose_project_name - self.cluster_info = ClusterInfo() - self.cluster_info.int(compose_files, compose_env_file) + print(f"Type: {type}") def connect_api(self): - config.load_kube_config(context=f"kind-{self.kind_cluster_name}") + if self.is_kind(): + config.load_kube_config(context=f"kind-{self.kind_cluster_name}") + else: + # Get the config file and pass to load_kube_config() + config.load_kube_config(config_file=self.deployment_dir.joinpath(constants.kube_config_filename).as_posix()) self.core_api = client.CoreV1Api() + self.networking_api = client.NetworkingV1Api() self.apps_api = client.AppsV1Api() + self.custom_obj_api = client.CustomObjectsApi() def up(self, detach, services): - # Create the kind cluster - create_cluster(self.kind_cluster_name, self.deployment_dir.joinpath("kind-config.yml")) + + if self.is_kind(): + # Create the kind cluster + create_cluster(self.kind_cluster_name, self.deployment_dir.joinpath(constants.kind_config_filename)) + # Ensure the referenced containers are copied into kind + load_images_into_kind(self.kind_cluster_name, self.cluster_info.image_set) self.connect_api() - # Ensure the referenced containers are copied into kind - load_images_into_kind(self.kind_cluster_name, self.cluster_info.image_set) # Create the host-path-mounted PVs for this deployment pvs = self.cluster_info.get_pvs() @@ -75,7 +111,7 @@ class K8sDeployer(Deployer): print("PVCs created:") print(f"{pvc_resp}") # Process compose files into a Deployment - deployment = self.cluster_info.get_deployment() + deployment = self.cluster_info.get_deployment(image_pull_policy=None if self.is_kind() else "Always") # Create the k8s objects if opts.o.debug: print(f"Sending this deployment: {deployment}") @@ -87,20 +123,170 @@ class K8sDeployer(Deployer): print(f"{deployment_resp.metadata.namespace} {deployment_resp.metadata.name} \ {deployment_resp.metadata.generation} {deployment_resp.spec.template.spec.containers[0].image}") + service: client.V1Service = self.cluster_info.get_service() + service_resp = self.core_api.create_namespaced_service( + namespace=self.k8s_namespace, + body=service + ) + if opts.o.debug: + print("Service created:") + print(f"{service_resp}") + + if not self.is_kind(): + ingress: client.V1Ingress = self.cluster_info.get_ingress() + + if opts.o.debug: + print(f"Sending this ingress: {ingress}") + ingress_resp = self.networking_api.create_namespaced_ingress( + namespace=self.k8s_namespace, + body=ingress + ) + if opts.o.debug: + print("Ingress created:") + print(f"{ingress_resp}") + def down(self, timeout, volumes): + self.connect_api() # Delete the k8s objects - # Destroy the kind cluster - destroy_cluster(self.kind_cluster_name) + # Create the host-path-mounted PVs for this deployment + pvs = self.cluster_info.get_pvs() + for pv in pvs: + if opts.o.debug: + print(f"Deleting this pv: {pv}") + try: + pv_resp = self.core_api.delete_persistent_volume(name=pv.metadata.name) + if opts.o.debug: + print("PV deleted:") + print(f"{pv_resp}") + except client.exceptions.ApiException as e: + _check_delete_exception(e) + + # Figure out the PVCs for this deployment + pvcs = self.cluster_info.get_pvcs() + for pvc in pvcs: + if opts.o.debug: + print(f"Deleting this pvc: {pvc}") + try: + pvc_resp = self.core_api.delete_namespaced_persistent_volume_claim( + name=pvc.metadata.name, namespace=self.k8s_namespace + ) + if opts.o.debug: + print("PVCs deleted:") + print(f"{pvc_resp}") + except client.exceptions.ApiException as e: + _check_delete_exception(e) + deployment = self.cluster_info.get_deployment() + if opts.o.debug: + print(f"Deleting this deployment: {deployment}") + try: + self.apps_api.delete_namespaced_deployment( + name=deployment.metadata.name, namespace=self.k8s_namespace + ) + except client.exceptions.ApiException as e: + _check_delete_exception(e) + + service: client.V1Service = self.cluster_info.get_service() + if opts.o.debug: + print(f"Deleting service: {service}") + try: + self.core_api.delete_namespaced_service( + namespace=self.k8s_namespace, + name=service.metadata.name + ) + except client.exceptions.ApiException as e: + _check_delete_exception(e) + + if not self.is_kind(): + ingress: client.V1Ingress = self.cluster_info.get_ingress() + if opts.o.debug: + print(f"Deleting this ingress: {ingress}") + try: + self.networking_api.delete_namespaced_ingress( + name=ingress.metadata.name, namespace=self.k8s_namespace + ) + except client.exceptions.ApiException as e: + _check_delete_exception(e) + + if self.is_kind(): + # Destroy the kind cluster + destroy_cluster(self.kind_cluster_name) + + def status(self): + self.connect_api() + # Call whatever API we need to get the running container list + all_pods = self.core_api.list_pod_for_all_namespaces(watch=False) + pods = [] + + if all_pods.items: + for p in all_pods.items: + if f"{self.cluster_info.app_name}-deployment" in p.metadata.name: + pods.append(p) + + if not pods: + return + + hostname = "?" + ip = "?" + tls = "?" + try: + ingress = self.networking_api.read_namespaced_ingress(namespace=self.k8s_namespace, + name=self.cluster_info.get_ingress().metadata.name) + + cert = self.custom_obj_api.get_namespaced_custom_object( + group="cert-manager.io", + version="v1", + namespace=self.k8s_namespace, + plural="certificates", + name=ingress.spec.tls[0].secret_name + ) + + hostname = ingress.spec.tls[0].hosts[0] + ip = ingress.status.load_balancer.ingress[0].ip + tls = "notBefore: %s, notAfter: %s" % (cert["status"]["notBefore"], cert["status"]["notAfter"]) + except: # noqa: E722 + pass + + print("Ingress:") + print("\tHostname:", hostname) + print("\tIP:", ip) + print("\tTLS:", tls) + print("") + print("Pods:") + + for p in pods: + if p.metadata.deletion_timestamp: + print(f"\t{p.metadata.namespace}/{p.metadata.name}: Terminating ({p.metadata.deletion_timestamp})") + else: + print(f"\t{p.metadata.namespace}/{p.metadata.name}: Running ({p.metadata.creation_timestamp})") def ps(self): self.connect_api() - # Call whatever API we need to get the running container list - ret = self.core_api.list_pod_for_all_namespaces(watch=False) - if ret.items: - for i in ret.items: - print("%s\t%s\t%s" % (i.status.pod_ip, i.metadata.namespace, i.metadata.name)) - ret = self.core_api.list_node(pretty=True, watch=False) - return [] + pods = self.core_api.list_pod_for_all_namespaces(watch=False) + + ret = [] + + for p in pods.items: + if f"{self.cluster_info.app_name}-deployment" in p.metadata.name: + pod_ip = p.status.pod_ip + ports = AttrDict() + for c in p.spec.containers: + if c.ports: + for prt in c.ports: + ports[str(prt.container_port)] = [AttrDict({ + "HostIp": pod_ip, + "HostPort": prt.container_port + })] + + ret.append(AttrDict({ + "id": f"{p.metadata.namespace}/{p.metadata.name}", + "name": p.metadata.name, + "namespace": p.metadata.namespace, + "network_settings": AttrDict({ + "ports": ports + }) + })) + + return ret def port(self, service, private_port): # Since we handle the port mapping, need to figure out where this comes from @@ -113,31 +299,73 @@ class K8sDeployer(Deployer): def logs(self, services, tail, follow, stream): self.connect_api() - pods = pods_in_deployment(self.core_api, "test-deployment") + pods = pods_in_deployment(self.core_api, self.cluster_info.app_name) if len(pods) > 1: print("Warning: more than one pod in the deployment") - k8s_pod_name = pods[0] - log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test") + if len(pods) == 0: + log_data = "******* Pods not running ********\n" + else: + k8s_pod_name = pods[0] + # If the pod is not yet started, the logs request below will throw an exception + try: + log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test") + except client.exceptions.ApiException as e: + if opts.o.debug: + print(f"Error from read_namespaced_pod_log: {e}") + log_data = "******* No logs available ********\n" return log_stream_from_string(log_data) - def run(self, image, command, user, volumes, entrypoint=None): + def update(self): + self.connect_api() + ref_deployment = self.cluster_info.get_deployment() + + deployment = self.apps_api.read_namespaced_deployment( + name=ref_deployment.metadata.name, + namespace=self.k8s_namespace + ) + + new_env = ref_deployment.spec.template.spec.containers[0].env + for container in deployment.spec.template.spec.containers: + old_env = container.env + if old_env != new_env: + container.env = new_env + + deployment.spec.template.metadata.annotations = { + "kubectl.kubernetes.io/restartedAt": datetime.utcnow() + .replace(tzinfo=timezone.utc) + .isoformat() + } + + self.apps_api.patch_namespaced_deployment( + name=ref_deployment.metadata.name, + namespace=self.k8s_namespace, + body=deployment + ) + + def run(self, image: str, command=None, user=None, volumes=None, entrypoint=None, env={}, ports=[], detach=False): # We need to figure out how to do this -- check why we're being called first pass + def is_kind(self): + return self.type == "k8s-kind" + class K8sDeployerConfigGenerator(DeployerConfigGenerator): - config_file_name: str = "kind-config.yml" + type: str - def __init__(self) -> None: + def __init__(self, type: str) -> None: + self.type = type super().__init__() def generate(self, deployment_dir: Path): - # Check the file isn't already there - # Get the config file contents - content = generate_kind_config(deployment_dir) - if opts.o.debug: - print(f"kind config is: {content}") - config_file = deployment_dir.joinpath(self.config_file_name) - # Write the file - with open(config_file, "w") as output_file: - output_file.write(content) + # No need to do this for the remote k8s case + if self.type == "k8s-kind": + # Check the file isn't already there + # Get the config file contents + content = generate_kind_config(deployment_dir) + if opts.o.debug: + print(f"kind config is: {content}") + config_file = deployment_dir.joinpath(constants.kind_config_filename) + # Write the file + with open(config_file, "w") as output_file: + output_file.write(content) diff --git a/stack_orchestrator/deploy/k8s/helpers.py b/stack_orchestrator/deploy/k8s/helpers.py index db1ef075..62545dfd 100644 --- a/stack_orchestrator/deploy/k8s/helpers.py +++ b/stack_orchestrator/deploy/k8s/helpers.py @@ -14,14 +14,14 @@ # along with this program. If not, see . from kubernetes import client -from dotenv import dotenv_values import os from pathlib import Path import subprocess -from typing import Any, Set, Mapping, List +from typing import Set, Mapping, List from stack_orchestrator.opts import opts -from stack_orchestrator.util import get_yaml +from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names +from stack_orchestrator.deploy.deployer import DeployerException def _run_command(command: str): @@ -30,10 +30,13 @@ def _run_command(command: str): result = subprocess.run(command, shell=True) if opts.o.debug: print(f"Result: {result}") + return result def create_cluster(name: str, config_file: str): - _run_command(f"kind create cluster --name {name} --config {config_file}") + result = _run_command(f"kind create cluster --name {name} --config {config_file}") + if result.returncode != 0: + raise DeployerException(f"kind create cluster failed: {result}") def destroy_cluster(name: str): @@ -42,12 +45,14 @@ def destroy_cluster(name: str): def load_images_into_kind(kind_cluster_name: str, image_set: Set[str]): for image in image_set: - _run_command(f"kind load docker-image {image} --name {kind_cluster_name}") + result = _run_command(f"kind load docker-image {image} --name {kind_cluster_name}") + if result.returncode != 0: + raise DeployerException(f"kind create cluster failed: {result}") def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str): pods = [] - pod_response = core_api.list_namespaced_pod(namespace="default", label_selector="app=test-app") + pod_response = core_api.list_namespaced_pod(namespace="default", label_selector=f"app={deployment_name}") if opts.o.debug: print(f"pod_response: {pod_response}") for pod_info in pod_response.items: @@ -133,17 +138,6 @@ def _make_absolute_host_path(data_mount_path: Path, deployment_dir: Path) -> Pat return Path.cwd().joinpath(deployment_dir.joinpath("compose").joinpath(data_mount_path)).resolve() -def parsed_pod_files_map_from_file_names(pod_files): - parsed_pod_yaml_map : Any = {} - for pod_file in pod_files: - with open(pod_file, "r") as pod_file_descriptor: - parsed_pod_file = get_yaml().load(pod_file_descriptor) - parsed_pod_yaml_map[pod_file] = parsed_pod_file - if opts.o.debug: - print(f"parsed_pod_yaml_map: {parsed_pod_yaml_map}") - return parsed_pod_yaml_map - - def _generate_kind_mounts(parsed_pod_files, deployment_dir): volume_definitions = [] volume_host_path_map = _get_host_paths_for_volumes(parsed_pod_files) @@ -235,7 +229,3 @@ def generate_kind_config(deployment_dir: Path): f"{port_mappings_yml}\n" f"{mounts_yml}\n" ) - - -def env_var_map_from_file(file: Path) -> Mapping[str, str]: - return dotenv_values(file) diff --git a/stack_orchestrator/deploy/spec.py b/stack_orchestrator/deploy/spec.py index 9ee893b9..c4f791bf 100644 --- a/stack_orchestrator/deploy/spec.py +++ b/stack_orchestrator/deploy/spec.py @@ -16,6 +16,7 @@ from pathlib import Path import typing from stack_orchestrator.util import get_yaml +from stack_orchestrator import constants class Spec: @@ -28,3 +29,14 @@ class Spec: def init_from_file(self, file_path: Path): with file_path: self.obj = get_yaml().load(open(file_path, "r")) + + def get_image_registry(self): + return (self.obj[constants.image_resigtry_key] + if self.obj and constants.image_resigtry_key in self.obj + else None) + + def get_http_proxy(self): + return (self.obj[constants.network_key][constants.http_proxy_key] + if self.obj and constants.network_key in self.obj + and constants.http_proxy_key in self.obj[constants.network_key] + else None) diff --git a/stack_orchestrator/deploy/webapp/__init__.py b/stack_orchestrator/deploy/webapp/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp.py b/stack_orchestrator/deploy/webapp/deploy_webapp.py new file mode 100644 index 00000000..aca2df35 --- /dev/null +++ b/stack_orchestrator/deploy/webapp/deploy_webapp.py @@ -0,0 +1,123 @@ +# Copyright ©2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import click +import os +from pathlib import Path +from urllib.parse import urlparse +from tempfile import NamedTemporaryFile + +from stack_orchestrator.util import error_exit, global_options2 +from stack_orchestrator.deploy.deployment_create import init_operation, create_operation +from stack_orchestrator.deploy.deploy import create_deploy_context +from stack_orchestrator.deploy.deploy_types import DeployCommandContext + + +def _fixup_container_tag(deployment_dir: str, image: str): + deployment_dir_path = Path(deployment_dir) + compose_file = deployment_dir_path.joinpath("compose", "docker-compose-webapp-template.yml") + # replace "cerc/webapp-container:local" in the file with our image tag + with open(compose_file) as rfile: + contents = rfile.read() + contents = contents.replace("cerc/webapp-container:local", image) + with open(compose_file, "w") as wfile: + wfile.write(contents) + + +def _fixup_url_spec(spec_file_name: str, url: str): + # url is like: https://example.com/path + parsed_url = urlparse(url) + http_proxy_spec = f''' + http-proxy: + - host-name: {parsed_url.hostname} + routes: + - path: '{parsed_url.path if parsed_url.path else "/"}' + proxy-to: webapp:3000 + ''' + spec_file_path = Path(spec_file_name) + with open(spec_file_path) as rfile: + contents = rfile.read() + contents = contents + http_proxy_spec + with open(spec_file_path, "w") as wfile: + wfile.write(contents) + + +def create_deployment(ctx, deployment_dir, image, url, kube_config, image_registry, env_file): + # Do the equivalent of: + # 1. laconic-so --stack webapp-template deploy --deploy-to k8s init --output webapp-spec.yml + # --config (eqivalent of the contents of my-config.env) + # 2. laconic-so --stack webapp-template deploy --deploy-to k8s create --deployment-dir test-deployment + # --spec-file webapp-spec.yml + # 3. Replace the container image tag with the specified image + deployment_dir_path = Path(deployment_dir) + # Check the deployment dir does not exist + if deployment_dir_path.exists(): + error_exit(f"Deployment dir {deployment_dir} already exists") + # Generate a temporary file name for the spec file + tf = NamedTemporaryFile(prefix="webapp-", suffix=".yml", delete=False) + spec_file_name = tf.name + # Specify the webapp template stack + stack = "webapp-template" + # TODO: support env file + deploy_command_context: DeployCommandContext = create_deploy_context( + global_options2(ctx), None, stack, None, None, None, env_file, "k8s" + ) + init_operation( + deploy_command_context, + stack, + "k8s", + None, + env_file, + kube_config, + image_registry, + spec_file_name, + None + ) + # Add the TLS and DNS spec + _fixup_url_spec(spec_file_name, url) + create_operation( + deploy_command_context, + spec_file_name, + deployment_dir, + None, + None + ) + # Fix up the container tag inside the deployment compose file + _fixup_container_tag(deployment_dir, image) + os.remove(spec_file_name) + + +@click.group() +@click.pass_context +def command(ctx): + '''manage a webapp deployment''' + + # Check that --stack wasn't supplied + if ctx.parent.obj.stack: + error_exit("--stack can't be supplied with the deploy-webapp command") + + +@command.command() +@click.option("--kube-config", help="Provide a config file for a k8s deployment") +@click.option("--image-registry", help="Provide a container image registry url for this k8s cluster") +@click.option("--deployment-dir", help="Create deployment files in this directory", required=True) +@click.option("--image", help="image to deploy", required=True) +@click.option("--url", help="url to serve", required=True) +@click.option("--env-file", help="environment file for webapp") +@click.pass_context +def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_file): + '''create a deployment for the specified webapp container''' + + return create_deployment(ctx, deployment_dir, image, url, kube_config, image_registry, env_file) diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py new file mode 100644 index 00000000..2e4544eb --- /dev/null +++ b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py @@ -0,0 +1,272 @@ +# Copyright ©2023 Vulcanize +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import hashlib +import json +import os +import shlex +import shutil +import sys +import tempfile + +import click + +from stack_orchestrator.deploy.webapp import deploy_webapp +from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient, + build_container_image, push_container_image, + file_hash, deploy_to_k8s, publish_deployment, + hostname_for_deployment_request, generate_hostname_for_app, + match_owner) + + +def process_app_deployment_request( + ctx, + laconic: LaconicRegistryClient, + app_deployment_request, + deployment_record_namespace, + dns_record_namespace, + dns_suffix, + deployment_parent_dir, + kube_config, + image_registry +): + # 1. look up application + app = laconic.get_record(app_deployment_request.attributes.application, require=True) + + # 2. determine dns + requested_name = hostname_for_deployment_request(app_deployment_request, laconic) + + # HACK + if "." in requested_name: + raise Exception("Only unqualified hostnames allowed at this time.") + + fqdn = f"{requested_name}.{dns_suffix}" + + # 3. check ownership of existing dnsrecord vs this request + # TODO: Support foreign DNS + dns_crn = f"{dns_record_namespace}/{fqdn}" + dns_record = laconic.get_record(dns_crn) + if dns_record: + matched_owner = match_owner(app_deployment_request, dns_record) + if not matched_owner and dns_record.request: + matched_owner = match_owner(app_deployment_request, laconic.get_record(dns_record.request, require=True)) + + if matched_owner: + print("Matched DnsRecord ownership:", matched_owner) + else: + raise Exception("Unable to confirm ownership of DnsRecord %s for request %s" % + (dns_record.id, app_deployment_request.id)) + + # 4. get build and runtime config from request + env_filename = None + if app_deployment_request.attributes.config and "env" in app_deployment_request.attributes.config: + env_filename = tempfile.mktemp() + with open(env_filename, 'w') as file: + for k, v in app_deployment_request.attributes.config["env"].items(): + file.write("%s=%s\n" % (k, shlex.quote(str(v)))) + + # 5. determine new or existing deployment + # a. check for deployment crn + app_deployment_crn = f"{deployment_record_namespace}/{fqdn}" + if app_deployment_request.attributes.deployment: + app_deployment_crn = app_deployment_request.attributes.deployment + if not app_deployment_crn.startswith(deployment_record_namespace): + raise Exception("Deployment CRN %s is not in a supported namespace" % app_deployment_request.attributes.deployment) + + deployment_record = laconic.get_record(app_deployment_crn) + deployment_dir = os.path.join(deployment_parent_dir, fqdn) + deployment_config_file = os.path.join(deployment_dir, "config.env") + deployment_container_tag = "laconic-webapp/%s:local" % hashlib.md5(deployment_dir.encode()).hexdigest() + # b. check for deployment directory (create if necessary) + if not os.path.exists(deployment_dir): + if deployment_record: + raise Exception("Deployment record %s exists, but not deployment dir %s. Please remove name." % + (app_deployment_crn, deployment_dir)) + print("deploy_webapp", deployment_dir) + deploy_webapp.create_deployment(ctx, deployment_dir, deployment_container_tag, + f"https://{fqdn}", kube_config, image_registry, env_filename) + elif env_filename: + shutil.copyfile(env_filename, deployment_config_file) + + needs_k8s_deploy = False + # 6. build container (if needed) + if not deployment_record or deployment_record.attributes.application != app.id: + build_container_image(app, deployment_container_tag) + push_container_image(deployment_dir) + needs_k8s_deploy = True + + # 7. update config (if needed) + if not deployment_record or file_hash(deployment_config_file) != deployment_record.attributes.meta.config: + needs_k8s_deploy = True + + # 8. update k8s deployment + if needs_k8s_deploy: + print("Deploying to k8s") + deploy_to_k8s( + deployment_record, + deployment_dir, + ) + + publish_deployment( + laconic, + app, + deployment_record, + app_deployment_crn, + dns_record, + dns_crn, + deployment_dir, + app_deployment_request + ) + + +def load_known_requests(filename): + if filename and os.path.exists(filename): + return json.load(open(filename, "r")) + return {} + + +def dump_known_requests(filename, requests): + if not filename: + return + known_requests = load_known_requests(filename) + for r in requests: + known_requests[r.id] = r.createTime + json.dump(known_requests, open(filename, "w")) + + +@click.command() +@click.option("--kube-config", help="Provide a config file for a k8s deployment") +@click.option("--laconic-config", help="Provide a config file for laconicd", required=True) +@click.option("--image-registry", help="Provide a container image registry url for this k8s cluster") +@click.option("--deployment-parent-dir", help="Create deployment directories beneath this directory", required=True) +@click.option("--request-id", help="The ApplicationDeploymentRequest to process") +@click.option("--discover", help="Discover and process all pending ApplicationDeploymentRequests", is_flag=True, default=False) +@click.option("--state-file", help="File to store state about previously seen requests.") +@click.option("--only-update-state", help="Only update the state file, don't process any requests anything.", is_flag=True) +@click.option("--dns-suffix", help="DNS domain to use eg, laconic.servesthe.world") +@click.option("--record-namespace-dns", help="eg, crn://laconic/dns") +@click.option("--record-namespace-deployments", help="eg, crn://laconic/deployments") +@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True) +@click.pass_context +def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir, + request_id, discover, state_file, only_update_state, + dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run): + if request_id and discover: + print("Cannot specify both --request-id and --discover", file=sys.stderr) + sys.exit(2) + + if not request_id and not discover: + print("Must specify either --request-id or --discover", file=sys.stderr) + sys.exit(2) + + if only_update_state and not state_file: + print("--only-update-state requires --state-file", file=sys.stderr) + sys.exit(2) + + if not only_update_state: + if not record_namespace_dns or not record_namespace_deployments or not dns_suffix: + print("--dns-suffix, --record-namespace-dns, and --record-namespace-deployments are all required", file=sys.stderr) + sys.exit(2) + + laconic = LaconicRegistryClient(laconic_config) + + # Find deployment requests. + # single request + if request_id: + requests = [laconic.get_record(request_id, require=True)] + # all requests + elif discover: + requests = laconic.app_deployment_requests() + + if only_update_state: + if not dry_run: + dump_known_requests(state_file, requests) + return + + previous_requests = load_known_requests(state_file) + + # Collapse related requests. + requests.sort(key=lambda r: r.createTime) + requests.reverse() + requests_by_name = {} + for r in requests: + app = laconic.get_record(r.attributes.application) + if not app: + print("Skipping request %s, cannot locate app." % r.id) + continue + + requested_name = r.attributes.dns + if not requested_name: + requested_name = generate_hostname_for_app(app) + print("Generating name %s for request %s." % (requested_name, r.id)) + + if requested_name not in requests_by_name: + print( + "Found request %s to run application %s on %s." + % (r.id, r.attributes.application, requested_name) + ) + requests_by_name[requested_name] = r + else: + print( + "Ignoring request %s, it is superseded by %s." + % (r.id, requests_by_name[requested_name].id) + ) + + # Find deployments. + deployments = laconic.app_deployments() + deployments_by_request = {} + for d in deployments: + if d.attributes.request: + deployments_by_request[d.attributes.request] = d + + # Find removal requests. + cancellation_requests = {} + removal_requests = laconic.app_deployment_removal_requests() + for r in removal_requests: + if r.attributes.request: + cancellation_requests[r.attributes.request] = r + + requests_to_execute = [] + for r in requests_by_name.values(): + if r.id in cancellation_requests and match_owner(cancellation_requests[r.id], r): + print(f"Found deployment cancellation request for {r.id} at {cancellation_requests[r.id].id}") + elif r.id in deployments_by_request: + print(f"Found satisfied request for {r.id} at {deployments_by_request[r.id].id}") + else: + if r.id not in previous_requests: + print(f"Request {r.id} needs to processed.") + requests_to_execute.append(r) + else: + print( + f"Skipping unsatisfied request {r.id} because we have seen it before." + ) + + print("Found %d unsatisfied request(s) to process." % len(requests_to_execute)) + + if not dry_run: + for r in requests_to_execute: + try: + process_app_deployment_request( + ctx, + laconic, + r, + record_namespace_deployments, + record_namespace_dns, + dns_suffix, + os.path.abspath(deployment_parent_dir), + kube_config, + image_registry + ) + finally: + dump_known_requests(state_file, [r]) diff --git a/stack_orchestrator/deploy/webapp/run_webapp.py b/stack_orchestrator/deploy/webapp/run_webapp.py new file mode 100644 index 00000000..4dbf234a --- /dev/null +++ b/stack_orchestrator/deploy/webapp/run_webapp.py @@ -0,0 +1,65 @@ +# Copyright © 2022, 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# Builds webapp containers + +# env vars: +# CERC_REPO_BASE_DIR defaults to ~/cerc + +# TODO: display the available list of containers; allow re-build of either all or specific containers + +import hashlib +import click +from dotenv import dotenv_values + +from stack_orchestrator import constants +from stack_orchestrator.deploy.deployer_factory import getDeployer + +WEBAPP_PORT = 3000 + + +@click.command() +@click.option("--image", help="image to deploy", required=True) +@click.option("--env-file", help="environment file for webapp") +@click.option("--port", help="port to use (default random)") +@click.pass_context +def command(ctx, image, env_file, port): + '''run the specified webapp container''' + + env = {} + if env_file: + env = dotenv_values(env_file) + + unique_cluster_descriptor = f"{image},{env}" + hash = hashlib.md5(unique_cluster_descriptor.encode()).hexdigest() + cluster = f"laconic-webapp-{hash}" + + deployer = getDeployer(type=constants.compose_deploy_type, + deployment_context=None, + compose_files=None, + compose_project_name=cluster, + compose_env_file=None) + + ports = [] + if port: + ports = [(port, WEBAPP_PORT)] + container = deployer.run(image, command=[], user=None, volumes=[], entrypoint=None, env=env, ports=ports, detach=True) + + # Make configurable? + webappPort = f"{WEBAPP_PORT}/tcp" + # TODO: This assumes a Docker container object... + if webappPort in container.network_settings.ports: + mapping = container.network_settings.ports[webappPort][0] + print(f"""Image: {image}\nID: {container.id}\nURL: http://localhost:{mapping['HostPort']}""") diff --git a/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py new file mode 100644 index 00000000..6c5cac85 --- /dev/null +++ b/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py @@ -0,0 +1,189 @@ +# Copyright ©2023 Vulcanize +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import json +import os +import shutil +import subprocess +import sys + +import click + +from stack_orchestrator.deploy.webapp.util import LaconicRegistryClient, match_owner + + +def process_app_removal_request(ctx, + laconic: LaconicRegistryClient, + app_removal_request, + deployment_parent_dir, + delete_volumes, + delete_names): + deployment_record = laconic.get_record(app_removal_request.attributes.deployment, require=True) + dns_record = laconic.get_record(deployment_record.attributes.dns, require=True) + deployment_dir = os.path.join(deployment_parent_dir, dns_record.attributes.name) + + if not os.path.exists(deployment_dir): + raise Exception("Deployment directory %s does not exist." % deployment_dir) + + # Check if the removal request is from the owner of the DnsRecord or deployment record. + matched_owner = match_owner(app_removal_request, deployment_record, dns_record) + + # Or of the original deployment request. + if not matched_owner and deployment_record.request: + matched_owner = match_owner(app_removal_request, laconic.get_record(deployment_record.request, require=True)) + + if matched_owner: + print("Matched deployment ownership:", matched_owner) + else: + raise Exception("Unable to confirm ownership of deployment %s for removal request %s" % + (deployment_record.id, app_removal_request.id)) + + # TODO(telackey): Call the function directly. The easiest way to build the correct click context is to + # exec the process, but it would be better to refactor so we could just call down_operation with the + # necessary parameters + down_command = [sys.argv[0], "deployment", "--dir", deployment_dir, "down"] + if delete_volumes: + down_command.append("--delete-volumes") + result = subprocess.run(down_command) + result.check_returncode() + + removal_record = { + "record": { + "type": "ApplicationDeploymentRemovalRecord", + "version": "1.0.0", + "request": app_removal_request.id, + "deployment": deployment_record.id, + } + } + laconic.publish(removal_record) + + if delete_names: + if deployment_record.names: + for name in deployment_record.names: + laconic.delete_name(name) + + if dns_record.names: + for name in dns_record.names: + laconic.delete_name(name) + + if delete_volumes: + shutil.rmtree(deployment_dir) + + +def load_known_requests(filename): + if filename and os.path.exists(filename): + return json.load(open(filename, "r")) + return {} + + +def dump_known_requests(filename, requests): + if not filename: + return + known_requests = load_known_requests(filename) + for r in requests: + known_requests[r.id] = r.createTime + json.dump(known_requests, open(filename, "w")) + + +@click.command() +@click.option("--laconic-config", help="Provide a config file for laconicd", required=True) +@click.option("--deployment-parent-dir", help="Create deployment directories beneath this directory", required=True) +@click.option("--request-id", help="The ApplicationDeploymentRemovalRequest to process") +@click.option("--discover", help="Discover and process all pending ApplicationDeploymentRemovalRequests", + is_flag=True, default=False) +@click.option("--state-file", help="File to store state about previously seen requests.") +@click.option("--only-update-state", help="Only update the state file, don't process any requests anything.", is_flag=True) +@click.option("--delete-names/--preserve-names", help="Delete all names associated with removed deployments.", default=True) +@click.option("--delete-volumes/--preserve-volumes", default=True, help="delete data volumes") +@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True) +@click.pass_context +def command(ctx, laconic_config, deployment_parent_dir, + request_id, discover, state_file, only_update_state, + delete_names, delete_volumes, dry_run): + if request_id and discover: + print("Cannot specify both --request-id and --discover", file=sys.stderr) + sys.exit(2) + + if not request_id and not discover: + print("Must specify either --request-id or --discover", file=sys.stderr) + sys.exit(2) + + if only_update_state and not state_file: + print("--only-update-state requires --state-file", file=sys.stderr) + sys.exit(2) + + laconic = LaconicRegistryClient(laconic_config) + + # Find deployment removal requests. + # single request + if request_id: + requests = [laconic.get_record(request_id, require=True)] + # TODO: assert record type + # all requests + elif discover: + requests = laconic.app_deployment_removal_requests() + + if only_update_state: + if not dry_run: + dump_known_requests(state_file, requests) + return + + previous_requests = load_known_requests(state_file) + requests.sort(key=lambda r: r.createTime) + + # Find deployments. + deployments = {} + for d in laconic.app_deployments(all=True): + deployments[d.id] = d + + # Find removal requests. + removals_by_deployment = {} + removals_by_request = {} + for r in laconic.app_deployment_removals(): + if r.attributes.deployment: + # TODO: should we handle CRNs? + removals_by_deployment[r.attributes.deployment] = r + + requests_to_execute = [] + for r in requests: + if not r.attributes.deployment: + print(f"Skipping removal request {r.id} since it was a cancellation.") + elif r.id in removals_by_request: + print(f"Found satisfied request for {r.id} at {removals_by_request[r.id].id}") + elif r.attributes.deployment in removals_by_deployment: + print( + f"Found removal record for indicated deployment {r.attributes.deployment} at " + f"{removals_by_deployment[r.attributes.deployment].id}") + else: + if r.id not in previous_requests: + print(f"Request {r.id} needs to processed.") + requests_to_execute.append(r) + else: + print(f"Skipping unsatisfied request {r.id} because we have seen it before.") + + print("Found %d unsatisfied request(s) to process." % len(requests_to_execute)) + + if not dry_run: + for r in requests_to_execute: + try: + process_app_removal_request( + ctx, + laconic, + r, + os.path.abspath(deployment_parent_dir), + delete_volumes, + delete_names + ) + finally: + dump_known_requests(state_file, [r]) diff --git a/stack_orchestrator/deploy/webapp/util.py b/stack_orchestrator/deploy/webapp/util.py new file mode 100644 index 00000000..ef98117b --- /dev/null +++ b/stack_orchestrator/deploy/webapp/util.py @@ -0,0 +1,327 @@ +# Copyright © 2023 Vulcanize + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import hashlib +import json +import os +import random +import subprocess +import sys +import tempfile +import uuid + +import yaml + + +class AttrDict(dict): + def __init__(self, *args, **kwargs): + super(AttrDict, self).__init__(*args, **kwargs) + self.__dict__ = self + + def __getattribute__(self, attr): + __dict__ = super(AttrDict, self).__getattribute__("__dict__") + if attr in __dict__: + v = super(AttrDict, self).__getattribute__(attr) + if isinstance(v, dict): + return AttrDict(v) + return v + + +def cmd(*vargs): + try: + result = subprocess.run(vargs, capture_output=True) + result.check_returncode() + return result.stdout.decode() + except Exception as err: + print(result.stderr.decode()) + raise err + + +def match_owner(recordA, *records): + for owner in recordA.owners: + for otherRecord in records: + if owner in otherRecord.owners: + return owner + return None + + +class LaconicRegistryClient: + def __init__(self, config_file): + self.config_file = config_file + self.cache = AttrDict( + { + "name_or_id": {}, + } + ) + + def list_records(self, criteria={}, all=False): + args = ["laconic", "-c", self.config_file, "cns", "record", "list"] + + if all: + args.append("--all") + + if criteria: + for k, v in criteria.items(): + args.append("--%s" % k) + args.append(str(v)) + + results = [AttrDict(r) for r in json.loads(cmd(*args))] + + # Most recent records first + results.sort(key=lambda r: r.createTime) + results.reverse() + + return results + + def is_crn(self, name_or_id: str): + if name_or_id: + return str(name_or_id).startswith("crn://") + return False + + def is_id(self, name_or_id: str): + return not self.is_crn(name_or_id) + + def _add_to_cache(self, records): + if not records: + return + + for p in records: + self.cache["name_or_id"][p.id] = p + if p.names: + for crn in p.names: + self.cache["name_or_id"][crn] = p + if p.attributes.type not in self.cache: + self.cache[p.attributes.type] = [] + self.cache[p.attributes.type].append(p) + + def resolve(self, name): + if not name: + return None + + if name in self.cache.name_or_id: + return self.cache.name_or_id[name] + + args = ["laconic", "-c", self.config_file, "cns", "name", "resolve", name] + + parsed = [AttrDict(r) for r in json.loads(cmd(*args))] + if parsed: + self._add_to_cache(parsed) + return parsed[0] + + return None + + def get_record(self, name_or_id, require=False): + if not name_or_id: + if require: + raise Exception("Cannot locate record:", name_or_id) + return None + + if name_or_id in self.cache.name_or_id: + return self.cache.name_or_id[name_or_id] + + if self.is_crn(name_or_id): + return self.resolve(name_or_id) + + args = [ + "laconic", + "-c", + self.config_file, + "cns", + "record", + "get", + "--id", + name_or_id, + ] + + parsed = [AttrDict(r) for r in json.loads(cmd(*args))] + if len(parsed): + self._add_to_cache(parsed) + return parsed[0] + + if require: + raise Exception("Cannot locate record:", name_or_id) + return None + + def app_deployment_requests(self, all=True): + return self.list_records({"type": "ApplicationDeploymentRequest"}, all) + + def app_deployments(self, all=True): + return self.list_records({"type": "ApplicationDeploymentRecord"}, all) + + def app_deployment_removal_requests(self, all=True): + return self.list_records({"type": "ApplicationDeploymentRemovalRequest"}, all) + + def app_deployment_removals(self, all=True): + return self.list_records({"type": "ApplicationDeploymentRemovalRecord"}, all) + + def publish(self, record, names=[]): + tmpdir = tempfile.mkdtemp() + try: + record_fname = os.path.join(tmpdir, "record.yml") + record_file = open(record_fname, 'w') + yaml.dump(record, record_file) + record_file.close() + print(open(record_fname, 'r').read()) + + new_record_id = json.loads( + cmd("laconic", "-c", self.config_file, "cns", "record", "publish", "--filename", record_fname) + )["id"] + for name in names: + self.set_name(name, new_record_id) + return new_record_id + finally: + cmd("rm", "-rf", tmpdir) + + def set_name(self, name, record_id): + cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, record_id) + + def delete_name(self, name): + cmd("laconic", "-c", self.config_file, "cns", "name", "delete", name) + + +def file_hash(filename): + return hashlib.sha1(open(filename).read().encode()).hexdigest() + + +def build_container_image(app_record, tag, extra_build_args=[]): + tmpdir = tempfile.mkdtemp() + + try: + record_id = app_record["id"] + ref = app_record.attributes.repository_ref + repo = random.choice(app_record.attributes.repository) + clone_dir = os.path.join(tmpdir, record_id) + + print(f"Cloning repository {repo} to {clone_dir} ...") + if ref: + # TODO: Determing branch or hash, and use depth 1 if we can. + git_env = dict(os.environ.copy()) + # Never prompt + git_env["GIT_TERMINAL_PROMPT"] = "0" + subprocess.check_call(["git", "clone", repo, clone_dir], env=git_env) + subprocess.check_call(["git", "checkout", ref], cwd=clone_dir, env=git_env) + else: + result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir]) + result.check_returncode() + + print("Building webapp ...") + build_command = [sys.argv[0], "build-webapp", "--source-repo", clone_dir, "--tag", tag] + if extra_build_args: + build_command.append("--extra-build-args") + build_command.append(" ".join(extra_build_args)) + + result = subprocess.run(build_command) + result.check_returncode() + finally: + cmd("rm", "-rf", tmpdir) + + +def push_container_image(deployment_dir): + print("Pushing image ...") + result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"]) + result.check_returncode() + + +def deploy_to_k8s(deploy_record, deployment_dir): + if not deploy_record: + command = "up" + else: + command = "update" + + result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command]) + result.check_returncode() + + +def publish_deployment(laconic: LaconicRegistryClient, + app_record, + deploy_record, + deployment_crn, + dns_record, + dns_crn, + deployment_dir, + app_deployment_request=None): + if not deploy_record: + deploy_ver = "0.0.1" + else: + deploy_ver = "0.0.%d" % (int(deploy_record.attributes.version.split(".")[-1]) + 1) + + if not dns_record: + dns_ver = "0.0.1" + else: + dns_ver = "0.0.%d" % (int(dns_record.attributes.version.split(".")[-1]) + 1) + + spec = yaml.full_load(open(os.path.join(deployment_dir, "spec.yml"))) + fqdn = spec["network"]["http-proxy"][0]["host-name"] + + uniq = uuid.uuid4() + + new_dns_record = { + "record": { + "type": "DnsRecord", + "version": dns_ver, + "name": fqdn, + "resource_type": "A", + "meta": { + "so": uniq.hex + }, + } + } + if app_deployment_request: + new_dns_record["record"]["request"] = app_deployment_request.id + + dns_id = laconic.publish(new_dns_record, [dns_crn]) + + new_deployment_record = { + "record": { + "type": "ApplicationDeploymentRecord", + "version": deploy_ver, + "url": f"https://{fqdn}", + "name": app_record.attributes.name, + "application": app_record.id, + "dns": dns_id, + "meta": { + "config": file_hash(os.path.join(deployment_dir, "config.env")), + "so": uniq.hex + }, + } + } + if app_deployment_request: + new_deployment_record["record"]["request"] = app_deployment_request.id + + deployment_id = laconic.publish(new_deployment_record, [deployment_crn]) + return {"dns": dns_id, "deployment": deployment_id} + + +def hostname_for_deployment_request(app_deployment_request, laconic): + dns_name = app_deployment_request.attributes.dns + if not dns_name: + app = laconic.get_record(app_deployment_request.attributes.application, require=True) + dns_name = generate_hostname_for_app(app) + elif dns_name.startswith("crn://"): + record = laconic.get_record(dns_name, require=True) + dns_name = record.attributes.name + return dns_name + + +def generate_hostname_for_app(app): + last_part = app.attributes.name.split("/")[-1] + m = hashlib.sha256() + m.update(app.attributes.name.encode()) + m.update(b"|") + if isinstance(app.attributes.repository, list): + m.update(app.attributes.repository[0].encode()) + else: + m.update(app.attributes.repository.encode()) + return "%s-%s" % (last_part, m.hexdigest()[0:10]) diff --git a/stack_orchestrator/main.py b/stack_orchestrator/main.py index 0b0585e0..a4644412 100644 --- a/stack_orchestrator/main.py +++ b/stack_orchestrator/main.py @@ -20,6 +20,10 @@ from stack_orchestrator.repos import setup_repositories from stack_orchestrator.build import build_containers from stack_orchestrator.build import build_npms from stack_orchestrator.build import build_webapp +from stack_orchestrator.deploy.webapp import (run_webapp, + deploy_webapp, + deploy_webapp_from_registry, + undeploy_webapp_from_registry) from stack_orchestrator.deploy import deploy from stack_orchestrator import version from stack_orchestrator.deploy import deployment @@ -50,6 +54,10 @@ cli.add_command(setup_repositories.command, "setup-repositories") cli.add_command(build_containers.command, "build-containers") cli.add_command(build_npms.command, "build-npms") cli.add_command(build_webapp.command, "build-webapp") +cli.add_command(run_webapp.command, "run-webapp") +cli.add_command(deploy_webapp.command, "deploy-webapp") +cli.add_command(deploy_webapp_from_registry.command, "deploy-webapp-from-registry") +cli.add_command(undeploy_webapp_from_registry.command, "undeploy-webapp-from-registry") cli.add_command(deploy.command, "deploy") # deploy is an alias for deploy-system cli.add_command(deploy.command, "deploy-system") cli.add_command(deployment.command, "deployment") diff --git a/stack_orchestrator/repos/setup_repositories.py b/stack_orchestrator/repos/setup_repositories.py index feca7897..3612aed0 100644 --- a/stack_orchestrator/repos/setup_repositories.py +++ b/stack_orchestrator/repos/setup_repositories.py @@ -25,7 +25,8 @@ import click import importlib.resources from pathlib import Path import yaml -from stack_orchestrator.util import include_exclude_check +from stack_orchestrator.constants import stack_file_name +from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit class GitProgress(git.RemoteProgress): @@ -238,13 +239,20 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches repos_in_scope = [] if stack: - # In order to be compatible with Python 3.8 we need to use this hack to get the path: - # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure - stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, "stack.yml") + if stack_is_external(stack): + stack_file_path = Path(stack).joinpath(stack_file_name) + else: + # In order to be compatible with Python 3.8 we need to use this hack to get the path: + # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure + stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, stack_file_name) + if not stack_file_path.exists(): + error_exit(f"stack {stack} does not exist") with stack_file_path: stack_config = yaml.safe_load(open(stack_file_path, "r")) - # TODO: syntax check the input here - repos_in_scope = stack_config['repos'] + if "repos" not in stack_config: + error_exit(f"stack {stack} does not define any repositories") + else: + repos_in_scope = stack_config["repos"] else: repos_in_scope = all_repos diff --git a/stack_orchestrator/util.py b/stack_orchestrator/util.py index d3b733a2..0bd1a609 100644 --- a/stack_orchestrator/util.py +++ b/stack_orchestrator/util.py @@ -18,6 +18,8 @@ import os.path import sys import ruamel.yaml from pathlib import Path +from dotenv import dotenv_values +from typing import Mapping def include_exclude_check(s, include, exclude): @@ -150,6 +152,12 @@ def get_parsed_deployment_spec(spec_file): sys.exit(1) +def stack_is_external(stack: str): + # Bit of a hack: if the supplied stack string represents + # a path that exists then we assume it must be external + return Path(stack).exists() if stack is not None else False + + def get_yaml(): # See: https://stackoverflow.com/a/45701840/1701505 yaml = ruamel.yaml.YAML() @@ -167,3 +175,12 @@ def global_options(ctx): # TODO: hack def global_options2(ctx): return ctx.parent.obj + + +def error_exit(s): + print(f"ERROR: {s}") + sys.exit(1) + + +def env_var_map_from_file(file: Path) -> Mapping[str, str]: + return dotenv_values(file) diff --git a/tests/k8s-deploy/run-deploy-test.sh b/tests/k8s-deploy/run-deploy-test.sh index 91c7890c..15eb2d3b 100755 --- a/tests/k8s-deploy/run-deploy-test.sh +++ b/tests/k8s-deploy/run-deploy-test.sh @@ -1,14 +1,59 @@ #!/usr/bin/env bash set -e if [ -n "$CERC_SCRIPT_DEBUG" ]; then - set -x + set -x + # Dump environment variables for debugging + echo "Environment variables:" + env fi + +# Helper functions: TODO move into a separate file +wait_for_pods_started () { + for i in {1..5} + do + local ps_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir ps ) + + if [[ "$ps_output" == *"Running containers:"* ]]; then + # if ready, return + return + else + # if not ready, wait + sleep 5 + fi + done + # Timed out, error exit + echo "waiting for pods to start: FAILED" + delete_cluster_exit +} + +wait_for_log_output () { + for i in {1..5} + do + + local log_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs ) + + if [[ ! -z "$log_output" ]]; then + # if ready, return + return + else + # if not ready, wait + sleep 5 + fi + done + # Timed out, error exit + echo "waiting for pods log content: FAILED" + delete_cluster_exit +} + + +delete_cluster_exit () { + $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes + exit 1 +} + # Note: eventually this test should be folded into ../deploy/ # but keeping it separate for now for convenience TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 ) -# Dump environment variables for debugging -echo "Environment variables:" -env # Set a non-default repo dir export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir echo "Testing this package: $TEST_TARGET_SO" @@ -18,10 +63,12 @@ echo "Version reported is: ${reported_version_string}" echo "Cloning repositories into: $CERC_REPO_BASE_DIR" rm -rf $CERC_REPO_BASE_DIR mkdir -p $CERC_REPO_BASE_DIR -# Test basic stack-orchestrator deploy +$TEST_TARGET_SO --stack test setup-repositories +$TEST_TARGET_SO --stack test build-containers +# Test basic stack-orchestrator deploy to k8s test_deployment_dir=$CERC_REPO_BASE_DIR/test-deployment-dir test_deployment_spec=$CERC_REPO_BASE_DIR/test-deployment-spec.yml -$TEST_TARGET_SO --stack test deploy --deploy-to k8s init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED +$TEST_TARGET_SO --stack test deploy --deploy-to k8s-kind init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED # Check the file now exists if [ ! -f "$test_deployment_spec" ]; then echo "deploy init test: spec file not present" @@ -53,23 +100,36 @@ fi echo "deploy create output file test: passed" # Try to start the deployment $TEST_TARGET_SO deployment --dir $test_deployment_dir start -# TODO: add a check to see if the container is up -# Sleep because k8s not up yet -sleep 30 +wait_for_pods_started # Check logs command works +wait_for_log_output log_output_3=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs ) if [[ "$log_output_3" == *"Filesystem is fresh"* ]]; then echo "deployment logs test: passed" else echo "deployment logs test: FAILED" - exit 1 + delete_cluster_exit fi # Check the config variable CERC_TEST_PARAM_1 was passed correctly if [[ "$log_output_3" == *"Test-param-1: PASSED"* ]]; then echo "deployment config test: passed" else echo "deployment config test: FAILED" - exit 1 + delete_cluster_exit +fi +# Stop then start again and check the volume was preserved +$TEST_TARGET_SO deployment --dir $test_deployment_dir stop +# Sleep a bit just in case +sleep 2 +$TEST_TARGET_SO deployment --dir $test_deployment_dir start +wait_for_pods_started +wait_for_log_output +log_output_4=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs ) +if [[ "$log_output_4" == *"Filesystem is old"* ]]; then + echo "Retain volumes test: passed" +else + echo "Retain volumes test: FAILED" + delete_cluster_exit fi # Stop and clean up $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes diff --git a/tests/scripts/install-kind.sh b/tests/scripts/install-kind.sh new file mode 100755 index 00000000..254c3288 --- /dev/null +++ b/tests/scripts/install-kind.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# TODO: handle ARM +curl --silent -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 +chmod +x ./kind +mv ./kind /usr/local/bin diff --git a/tests/scripts/install-kubectl.sh b/tests/scripts/install-kubectl.sh new file mode 100755 index 00000000..7a5062fe --- /dev/null +++ b/tests/scripts/install-kubectl.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# TODO: handle ARM +curl --silent -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" +chmod +x ./kubectl +mv ./kubectl /usr/local/bin diff --git a/tests/webapp-test/run-webapp-test.sh b/tests/webapp-test/run-webapp-test.sh index 71b4da16..5db382f8 100755 --- a/tests/webapp-test/run-webapp-test.sh +++ b/tests/webapp-test/run-webapp-test.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash set -e + if [ -n "$CERC_SCRIPT_DEBUG" ]; then set -x fi + # Dump environment variables for debugging echo "Environment variables:" env @@ -24,26 +26,28 @@ git clone https://git.vdb.to/cerc-io/test-progressive-web-app.git $CERC_REPO_BAS # Test webapp command execution $TEST_TARGET_SO build-webapp --source-repo $CERC_REPO_BASE_DIR/test-progressive-web-app -UUID=`uuidgen` +CHECK="SPECIAL_01234567890_TEST_STRING" set +e -CONTAINER_ID=$(docker run -p 3000:3000 -d cerc/test-progressive-web-app:local) +CONTAINER_ID=$(docker run -p 3000:3000 -d -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG cerc/test-progressive-web-app:local) sleep 3 -wget -O test.before -m http://localhost:3000 +wget -t 7 -O test.before -m http://localhost:3000 +docker logs $CONTAINER_ID docker remove -f $CONTAINER_ID -CONTAINER_ID=$(docker run -p 3000:3000 -e CERC_WEBAPP_DEBUG=$UUID -d cerc/test-progressive-web-app:local) +CONTAINER_ID=$(docker run -p 3000:3000 -e CERC_WEBAPP_DEBUG=$CHECK -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG -d cerc/test-progressive-web-app:local) sleep 3 -wget -O test.after -m http://localhost:3000 +wget -t 7 -O test.after -m http://localhost:3000 +docker logs $CONTAINER_ID docker remove -f $CONTAINER_ID echo "###########################################################################" echo "" -grep "$UUID" test.before > /dev/null +grep "$CHECK" test.before > /dev/null if [ $? -ne 1 ]; then echo "BEFORE: FAILED" exit 1 @@ -51,7 +55,7 @@ else echo "BEFORE: PASSED" fi -grep "$UUID" test.after > /dev/null +grep "$CHECK" test.after > /dev/null if [ $? -ne 0 ]; then echo "AFTER: FAILED" exit 1