Basic webapp deployer stack. (#722)

* Support other webapp types (react, static).

.gitea/workflows/fixturenet-eth-plugeth-test.yml

@@ -6,6 +6,8 @@ on:
     paths:
       - '!**'
       - '.gitea/workflows/triggers/fixturenet-eth-plugeth-test'
+  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
+    - cron: '2 14 * * *'
 
 # Needed until we can incorporate docker startup into the executor container
 env:

.gitea/workflows/fixturenet-laconicd-test.yml

@@ -6,11 +6,8 @@ on:
     paths:
       - '!**'
       - '.gitea/workflows/triggers/fixturenet-laconicd-test'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
+  schedule:
+    - cron: '1 13 * * *'
 
 jobs:
   test:
@@ -47,9 +44,5 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
       - name: "Run fixturenet-laconicd tests"
         run: ./tests/fixturenet-laconicd/run-test.sh

.gitea/workflows/test-k8s-deploy.yml

@@ -1,23 +1,20 @@
 name: K8s Deploy Test
 
 on:
-  pull_request:
-    branches: '*'
   push:
-    branches:
-      - main
-      - ci-test
-    paths-ignore:
-      - '.gitea/workflows/triggers/*'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
+    branches: '*'
+    paths:
+      - '!**'
+      - '.gitea/workflows/triggers/test-k8s-deploy'
+      - '.gitea/workflows/test-k8s-deploy.yml'
+      - 'tests/k8s-deploy/run-deploy-test.sh'
+  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
+    - cron: '3 15 * * *'
 
 jobs:
   test:
-    name: "Run deploy test suite"
-    runs-on: ubuntu-latest
+    name: "Run deploy test suite on kind/k8s"
+    runs-on: ubuntu-22.04
     steps:
       - name: "Clone project repository"
         uses: actions/checkout@v3
@@ -41,15 +38,15 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
+      - name: "Check cgroups version"
+        run: mount | grep cgroup
+      - name: "Install kind"
+        run: ./tests/scripts/install-kind.sh
+      - name: "Install Kubectl"
+        run: ./tests/scripts/install-kubectl.sh
+      - name: "Run k8s deployment test"
         run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
-      - name: "Install Go"
-        uses: actions/setup-go@v4
-        with:
-          go-version: '1.21'
-      - name: "Install Kind"
-        run: go install sigs.k8s.io/kind@v0.20.0
-      - name: "Debug Kind"
-        run: kind create cluster --retain && docker logs kind-control-plane
+          source /opt/bash-utils/cgroup-helper.sh
+          join_cgroup
+          ./tests/k8s-deploy/run-deploy-test.sh
+

.gitea/workflows/test-webapp.yml

@@ -41,6 +41,8 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
+      - name: "Install wget" # 20240109 - Only needed until the executors are updated.
+        run: apt update && apt install -y wget
       - name: Start dockerd # Also needed until we can incorporate into the executor
         run: |
           dockerd -H $DOCKER_HOST --userland-proxy=false &

.gitea/workflows/triggers/fixturenet-eth-plugeth-test

@@ -1,2 +1,3 @@
 Change this file to trigger running the fixturenet-eth-plugeth-test CI job
 trigger
+trigger

.gitea/workflows/triggers/fixturenet-laconicd-test

@@ -1,2 +1,3 @@
 Change this file to trigger running the fixturenet-laconicd-test CI job
-
+Trigger
+Trigger

.gitea/workflows/triggers/test-k8s-deploy

@@ -0,0 +1 @@
+Change this file to trigger running the test-k8s-deploy CI job

stack_orchestrator/build/build_containers.py

@@ -33,6 +33,7 @@ from stack_orchestrator.base import get_npm_registry_url
 # TODO: find a place for this
 #    epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
 
+
 def make_container_build_env(dev_root_path: str,
                              container_build_dir: str,
                              debug: bool,
@@ -104,6 +105,9 @@ def process_container(stack: str,
         build_command = os.path.join(container_build_dir,
                                      "default-build.sh") + f" {default_container_tag} {repo_dir_or_build_dir}"
     if not dry_run:
+        # No PATH at all causes failures with podman.
+        if "PATH" not in container_build_env:
+            container_build_env["PATH"] = os.environ["PATH"]
         if verbose:
             print(f"Executing: {build_command} with environment: {container_build_env}")
         build_result = subprocess.run(build_command, shell=True, env=container_build_env)
@@ -119,6 +123,7 @@ def process_container(stack: str,
     else:
         print("Skipped")
 
+
 @click.command()
 @click.option('--include', help="only build these containers")
 @click.option('--exclude', help="don\'t build these containers")

stack_orchestrator/build/build_webapp.py

@@ -25,10 +25,11 @@ from decouple import config
 import click
 from pathlib import Path
 from stack_orchestrator.build import build_containers
+from stack_orchestrator.deploy.webapp.util import determine_base_container
 
 
 @click.command()
-@click.option('--base-container', default="cerc/nextjs-base")
+@click.option('--base-container')
 @click.option('--source-repo', help="directory containing the webapp to build", required=True)
 @click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild")
 @click.option("--extra-build-args", help="Supply extra arguments to build")
@@ -57,6 +58,9 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
     if not quiet:
         print(f'Dev Root is: {dev_root_path}')
 
+    if not base_container:
+        base_container = determine_base_container(source_repo)
+
     # First build the base container.
     container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug,
                                                                     force_rebuild, extra_build_args)
@@ -64,13 +68,12 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
     build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet,
                                        verbose, dry_run, continue_on_error)
 
-
     # Now build the target webapp.  We use the same build script, but with a different Dockerfile and work dir.
     container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true"
     container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"] = os.path.abspath(source_repo)
     container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = os.path.join(container_build_dir,
-                                                                    base_container.replace("/", "-"),
-                                                                    "Dockerfile.webapp")
+                                                                          base_container.replace("/", "-"),
+                                                                          "Dockerfile.webapp")
     if not tag:
         webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1]
         container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local"

stack_orchestrator/data/compose/docker-compose-grafana.yml

@@ -2,8 +2,10 @@ version: "3.7"
 
 services:
   grafana:
-    image: grafana/grafana
+    image: grafana/grafana:10.2.2
     restart: always
+    environment:
+      GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL}
     volumes:
       - ../config/monitoring/grafana/provisioning:/etc/grafana/provisioning
       - ../config/monitoring/grafana/dashboards:/etc/grafana/dashboards

stack_orchestrator/data/compose/docker-compose-node-exporter.yml

@@ -0,0 +1,15 @@
+version: '3.8'
+
+services:
+  node-exporter:
+    image: prom/node-exporter:latest
+    restart: unless-stopped
+    command:
+      - '--web.listen-address=:9100'
+      - '--path.rootfs=/host'
+      - '--collector.systemd'
+      - '--collector.processes'
+    network_mode: host
+    pid: host
+    volumes:
+      - '/:/host:ro,rslave'

stack_orchestrator/data/compose/docker-compose-osmosis-front-end-urbit.yml

@@ -6,12 +6,14 @@ services:
     restart: on-failure
     environment:
       - NEXT_PUBLIC_WEB_API_BASE_URL=${CERC_WEB_API_BASE_URL}
+      - ASSET_LIST_COMMIT_HASH=a326bcefc51372b4912be5a2a2fa84a5d142a438
       - NEXT_PUBLIC_BASEPATH=/apps/osmosis
       - NEXT_PUBLIC_URBIT_DEPLOYMENT=true
     working_dir: /app/packages/web
     command: ["./build-app-for-urbit.sh"]
     volumes:
       - ../config/osmosis/build-app-for-urbit.sh:/app/packages/web/build-app-for-urbit.sh
+      - ../config/osmosis/.env.production:/app/packages/web/.env.production
       - urbit_app_builds:/app-builds
       - ../config/osmosis/urbit-files/mar:/app/packages/web/mar
       - ../config/osmosis/urbit-files/desk.docket-0:/app/packages/web/desk.docket-0

stack_orchestrator/data/compose/docker-compose-osmosis-front-end.yml

@@ -6,10 +6,12 @@ services:
     restart: on-failure
     environment:
       - NEXT_PUBLIC_WEB_API_BASE_URL=${CERC_WEB_API_BASE_URL}
+      - ASSET_LIST_COMMIT_HASH=a326bcefc51372b4912be5a2a2fa84a5d142a438
     working_dir: /app/packages/web
     command: ["./build-app.sh"]
     volumes:
       - ../config/osmosis/build-app.sh:/app/packages/web/build-app.sh
+      - ../config/osmosis/.env.production:/app/packages/web/.env.production
       - app_builds:/app-builds
 
   nginx:

stack_orchestrator/data/compose/docker-compose-prom-server.yml

@@ -2,7 +2,7 @@ version: "3.7"
 
 services:
   prometheus:
-    image: prom/prometheus
+    image: prom/prometheus:v2.49.1
     restart: always
     volumes:
       - ../config/monitoring/prometheus:/etc/prometheus
@@ -18,5 +18,40 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
+  blackbox:
+    image: prom/blackbox-exporter:latest
+    restart: always
+    volumes:
+      - ../config/monitoring/blackbox.yml:/etc/blackbox_exporter/config.yml
+    ports:
+      - '9115'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  chain-head-exporter:
+    image: cerc/watcher-ts:local
+    restart: always
+    working_dir: /app/packages/cli
+    environment:
+      ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      FIL_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT}
+      ETH_RPC_API_KEY: ${CERC_INFURA_KEY}
+      PORT: ${CERC_METRICS_PORT}
+    command: ["sh", "-c", "yarn export-metrics:chain-heads"]
+    ports:
+      - '5000'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  postgres-exporter:
+    image: quay.io/prometheuscommunity/postgres-exporter
+    restart: always
+    volumes:
+      - ../config/monitoring/postgres-exporter.yml:/postgres_exporter.yml
+    ports:
+      - '9187'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
 volumes:
   prometheus_data:

stack_orchestrator/data/config/monitoring/blackbox.yml

@@ -0,0 +1,7 @@
+modules:
+  http_2xx:
+    prober: http
+    timeout: 5s
+    http:
+      valid_status_codes: [] #default to 2xx
+      method: GET

stack_orchestrator/data/config/monitoring/grafana/dashboards/nodejs-app-dashboard.json

@@ -0,0 +1,943 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "node.js prometheus client basic metrics",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "gnetId": 11159,
+  "graphTooltip": 0,
+  "id": 15,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 10,
+        "x": 0,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 6,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "irate(process_cpu_user_seconds_total{instance=~\"$instance\"}[2m]) * 100",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "User CPU - {{instance}}",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "irate(process_cpu_system_seconds_total{instance=~\"$instance\"}[2m]) * 100",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Sys CPU - {{instance}}",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Process CPU Usage",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "percent",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 9,
+        "x": 10,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 8,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_eventloop_lag_seconds{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "{{instance}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Event Loop Lag",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "s",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "text",
+            "mode": "fixed"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 5,
+        "x": 19,
+        "y": 0
+      },
+      "id": 2,
+      "interval": "",
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "name",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.2.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(nodejs_version_info{instance=~\"$instance\"}) by (version)",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{version}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Node.js Version",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "#F2495C",
+            "mode": "fixed"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 5,
+        "x": 19,
+        "y": 3
+      },
+      "id": 4,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.2.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(changes(process_start_time_seconds{instance=~\"$instance\"}[1m]))",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "{{instance}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Process Restart Times",
+      "type": "stat"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 16,
+        "x": 0,
+        "y": 7
+      },
+      "hiddenSeries": false,
+      "id": 7,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "rightSide": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "process_resident_memory_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Process Memory - {{instance}}",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_heap_size_total_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Heap Total - {{instance}}",
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_heap_size_used_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Heap Used - {{instance}}",
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_external_memory_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "External Memory - {{instance}}",
+          "refId": "D"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Process Memory Usage",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 8,
+        "x": 16,
+        "y": 7
+      },
+      "hiddenSeries": false,
+      "id": 9,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_active_handles_total{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Active Handler - {{instance}}",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_active_requests_total{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Active Request - {{instance}}",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Active Handlers/Requests Total",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 14
+      },
+      "hiddenSeries": false,
+      "id": 10,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_heap_space_size_total_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Heap Total - {{instance}} - {{space}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Heap Total Detail",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 8,
+        "y": 14
+      },
+      "hiddenSeries": false,
+      "id": 11,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_heap_space_size_used_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Heap Used - {{instance}} - {{space}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Heap Used Detail",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 16,
+        "y": 14
+      },
+      "hiddenSeries": false,
+      "id": 12,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_heap_space_size_available_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Heap Used - {{instance}} - {{space}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Heap Available Detail",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    }
+  ],
+  "refresh": "10s",
+  "schemaVersion": 38,
+  "tags": [
+    "nodejs"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": true,
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
+        "definition": "label_values(nodejs_version_info, instance)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "instance",
+        "multi": true,
+        "name": "instance",
+        "options": [],
+        "query": "label_values(nodejs_version_info, instance)",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-15m",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "",
+  "title": "NodeJS Application Dashboard",
+  "uid": "PTSqcpJWk",
+  "version": 3,
+  "weekStart": ""
+}

stack_orchestrator/data/config/monitoring/grafana/provisioning/alerting/contactpoints.yml

@@ -0,0 +1,14 @@
+# https://www.clever-cloud.com/blog/features/2021/12/03/slack-alerts-for-grafana/
+
+apiVersion: 1
+
+contactPoints:
+  - orgId: 1
+    name: SlackNotifier
+    receivers:
+      - uid: a71b06e3-58b6-41fe-af65-fbbb29653951
+        type: slack
+        settings:
+          # Slack hook URL (see https://api.slack.com/messaging/webhooks)
+          url: <YOUR_SLACK_HOOK_URL>
+        disableResolveMessage: false

stack_orchestrator/data/config/monitoring/grafana/provisioning/alerting/policies.yml

@@ -0,0 +1,15 @@
+# https://grafana.com/docs/grafana/latest/alerting/alerting-rules/create-notification-policy/
+
+apiVersion: 1
+
+policies:
+  - orgId: 1
+    receiver: grafana-default-email
+    group_by:
+      - grafana_folder
+      - alertname
+    routes:
+      - receiver: SlackNotifier
+        object_matchers:
+          # Add matchers below
+          # - ['grafana_folder', '=', 'MyAlerts']

stack_orchestrator/data/config/monitoring/grafana/provisioning/dashboards/dashboards.yml

@@ -4,6 +4,7 @@ providers:
   - name: dashboards
     type: file
     updateIntervalSeconds: 10
+    allowUiUpdates: true
     options:
       path: /etc/grafana/dashboards
       foldersFromFilesStructure: true

stack_orchestrator/data/config/monitoring/grafana/watcher-dashboard-template.json

@@ -1,887 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      }
-    ]
-  },
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 0,
-  "links": [],
-  "liveNow": false,
-  "panels": [
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "green",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 0,
-        "y": 0
-      },
-      "id": 1,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", kind=\"latest_indexed\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "__auto",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        }
-      ],
-      "title": "Latest indexed block",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Chain head block number of upstream endpoint",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "orange",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 4,
-        "y": 0
-      },
-      "id": 11,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "title": "Upstream chain head",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Chain head from a public endpoint (etherscan)",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "red",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 8,
-        "y": 0
-      },
-      "id": 12,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "title": "External chain head",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "semi-dark-green",
-            "mode": "thresholds"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "semi-dark-green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 35
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 12,
-        "y": 0
-      },
-      "id": 3,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "last_block_process_duration_seconds{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Last block process duration (s)",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Watcher sync status block numbers",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "blue",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 12,
-        "x": 0,
-        "y": 3
-      },
-      "id": 13,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "last_processed_block_number{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "latest_processed",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", kind=\"latest_canonical\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{kind}}",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", kind=\"intial_indexed\"}",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "{{kind}}",
-          "range": true,
-          "refId": "C"
-        }
-      ],
-      "title": "Sync status",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "orange",
-            "mode": "shades"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 5,
-        "w": 12,
-        "x": 12,
-        "y": 6
-      },
-      "id": 6,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "last_block_num_events_total{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Number of events in the last block",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of entries in the block_progress and event table",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "dark-yellow",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "orange",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 0,
-        "y": 7
-      },
-      "id": 7,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "block_progress_total{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "block_progress",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "event_total{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "event",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        }
-      ],
-      "title": "DB indexed count ",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "purple",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "yellow",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 6,
-        "y": 7
-      },
-      "id": 4,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "database_size_bytes{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", type=\"watcher\"} / 1024 / 1024",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{type}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "editorMode": "code",
-          "expr": "database_size_bytes{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", type=\"job-queue\"} / 1024 / 1024",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "{{type}}",
-          "range": true,
-          "refId": "B"
-        }
-      ],
-      "title": "DB size (MB)",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of pgboss jobs in the db",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 0,
-        "y": 11
-      },
-      "id": 5,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "code",
-          "expr": "pgboss_jobs_total{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{state}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Num block-processing jobs",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of pgboss jobs in the db",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 12,
-        "y": 11
-      },
-      "id": 15,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "code",
-          "expr": "pgboss_jobs_total{job=\"REPLACE_WITH_WATCHER_JOB\", instance=\"REPLACE_WITH_WATCHER_NAME\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{state}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Num event-processing jobs",
-      "type": "timeseries"
-    }
-  ],
-  "refresh": "5s",
-  "schemaVersion": 38,
-  "tags": [
-    "REPLACE_WITH_WATCHER_CHAIN",
-    "REPLACE_WITH_WATCHER_JOB"
-  ],
-  "templating": {
-    "list": []
-  },
-  "time": {
-    "from": "now-15m",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "",
-  "title": "REPLACE_WITH_WATCHER_NAME watcher",
-  "version": 3,
-  "weekStart": ""
-}

stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/azimuth-watcher-dashboard.json

@@ -1,887 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      }
-    ]
-  },
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 0,
-  "links": [],
-  "liveNow": false,
-  "panels": [
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "green",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 0,
-        "y": 0
-      },
-      "id": 1,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"azimuth\", instance=\"azimuth\", kind=\"latest_indexed\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "__auto",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        }
-      ],
-      "title": "Latest indexed block",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Chain head block number of upstream endpoint",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "orange",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 4,
-        "y": 0
-      },
-      "id": 11,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "title": "Upstream chain head",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Chain head from a public endpoint (etherscan)",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "red",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 8,
-        "y": 0
-      },
-      "id": 12,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "title": "External chain head",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "semi-dark-green",
-            "mode": "thresholds"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "semi-dark-green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 35
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 12,
-        "y": 0
-      },
-      "id": 3,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "last_block_process_duration_seconds{job=\"azimuth\", instance=\"azimuth\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Last block process duration (s)",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Watcher sync status block numbers",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "blue",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 12,
-        "x": 0,
-        "y": 3
-      },
-      "id": 13,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "last_processed_block_number{job=\"azimuth\", instance=\"azimuth\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "latest_processed",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"azimuth\", instance=\"azimuth\", kind=\"latest_canonical\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{kind}}",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"azimuth\", instance=\"azimuth\", kind=\"intial_indexed\"}",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "{{kind}}",
-          "range": true,
-          "refId": "C"
-        }
-      ],
-      "title": "Sync status",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "orange",
-            "mode": "shades"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 5,
-        "w": 12,
-        "x": 12,
-        "y": 6
-      },
-      "id": 6,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "last_block_num_events_total{job=\"azimuth\", instance=\"azimuth\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Number of events in the last block",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of entries in the block_progress and event table",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "dark-yellow",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "orange",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 0,
-        "y": 7
-      },
-      "id": 7,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "block_progress_total{job=\"azimuth\", instance=\"azimuth\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "block_progress",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "event_total{job=\"azimuth\", instance=\"azimuth\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "event",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        }
-      ],
-      "title": "DB indexed count ",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "purple",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "yellow",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 6,
-        "y": 7
-      },
-      "id": 4,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "database_size_bytes{job=\"azimuth\", instance=\"azimuth\", type=\"watcher\"} / 1024 / 1024",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{type}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "editorMode": "code",
-          "expr": "database_size_bytes{job=\"azimuth\", instance=\"azimuth\", type=\"job-queue\"} / 1024 / 1024",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "{{type}}",
-          "range": true,
-          "refId": "B"
-        }
-      ],
-      "title": "DB size (MB)",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of pgboss jobs in the db",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 0,
-        "y": 11
-      },
-      "id": 5,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "code",
-          "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"azimuth\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{state}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Num block-processing jobs",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of pgboss jobs in the db",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 12,
-        "y": 11
-      },
-      "id": 15,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "code",
-          "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"azimuth\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{state}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Num event-processing jobs",
-      "type": "timeseries"
-    }
-  ],
-  "refresh": "5s",
-  "schemaVersion": 38,
-  "tags": [
-    "ethereum",
-    "azimuth"
-  ],
-  "templating": {
-    "list": []
-  },
-  "time": {
-    "from": "now-15m",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "",
-  "title": "azimuth watcher",
-  "version": 3,
-  "weekStart": ""
-}

stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/censures-watcher-dashboard.json

@@ -1,887 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      }
-    ]
-  },
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 0,
-  "links": [],
-  "liveNow": false,
-  "panels": [
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "green",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 0,
-        "y": 0
-      },
-      "id": 1,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"azimuth\", instance=\"censures\", kind=\"latest_indexed\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "__auto",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        }
-      ],
-      "title": "Latest indexed block",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Chain head block number of upstream endpoint",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "orange",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 4,
-        "y": 0
-      },
-      "id": 11,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "title": "Upstream chain head",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Chain head from a public endpoint (etherscan)",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "red",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 8,
-        "y": 0
-      },
-      "id": 12,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "title": "External chain head",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "semi-dark-green",
-            "mode": "thresholds"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "semi-dark-green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 35
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 12,
-        "y": 0
-      },
-      "id": 3,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "last_block_process_duration_seconds{job=\"azimuth\", instance=\"censures\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Last block process duration (s)",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Watcher sync status block numbers",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "blue",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 12,
-        "x": 0,
-        "y": 3
-      },
-      "id": 13,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "last_processed_block_number{job=\"azimuth\", instance=\"censures\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "latest_processed",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"azimuth\", instance=\"censures\", kind=\"latest_canonical\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{kind}}",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"azimuth\", instance=\"censures\", kind=\"intial_indexed\"}",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "{{kind}}",
-          "range": true,
-          "refId": "C"
-        }
-      ],
-      "title": "Sync status",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "orange",
-            "mode": "shades"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 5,
-        "w": 12,
-        "x": 12,
-        "y": 6
-      },
-      "id": 6,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "last_block_num_events_total{job=\"azimuth\", instance=\"censures\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Number of events in the last block",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of entries in the block_progress and event table",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "dark-yellow",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "orange",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 0,
-        "y": 7
-      },
-      "id": 7,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "block_progress_total{job=\"azimuth\", instance=\"censures\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "block_progress",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "event_total{job=\"azimuth\", instance=\"censures\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "event",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        }
-      ],
-      "title": "DB indexed count ",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "purple",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "yellow",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 6,
-        "y": 7
-      },
-      "id": 4,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "database_size_bytes{job=\"azimuth\", instance=\"censures\", type=\"watcher\"} / 1024 / 1024",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{type}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "editorMode": "code",
-          "expr": "database_size_bytes{job=\"azimuth\", instance=\"censures\", type=\"job-queue\"} / 1024 / 1024",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "{{type}}",
-          "range": true,
-          "refId": "B"
-        }
-      ],
-      "title": "DB size (MB)",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of pgboss jobs in the db",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 0,
-        "y": 11
-      },
-      "id": 5,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "code",
-          "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"censures\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{state}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Num block-processing jobs",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of pgboss jobs in the db",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 12,
-        "y": 11
-      },
-      "id": 15,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "code",
-          "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"censures\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{state}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Num event-processing jobs",
-      "type": "timeseries"
-    }
-  ],
-  "refresh": "5s",
-  "schemaVersion": 38,
-  "tags": [
-    "ethereum",
-    "azimuth"
-  ],
-  "templating": {
-    "list": []
-  },
-  "time": {
-    "from": "now-15m",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "",
-  "title": "censures watcher",
-  "version": 3,
-  "weekStart": ""
-}

stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/claims-watcher-dashboard.json

@@ -1,887 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      }
-    ]
-  },
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 0,
-  "links": [],
-  "liveNow": false,
-  "panels": [
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "green",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 0,
-        "y": 0
-      },
-      "id": 1,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"azimuth\", instance=\"claims\", kind=\"latest_indexed\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "__auto",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        }
-      ],
-      "title": "Latest indexed block",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Chain head block number of upstream endpoint",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "orange",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 4,
-        "y": 0
-      },
-      "id": 11,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "title": "Upstream chain head",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Chain head from a public endpoint (etherscan)",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "red",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 8,
-        "y": 0
-      },
-      "id": 12,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "title": "External chain head",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "semi-dark-green",
-            "mode": "thresholds"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "semi-dark-green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 35
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 12,
-        "y": 0
-      },
-      "id": 3,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "last_block_process_duration_seconds{job=\"azimuth\", instance=\"claims\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Last block process duration (s)",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Watcher sync status block numbers",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "blue",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 12,
-        "x": 0,
-        "y": 3
-      },
-      "id": 13,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "last_processed_block_number{job=\"azimuth\", instance=\"claims\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "latest_processed",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"azimuth\", instance=\"claims\", kind=\"latest_canonical\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{kind}}",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"azimuth\", instance=\"claims\", kind=\"intial_indexed\"}",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "{{kind}}",
-          "range": true,
-          "refId": "C"
-        }
-      ],
-      "title": "Sync status",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "orange",
-            "mode": "shades"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 5,
-        "w": 12,
-        "x": 12,
-        "y": 6
-      },
-      "id": 6,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "last_block_num_events_total{job=\"azimuth\", instance=\"claims\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Number of events in the last block",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of entries in the block_progress and event table",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "dark-yellow",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "orange",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 0,
-        "y": 7
-      },
-      "id": 7,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "block_progress_total{job=\"azimuth\", instance=\"claims\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "block_progress",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "event_total{job=\"azimuth\", instance=\"claims\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "event",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        }
-      ],
-      "title": "DB indexed count ",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "purple",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "yellow",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 6,
-        "y": 7
-      },
-      "id": 4,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "database_size_bytes{job=\"azimuth\", instance=\"claims\", type=\"watcher\"} / 1024 / 1024",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{type}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "editorMode": "code",
-          "expr": "database_size_bytes{job=\"azimuth\", instance=\"claims\", type=\"job-queue\"} / 1024 / 1024",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "{{type}}",
-          "range": true,
-          "refId": "B"
-        }
-      ],
-      "title": "DB size (MB)",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of pgboss jobs in the db",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 0,
-        "y": 11
-      },
-      "id": 5,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "code",
-          "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"claims\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{state}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Num block-processing jobs",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of pgboss jobs in the db",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 12,
-        "y": 11
-      },
-      "id": 15,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "code",
-          "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"claims\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{state}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Num event-processing jobs",
-      "type": "timeseries"
-    }
-  ],
-  "refresh": "5s",
-  "schemaVersion": 38,
-  "tags": [
-    "ethereum",
-    "azimuth"
-  ],
-  "templating": {
-    "list": []
-  },
-  "time": {
-    "from": "now-15m",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "",
-  "title": "claims watcher",
-  "version": 3,
-  "weekStart": ""
-}

stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/conditional-star-release-watcher-dashboard.json

@@ -1,887 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      }
-    ]
-  },
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 0,
-  "links": [],
-  "liveNow": false,
-  "panels": [
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "green",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 0,
-        "y": 0
-      },
-      "id": 1,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"azimuth\", instance=\"conditional_star_release\", kind=\"latest_indexed\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "__auto",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        }
-      ],
-      "title": "Latest indexed block",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Chain head block number of upstream endpoint",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "orange",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 4,
-        "y": 0
-      },
-      "id": 11,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "title": "Upstream chain head",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Chain head from a public endpoint (etherscan)",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "red",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 8,
-        "y": 0
-      },
-      "id": 12,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "title": "External chain head",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "semi-dark-green",
-            "mode": "thresholds"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "semi-dark-green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 35
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 12,
-        "y": 0
-      },
-      "id": 3,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "last_block_process_duration_seconds{job=\"azimuth\", instance=\"conditional_star_release\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Last block process duration (s)",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Watcher sync status block numbers",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "blue",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 12,
-        "x": 0,
-        "y": 3
-      },
-      "id": 13,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "last_processed_block_number{job=\"azimuth\", instance=\"conditional_star_release\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "latest_processed",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"azimuth\", instance=\"conditional_star_release\", kind=\"latest_canonical\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{kind}}",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"azimuth\", instance=\"conditional_star_release\", kind=\"intial_indexed\"}",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "{{kind}}",
-          "range": true,
-          "refId": "C"
-        }
-      ],
-      "title": "Sync status",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "orange",
-            "mode": "shades"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 5,
-        "w": 12,
-        "x": 12,
-        "y": 6
-      },
-      "id": 6,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "last_block_num_events_total{job=\"azimuth\", instance=\"conditional_star_release\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Number of events in the last block",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of entries in the block_progress and event table",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "dark-yellow",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "orange",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 0,
-        "y": 7
-      },
-      "id": 7,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "block_progress_total{job=\"azimuth\", instance=\"conditional_star_release\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "block_progress",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "event_total{job=\"azimuth\", instance=\"conditional_star_release\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "event",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        }
-      ],
-      "title": "DB indexed count ",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "purple",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "yellow",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 6,
-        "y": 7
-      },
-      "id": 4,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "database_size_bytes{job=\"azimuth\", instance=\"conditional_star_release\", type=\"watcher\"} / 1024 / 1024",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{type}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "editorMode": "code",
-          "expr": "database_size_bytes{job=\"azimuth\", instance=\"conditional_star_release\", type=\"job-queue\"} / 1024 / 1024",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "{{type}}",
-          "range": true,
-          "refId": "B"
-        }
-      ],
-      "title": "DB size (MB)",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of pgboss jobs in the db",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 0,
-        "y": 11
-      },
-      "id": 5,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "code",
-          "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"conditional_star_release\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{state}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Num block-processing jobs",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of pgboss jobs in the db",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 12,
-        "y": 11
-      },
-      "id": 15,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "code",
-          "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"conditional_star_release\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{state}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Num event-processing jobs",
-      "type": "timeseries"
-    }
-  ],
-  "refresh": "5s",
-  "schemaVersion": 38,
-  "tags": [
-    "ethereum",
-    "azimuth"
-  ],
-  "templating": {
-    "list": []
-  },
-  "time": {
-    "from": "now-15m",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "",
-  "title": "conditional star release watcher",
-  "version": 3,
-  "weekStart": ""
-}

stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/delegated-sending-watcher-watcher-dashboard.json

@@ -1,887 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      }
-    ]
-  },
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 0,
-  "links": [],
-  "liveNow": false,
-  "panels": [
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "green",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 0,
-        "y": 0
-      },
-      "id": 1,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"azimuth\", instance=\"delegated_sending_watcher\", kind=\"latest_indexed\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "__auto",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        }
-      ],
-      "title": "Latest indexed block",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Chain head block number of upstream endpoint",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "orange",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 4,
-        "y": 0
-      },
-      "id": 11,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "title": "Upstream chain head",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Chain head from a public endpoint (etherscan)",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "red",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 8,
-        "y": 0
-      },
-      "id": 12,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "title": "External chain head",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "semi-dark-green",
-            "mode": "thresholds"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "semi-dark-green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 35
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 12,
-        "y": 0
-      },
-      "id": 3,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "last_block_process_duration_seconds{job=\"azimuth\", instance=\"delegated_sending_watcher\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Last block process duration (s)",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Watcher sync status block numbers",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "blue",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 12,
-        "x": 0,
-        "y": 3
-      },
-      "id": 13,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "last_processed_block_number{job=\"azimuth\", instance=\"delegated_sending_watcher\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "latest_processed",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"azimuth\", instance=\"delegated_sending_watcher\", kind=\"latest_canonical\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{kind}}",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"azimuth\", instance=\"delegated_sending_watcher\", kind=\"intial_indexed\"}",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "{{kind}}",
-          "range": true,
-          "refId": "C"
-        }
-      ],
-      "title": "Sync status",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "orange",
-            "mode": "shades"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 5,
-        "w": 12,
-        "x": 12,
-        "y": 6
-      },
-      "id": 6,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "last_block_num_events_total{job=\"azimuth\", instance=\"delegated_sending_watcher\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Number of events in the last block",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of entries in the block_progress and event table",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "dark-yellow",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "orange",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 0,
-        "y": 7
-      },
-      "id": 7,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "block_progress_total{job=\"azimuth\", instance=\"delegated_sending_watcher\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "block_progress",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "event_total{job=\"azimuth\", instance=\"delegated_sending_watcher\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "event",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        }
-      ],
-      "title": "DB indexed count ",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "purple",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "yellow",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 6,
-        "y": 7
-      },
-      "id": 4,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "database_size_bytes{job=\"azimuth\", instance=\"delegated_sending_watcher\", type=\"watcher\"} / 1024 / 1024",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{type}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "editorMode": "code",
-          "expr": "database_size_bytes{job=\"azimuth\", instance=\"delegated_sending_watcher\", type=\"job-queue\"} / 1024 / 1024",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "{{type}}",
-          "range": true,
-          "refId": "B"
-        }
-      ],
-      "title": "DB size (MB)",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of pgboss jobs in the db",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 0,
-        "y": 11
-      },
-      "id": 5,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "code",
-          "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"delegated_sending_watcher\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{state}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Num block-processing jobs",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of pgboss jobs in the db",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 12,
-        "y": 11
-      },
-      "id": 15,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "code",
-          "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"delegated_sending_watcher\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{state}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Num event-processing jobs",
-      "type": "timeseries"
-    }
-  ],
-  "refresh": "5s",
-  "schemaVersion": 38,
-  "tags": [
-    "ethereum",
-    "azimuth"
-  ],
-  "templating": {
-    "list": []
-  },
-  "time": {
-    "from": "now-15m",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "",
-  "title": "delegated sending watcher watcher",
-  "version": 3,
-  "weekStart": ""
-}

stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/ecliptic-watcher-dashboard.json

@@ -1,887 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      }
-    ]
-  },
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 0,
-  "links": [],
-  "liveNow": false,
-  "panels": [
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "green",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 0,
-        "y": 0
-      },
-      "id": 1,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"azimuth\", instance=\"ecliptic\", kind=\"latest_indexed\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "__auto",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        }
-      ],
-      "title": "Latest indexed block",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Chain head block number of upstream endpoint",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "orange",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 4,
-        "y": 0
-      },
-      "id": 11,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "title": "Upstream chain head",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Chain head from a public endpoint (etherscan)",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "red",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 8,
-        "y": 0
-      },
-      "id": 12,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "title": "External chain head",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "semi-dark-green",
-            "mode": "thresholds"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "semi-dark-green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 35
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 12,
-        "y": 0
-      },
-      "id": 3,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "last_block_process_duration_seconds{job=\"azimuth\", instance=\"ecliptic\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Last block process duration (s)",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Watcher sync status block numbers",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "blue",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 12,
-        "x": 0,
-        "y": 3
-      },
-      "id": 13,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "last_processed_block_number{job=\"azimuth\", instance=\"ecliptic\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "latest_processed",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"azimuth\", instance=\"ecliptic\", kind=\"latest_canonical\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{kind}}",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"azimuth\", instance=\"ecliptic\", kind=\"intial_indexed\"}",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "{{kind}}",
-          "range": true,
-          "refId": "C"
-        }
-      ],
-      "title": "Sync status",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "orange",
-            "mode": "shades"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 5,
-        "w": 12,
-        "x": 12,
-        "y": 6
-      },
-      "id": 6,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "last_block_num_events_total{job=\"azimuth\", instance=\"ecliptic\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Number of events in the last block",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of entries in the block_progress and event table",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "dark-yellow",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "orange",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 0,
-        "y": 7
-      },
-      "id": 7,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "block_progress_total{job=\"azimuth\", instance=\"ecliptic\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "block_progress",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "event_total{job=\"azimuth\", instance=\"ecliptic\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "event",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        }
-      ],
-      "title": "DB indexed count ",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "purple",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "yellow",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 6,
-        "y": 7
-      },
-      "id": 4,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "database_size_bytes{job=\"azimuth\", instance=\"ecliptic\", type=\"watcher\"} / 1024 / 1024",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{type}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "editorMode": "code",
-          "expr": "database_size_bytes{job=\"azimuth\", instance=\"ecliptic\", type=\"job-queue\"} / 1024 / 1024",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "{{type}}",
-          "range": true,
-          "refId": "B"
-        }
-      ],
-      "title": "DB size (MB)",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of pgboss jobs in the db",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 0,
-        "y": 11
-      },
-      "id": 5,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "code",
-          "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"ecliptic\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{state}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Num block-processing jobs",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of pgboss jobs in the db",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 12,
-        "y": 11
-      },
-      "id": 15,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "code",
-          "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"ecliptic\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{state}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Num event-processing jobs",
-      "type": "timeseries"
-    }
-  ],
-  "refresh": "5s",
-  "schemaVersion": 38,
-  "tags": [
-    "ethereum",
-    "azimuth"
-  ],
-  "templating": {
-    "list": []
-  },
-  "time": {
-    "from": "now-15m",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "",
-  "title": "ecliptic watcher",
-  "version": 3,
-  "weekStart": ""
-}

stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/linear-star-release-watcher-dashboard.json

@@ -1,887 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      }
-    ]
-  },
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 0,
-  "links": [],
-  "liveNow": false,
-  "panels": [
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "green",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 0,
-        "y": 0
-      },
-      "id": 1,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"azimuth\", instance=\"linear_star_release\", kind=\"latest_indexed\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "__auto",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        }
-      ],
-      "title": "Latest indexed block",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Chain head block number of upstream endpoint",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "orange",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 4,
-        "y": 0
-      },
-      "id": 11,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "title": "Upstream chain head",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Chain head from a public endpoint (etherscan)",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "red",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 8,
-        "y": 0
-      },
-      "id": 12,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "title": "External chain head",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "semi-dark-green",
-            "mode": "thresholds"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "semi-dark-green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 35
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 12,
-        "y": 0
-      },
-      "id": 3,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "last_block_process_duration_seconds{job=\"azimuth\", instance=\"linear_star_release\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Last block process duration (s)",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Watcher sync status block numbers",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "blue",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 12,
-        "x": 0,
-        "y": 3
-      },
-      "id": 13,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "last_processed_block_number{job=\"azimuth\", instance=\"linear_star_release\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "latest_processed",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"azimuth\", instance=\"linear_star_release\", kind=\"latest_canonical\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{kind}}",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"azimuth\", instance=\"linear_star_release\", kind=\"intial_indexed\"}",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "{{kind}}",
-          "range": true,
-          "refId": "C"
-        }
-      ],
-      "title": "Sync status",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "orange",
-            "mode": "shades"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 5,
-        "w": 12,
-        "x": 12,
-        "y": 6
-      },
-      "id": 6,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "last_block_num_events_total{job=\"azimuth\", instance=\"linear_star_release\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Number of events in the last block",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of entries in the block_progress and event table",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "dark-yellow",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "orange",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 0,
-        "y": 7
-      },
-      "id": 7,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "block_progress_total{job=\"azimuth\", instance=\"linear_star_release\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "block_progress",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "event_total{job=\"azimuth\", instance=\"linear_star_release\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "event",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        }
-      ],
-      "title": "DB indexed count ",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "purple",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "yellow",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 6,
-        "y": 7
-      },
-      "id": 4,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "database_size_bytes{job=\"azimuth\", instance=\"linear_star_release\", type=\"watcher\"} / 1024 / 1024",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{type}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "editorMode": "code",
-          "expr": "database_size_bytes{job=\"azimuth\", instance=\"linear_star_release\", type=\"job-queue\"} / 1024 / 1024",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "{{type}}",
-          "range": true,
-          "refId": "B"
-        }
-      ],
-      "title": "DB size (MB)",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of pgboss jobs in the db",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 0,
-        "y": 11
-      },
-      "id": 5,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "code",
-          "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"linear_star_release\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{state}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Num block-processing jobs",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of pgboss jobs in the db",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 12,
-        "y": 11
-      },
-      "id": 15,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "code",
-          "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"linear_star_release\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{state}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Num event-processing jobs",
-      "type": "timeseries"
-    }
-  ],
-  "refresh": "5s",
-  "schemaVersion": 38,
-  "tags": [
-    "ethereum",
-    "azimuth"
-  ],
-  "templating": {
-    "list": []
-  },
-  "time": {
-    "from": "now-15m",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "",
-  "title": "linear star release watcher",
-  "version": 3,
-  "weekStart": ""
-}

stack_orchestrator/data/config/monitoring/grafana/watcher-dashboards/azimuth/polls-watcher-dashboard.json

@@ -1,887 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      }
-    ]
-  },
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 0,
-  "links": [],
-  "liveNow": false,
-  "panels": [
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "green",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 0,
-        "y": 0
-      },
-      "id": 1,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"azimuth\", instance=\"polls\", kind=\"latest_indexed\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "__auto",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        }
-      ],
-      "title": "Latest indexed block",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Chain head block number of upstream endpoint",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "orange",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 4,
-        "y": 0
-      },
-      "id": 11,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "title": "Upstream chain head",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Chain head from a public endpoint (etherscan)",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "red",
-            "mode": "shades"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 3,
-        "w": 4,
-        "x": 8,
-        "y": 0
-      },
-      "id": 12,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "title": "External chain head",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "semi-dark-green",
-            "mode": "thresholds"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "semi-dark-green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 35
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 12,
-        "y": 0
-      },
-      "id": 3,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "last_block_process_duration_seconds{job=\"azimuth\", instance=\"polls\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Last block process duration (s)",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Watcher sync status block numbers",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "blue",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 12,
-        "x": 0,
-        "y": 3
-      },
-      "id": 13,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "last_processed_block_number{job=\"azimuth\", instance=\"polls\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "latest_processed",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"azimuth\", instance=\"polls\", kind=\"latest_canonical\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{kind}}",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "editorMode": "builder",
-          "expr": "sync_status_block_number{job=\"azimuth\", instance=\"polls\", kind=\"intial_indexed\"}",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "{{kind}}",
-          "range": true,
-          "refId": "C"
-        }
-      ],
-      "title": "Sync status",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "orange",
-            "mode": "shades"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 5,
-        "w": 12,
-        "x": 12,
-        "y": 6
-      },
-      "id": 6,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "last_block_num_events_total{job=\"azimuth\", instance=\"polls\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Number of events in the last block",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of entries in the block_progress and event table",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "dark-yellow",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "orange",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 0,
-        "y": 7
-      },
-      "id": 7,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "block_progress_total{job=\"azimuth\", instance=\"polls\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "block_progress",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "event_total{job=\"azimuth\", instance=\"polls\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "event",
-          "range": true,
-          "refId": "B",
-          "useBackend": false
-        }
-      ],
-      "title": "DB indexed count ",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "fixedColor": "purple",
-            "mode": "fixed"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "yellow",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 6,
-        "y": 7
-      },
-      "id": 4,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto",
-        "wideLayout": true
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "database_size_bytes{job=\"azimuth\", instance=\"polls\", type=\"watcher\"} / 1024 / 1024",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{type}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "editorMode": "code",
-          "expr": "database_size_bytes{job=\"azimuth\", instance=\"polls\", type=\"job-queue\"} / 1024 / 1024",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "{{type}}",
-          "range": true,
-          "refId": "B"
-        }
-      ],
-      "title": "DB size (MB)",
-      "type": "stat"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of pgboss jobs in the db",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 0,
-        "y": 11
-      },
-      "id": 5,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "code",
-          "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"polls\", state=~\"active|failed|cancelled|expired\", name=\"block-processing\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{state}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Num block-processing jobs",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "description": "Total number of pgboss jobs in the db",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 6,
-        "w": 12,
-        "x": 12,
-        "y": 11
-      },
-      "id": 15,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.2.2",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "disableTextWrap": false,
-          "editorMode": "code",
-          "expr": "pgboss_jobs_total{job=\"azimuth\", instance=\"polls\", state=~\"active|failed|cancelled|expired\", name=\"event-processing\"}",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{state}}",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Num event-processing jobs",
-      "type": "timeseries"
-    }
-  ],
-  "refresh": "5s",
-  "schemaVersion": 38,
-  "tags": [
-    "ethereum",
-    "azimuth"
-  ],
-  "templating": {
-    "list": []
-  },
-  "time": {
-    "from": "now-15m",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "",
-  "title": "polls watcher",
-  "version": 3,
-  "weekStart": ""
-}

stack_orchestrator/data/config/monitoring/postgres-exporter.yml

@@ -0,0 +1,8 @@
+auth_modules:
+  foo:
+    type: userpass
+    userpass:
+      username: username
+      password: password
+    options:
+      sslmode: disable

stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml

@@ -1,5 +1,5 @@
 global:
-  scrape_interval:     15s
+  scrape_interval:     10s
   evaluation_interval: 15s
 
 rule_files:
@@ -10,3 +10,58 @@ scrape_configs:
   - job_name: prometheus
     static_configs:
       - targets: ['localhost:9090']
+
+  - job_name: node
+    static_configs:
+      # Add node-exporter targets to be monitored below
+      # - targets: ['example-host:9100']
+      #   labels:
+      #     instance: 'my-host'
+
+  - job_name: 'blackbox'
+    scrape_interval: 10s
+    metrics_path: /probe
+    params:
+      module: [http_2xx]
+    static_configs:
+      # Add URLs to be monitored below
+      - targets:
+        # - https://github.com
+    relabel_configs:
+      - source_labels: [__address__]
+        regex: (.*)(:80)?
+        target_label: __param_target
+      - source_labels: [__param_target]
+        regex: (.*)
+        target_label: instance
+        replacement: ${1}
+      - source_labels: []
+        regex: .*
+        target_label: __address__
+        replacement: blackbox:9115
+
+  - job_name: chain_heads
+    scrape_interval: 10s
+    metrics_path: /metrics
+    scheme: http
+    static_configs:
+      - targets: ['chain-head-exporter:5000']
+
+  - job_name: 'postgres'
+    scrape_interval: 30s
+    scrape_timeout: 30s
+    static_configs:
+      # Add DB targets below
+      # - targets: [example-server:5432]
+        # labels:
+          # instance: 'example-label'
+    metrics_path: /probe
+    params:
+      auth_module: [foo]
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: postgres-exporter:9187

stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml

@@ -0,0 +1,933 @@
+# https://grafana.com/docs/grafana/latest/alerting/alerting-rules/create-grafana-managed-rule/
+
+apiVersion: 1
+
+groups:
+  - orgId: 1
+    name: watcher
+    folder: WatcherAlerts
+    interval: 30s
+    rules:
+      # Azimuth
+      - uid: azimuth_diff_external
+        title: azimuth_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: censures_diff_external
+        title: censures_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: claims_diff_external
+        title: claims_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: conditional_star_release_diff_external
+        title: conditional_star_release_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: delegated_sending_diff_external
+        title: delegated_sending_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: ecliptic_diff_external
+        title: ecliptic_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: linear_star_release_diff_external
+        title: linear_star_release_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="linear_star_release", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: polls_diff_external
+        title: polls_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+
+      # Sushi
+      - uid: sushiswap_diff_external
+        title: sushiswap_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: merkl_sushiswap_diff_external
+        title: merkl_sushiswap_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false

stack_orchestrator/data/config/osmosis/.env.production

@@ -0,0 +1,3 @@
+NEXT_PUBLIC_COINGECKO_URL=https://api.coingecko.com
+
+NEXT_PUBLIC_ENABLE_FEATURES=true

stack_orchestrator/data/config/osmosis/build-app-for-urbit.sh

@@ -12,7 +12,7 @@ if [ -d /app-builds/osmosis/build ]; then
 fi
 
 yarn build:static
-./build.sh
+./build-urbit.sh
 
 # Move build to app-builds
 mkdir -p /app-builds/osmosis

stack_orchestrator/data/config/osmosis/urbit-files/desk.docket-0

@@ -1,10 +1,10 @@
 :~  title+'Osmosis'
-    info+'Self-hosted osmosis frontend.'
+    info+'Osmosis DEX Frontend, built and maintained by Laconic'
     color+0xcd.75df
-    image+'https://logowik.com/content/uploads/images/osmosis8548.logowik.com.webp'
+    image+'https://altcoinsbox.com/wp-content/uploads/2023/03/osmosis-logo.png'
     base+'osmosis'
     glob-http+['REPLACE_WITH_GLOB_URL' REPLACE_WITH_GLOB_HASH]
-    version+[0 0 1]
+    version+[0 0 2]
     website+'https://osmosis.zone/'
     license+'MIT'
 ==

stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml

@@ -84,8 +84,6 @@
   subgraphEventsOrder = true
   # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
   blockDelayInMilliSecs = 30000
-  prefetchBlocksInMem = false
-  prefetchBlockCount = 10
 
   # Boolean to switch between modes of processing events when starting the server.
   # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.

stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml

@@ -84,8 +84,6 @@
   subgraphEventsOrder = true
   # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
   blockDelayInMilliSecs = 30000
-  prefetchBlocksInMem = false
-  prefetchBlockCount = 10
 
   # Boolean to switch between modes of processing events when starting the server.
   # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.

stack_orchestrator/data/container-build/cerc-act-runner-task-executor/build.sh

@@ -2,4 +2,6 @@
 # Build a local version of the task executor for act-runner
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-docker build -t cerc/act-runner-task-executor:local -f ${CERC_REPO_BASE_DIR}/hosting/act-runner/Dockerfile.task-executor ${build_command_args} ${SCRIPT_DIR}
+
+cd ${CERC_REPO_BASE_DIR}/hosting/act-runner
+docker build -t cerc/act-runner-task-executor:local -f Dockerfile.task-executor ${build_command_args} .

stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/Dockerfile

@@ -10,9 +10,10 @@ COPY genesis /opt/genesis
 COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/
 COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/
 COPY --from=ethgen /apps /apps
-RUN cd /apps/el-gen && pip3 install -r requirements.txt
+RUN cd /apps/el-gen && pip3 install --break-system-packages -r requirements.txt
 # web3==5.24.0 used by el-gen is broken on python 3.11
-RUN pip3 install --upgrade "web3==6.5.0"
+RUN pip3 install --break-system-packages --upgrade "web3==6.5.0"
+RUN pip3 install --break-system-packages --upgrade "typing-extensions"
 
 # Build genesis config
 RUN apk add --no-cache make bash envsubst jq

stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile

@@ -30,13 +30,13 @@ RUN \
 
 # [Optional] Uncomment this section to install additional OS packages.
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends jq gettext-base
+    && apt-get -y install --no-install-recommends jq gettext-base procps
 
 # [Optional] Uncomment if you want to install more global node modules
 # RUN su node -c "npm install -g <your-package-list-here>"
 
 # Expose port for http
-EXPOSE 3000
+EXPOSE 80
 
 COPY /scripts /scripts
 

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh

@@ -21,6 +21,11 @@ WORK_DIR="${1:-/app}"
 
 cd "${WORK_DIR}" || exit 1
 
+# If this file doesn't exist at all, we'll get errors below.
+if [ ! -f "next.config.js" ]; then
+  touch next.config.js
+fi
+
 if [ ! -f "next.config.dist" ]; then
   cp next.config.js next.config.dist
 fi

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh

@@ -58,4 +58,4 @@ if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then
   fi
 fi
 
-$CERC_BUILD_TOOL start . -p ${CERC_LISTEN_PORT:-3000}
+$CERC_BUILD_TOOL start . -- -p ${CERC_LISTEN_PORT:-80}

stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile

@@ -1,6 +1,6 @@
 # Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile
 # [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
-ARG VARIANT=18-bullseye
+ARG VARIANT=20-bullseye
 FROM node:${VARIANT}
 
 ARG USERNAME=node
@@ -28,7 +28,7 @@ RUN \
 
 # [Optional] Uncomment this section to install additional OS packages.
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends jq
+    && apt-get -y install --no-install-recommends jq gettext-base
 
 # [Optional] Uncomment if you want to install an additional version of node using nvm
 # ARG EXTRA_NODE_VERSION=10
@@ -37,9 +37,7 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
 # We do this to get a yq binary from the published container, for the correct architecture we're building here
 COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq
 
-RUN mkdir -p /scripts
-COPY ./apply-webapp-config.sh /scripts
-COPY ./start-serving-app.sh /scripts
+COPY scripts /scripts
 
 # [Optional] Uncomment if you want to install more global node modules
 # RUN su node -c "npm install -g <your-package-list-here>"

stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile.webapp

@@ -0,0 +1,11 @@
+FROM cerc/webapp-base:local as builder
+
+ARG CERC_BUILD_TOOL
+
+WORKDIR /app
+COPY . .
+RUN rm -rf node_modules build .next*
+RUN /scripts/build-app.sh /app build /data
+
+FROM cerc/webapp-base:local
+COPY --from=builder /data /data

stack_orchestrator/data/container-build/cerc-webapp-base/build.sh

@@ -1,9 +1,29 @@
 #!/usr/bin/env bash
-# Build cerc/laconic-registry-cli
+# Build cerc/webapp-base
 
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 
 # See: https://stackoverflow.com/a/246128/1701505
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-docker build -t cerc/webapp-base:local ${build_command_args} -f ${SCRIPT_DIR}/Dockerfile ${SCRIPT_DIR}
+CERC_CONTAINER_BUILD_WORK_DIR=${CERC_CONTAINER_BUILD_WORK_DIR:-$SCRIPT_DIR}
+CERC_CONTAINER_BUILD_DOCKERFILE=${CERC_CONTAINER_BUILD_DOCKERFILE:-$SCRIPT_DIR/Dockerfile}
+CERC_CONTAINER_BUILD_TAG=${CERC_CONTAINER_BUILD_TAG:-cerc/webapp-base:local}
+
+docker build -t $CERC_CONTAINER_BUILD_TAG ${build_command_args} -f $CERC_CONTAINER_BUILD_DOCKERFILE $CERC_CONTAINER_BUILD_WORK_DIR
+
+if [ $? -eq 0 ] && [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/webapp-base:local" ]; then
+  cat <<EOF
+
+#################################################################
+
+Built host container for $CERC_CONTAINER_BUILD_WORK_DIR with tag:
+
+    $CERC_CONTAINER_BUILD_TAG
+
+To test locally run:
+
+    laconic-so run-webapp --image $CERC_CONTAINER_BUILD_TAG --env-file /path/to/environment.env
+
+EOF
+fi

stack_orchestrator/data/container-build/cerc-webapp-base/scripts/apply-runtime-env.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+    set -x
+fi
+
+WORK_DIR="${1:-./}"
+
+cd "${WORK_DIR}" || exit 1
+
+if [ -f ".env" ]; then
+  TMP_ENV=`mktemp`
+  declare -px > $TMP_ENV
+  set -a
+  source .env
+  source $TMP_ENV
+  set +a
+  rm -f $TMP_ENV
+fi
+
+for f in $(find . -regex ".*.[tj]sx?$" -type f | grep -v 'node_modules'); do
+  for e in $(cat "${f}" | tr -s '[:blank:]' '\n' | tr -s '[{},();]' '\n' | egrep -o -e '^"CERC_RUNTIME_ENV_[^\"]+"' -e '^"LACONIC_HOSTED_CONFIG_[^\"]+"'); do
+    orig_name=$(echo -n "${e}" | sed 's/"//g')
+    cur_name=$(echo -n "${orig_name}" | sed 's/CERC_RUNTIME_ENV_//g')
+    cur_val=$(echo -n "\$${cur_name}" | envsubst)
+    if [ "$CERC_RETAIN_ENV_QUOTES" != "true" ]; then
+      cur_val=$(sed "s/^[\"']//" <<< "$cur_val" | sed "s/[\"']//")
+    fi
+    esc_val=$(sed 's/[&/\]/\\&/g' <<< "$cur_val")
+    echo "$f: $cur_name=$cur_val"
+    sed -i "s/$orig_name/$esc_val/g" $f
+  done
+done

stack_orchestrator/data/container-build/cerc-webapp-base/scripts/build-app.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+    set -x
+fi
+
+CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
+WORK_DIR="${1:-/app}"
+OUTPUT_DIR="${2:-build}"
+DEST_DIR="${3:-/data}"
+
+if [ -f "${WORK_DIR}/package.json" ]; then
+  echo "Building node-based webapp ..."
+  cd "${WORK_DIR}" || exit 1
+
+  if [ -z "$CERC_BUILD_TOOL" ]; then
+    if [ -f "yarn.lock" ]; then
+      CERC_BUILD_TOOL=yarn
+    else
+      CERC_BUILD_TOOL=npm
+    fi
+  fi
+
+  $CERC_BUILD_TOOL install || exit 1
+  $CERC_BUILD_TOOL build || exit 1
+
+  rm -rf "${DEST_DIR}"
+  mv "${WORK_DIR}/${OUTPUT_DIR}" "${DEST_DIR}"
+else
+  echo "Copying static app ..."
+  mv "${WORK_DIR}" "${DEST_DIR}"
+fi
+
+exit 0

stack_orchestrator/data/container-build/cerc-webapp-base/scripts/start-serving-app.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+    set -x
+fi
+
+CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/data}"
+CERC_ENABLE_CORS="${CERC_ENABLE_CORS:-false}"
+
+if [ "true" == "$CERC_ENABLE_CORS" ]; then
+  CERC_HTTP_EXTRA_ARGS="$CERC_HTTP_EXTRA_ARGS --cors"
+fi
+
+/scripts/apply-webapp-config.sh /config/config.yml ${CERC_WEBAPP_FILES_DIR}
+/scripts/apply-runtime-env.sh ${CERC_WEBAPP_FILES_DIR}
+http-server $CERC_HTTP_EXTRA_ARGS -p ${CERC_LISTEN_PORT:-80} ${CERC_WEBAPP_FILES_DIR}

stack_orchestrator/data/container-build/cerc-webapp-base/start-serving-app.sh

@@ -1,9 +0,0 @@
-#!/usr/bin/env bash
-if [ -n "$CERC_SCRIPT_DEBUG" ]; then
-    set -x
-fi
-
-CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/data}"
-
-/scripts/apply-webapp-config.sh /config/config.yml ${CERC_WEBAPP_FILES_DIR}
-http-server -p 80 ${CERC_WEBAPP_FILES_DIR}

stack_orchestrator/data/container-build/cerc-webapp-deployer-backend/build.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# Build cerc/webapp-deployer-backend
+
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/webapp-deployer-backend:local ${build_command_args} ${CERC_REPO_BASE_DIR}/webapp-deployment-status-api

stack_orchestrator/data/stacks/act-runner/stack.yml

@@ -3,7 +3,7 @@ name: act-runner
 description: "Local act-runner"
 repos:
   - git.vdb.to/cerc-io/hosting
-  - gitea.com/gitea/act_runner
+  - gitea.com/telackey/act_runner@telackey/entrypoint
 containers:
   - cerc/act-runner
   - cerc/act-runner-task-executor

stack_orchestrator/data/stacks/azimuth/README.md

@@ -40,20 +40,36 @@ network:
   ports:
     watcher-db:
      - 0.0.0.0:15432:5432
+    azimuth-watcher-job-runner:
+     - 0.0.0.0:9000:9000
     azimuth-watcher-server:
      - 0.0.0.0:3001:3001
+    censures-watcher-job-runner:
+     - 0.0.0.0:9002:9002
     censures-watcher-server:
      - 0.0.0.0:3002:3002
+    claims-watcher-job-runner:
+     - 0.0.0.0:9004:9004
     claims-watcher-server:
      - 0.0.0.0:3003:3003
+    conditional-star-release-watcher-job-runner:
+     - 0.0.0.0:9006:9006
     conditional-star-release-watcher-server:
      - 0.0.0.0:3004:3004
+    delegated-sending-watcher-job-runner:
+     - 0.0.0.0:9008:9008
     delegated-sending-watcher-server:
      - 0.0.0.0:3005:3005
+    ecliptic-watcher-job-runner:
+     - 0.0.0.0:9010:9010
     ecliptic-watcher-server:
      - 0.0.0.0:3006:3006
+    linear-star-release-watcher-job-runner:
+     - 0.0.0.0:9012:9012
     linear-star-release-watcher-server:
      - 0.0.0.0:3007:3007
+    polls-watcher-job-runner:
+     - 0.0.0.0:9014:9014
     polls-watcher-server:
      - 0.0.0.0:3008:3008
     gateway-server:

stack_orchestrator/data/stacks/azimuth/stack.yml

@@ -1,7 +1,7 @@
 version: "1.0"
 name: azimuth
 repos:
-  - github.com/cerc-io/azimuth-watcher-ts@v0.1.2
+  - github.com/cerc-io/azimuth-watcher-ts@v0.1.3
 containers:
   - cerc/watcher-azimuth
 pods:

stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml

@@ -2,7 +2,7 @@ version: "1.0"
 name: merkl-sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.4
+  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.6
 containers:
   - cerc/watcher-merkl-sushiswap-v3
 pods:

stack_orchestrator/data/stacks/monitoring/README.md

@@ -1,8 +1,29 @@
 # monitoring
 
 * Instructions to setup and run a Prometheus server and Grafana dashboard
+* Comes with the following built-in exporters / dashboards:
+  * Chain Head Exporter - for tracking chain heads given external ETH RPC endpoints
+  * Watchers dashboard
+  * [Prometheus Blackbox](https://grafana.com/grafana/dashboards/7587-prometheus-blackbox-exporter/) - for tracking HTTP endpoints
+  * [NodeJS Application Dashboard](https://grafana.com/grafana/dashboards/11159-nodejs-application-dashboard/) - for default NodeJS metrics
+  * [PostgreSQL Database](https://grafana.com/grafana/dashboards/9628-postgresql-database/) - for monitoring Postgres dbs
+  * [Node Exporter Full](https://grafana.com/grafana/dashboards/1860-node-exporter-full/) - for monitoring system metrics
 * See [monitoring-watchers.md](./monitoring-watchers.md) for an example usage of the stack with pre-configured dashboards for watchers
 
+## Setup
+
+Clone required repositories:
+
+```bash
+laconic-so --stack monitoring setup-repositories --git-ssh --pull
+```
+
+Build the container images:
+
+```bash
+laconic-so --stack monitoring build-containers
+```
+
 ## Create a deployment
 
 First, create a spec file for the deployment, which will map the stack's ports and volumes to the host:
@@ -43,7 +64,7 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --de
 
 ### Prometheus Config
 
-Add desired scrape configs to prometheus config file (`monitoring-deployment/config/monitoring/prometheus/prometheus.yml`) in the deployment folder; for example:
+* Add desired scrape configs to prometheus config file (`monitoring-deployment/config/monitoring/prometheus/prometheus.yml`) in the deployment folder; for example:
 
   ```yml
   ...
@@ -54,12 +75,77 @@ Add desired scrape configs to prometheus config file (`monitoring-deployment/con
       - targets: ['<METRICS_ENDPOINT_HOST>:<METRICS_ENDPOINT_PORT>']
   ```
 
+* Node exporter: update the `node` job to add any node-exporter targets to be monitored:
+
+  ```yml
+  ...
+  - job_name: 'node'
+    ...
+    static_configs:
+      # Add node-exporter targets to be monitored below
+      - targets: [example-host:9100]
+        labels:
+          instance: 'my-host'
+  ```
+
+* Blackbox (in-stack exporter): update the `blackbox` job to add any endpoints to be monitored on the Blackbox dashboard:
+
+  ```yml
+  ...
+  - job_name: 'blackbox'
+    ...
+    static_configs:
+      # Add URLs to be monitored below
+      - targets:
+        - <HTTP_ENDPOINT_1>
+        - <HTTP_ENDPOINT_2>
+  ```
+
+* Postgres (in-stack exporter):
+  * Update the `postgres` job to add Postgres db targets to be monitored:
+
+    ```yml
+    ...
+    - job_name: 'postgres'
+      ...
+      static_configs:
+        # Add DB targets below
+        - targets: [example-server:5432]
+          labels:
+            instance: 'example-db'
+    ```
+  * Add database credentials to be used in `auth_modules` in the postgres-exporter config file (`monitoring-deployment/config/monitoring/postgres-exporter.yml`)
+
 Note: Use `host.docker.internal` as host to access ports on the host machine
 
 ### Grafana Config
 
 Place the dashboard json files in grafana dashboards config directory (`monitoring-deployment/config/monitoring/grafana/dashboards`) in the deployment folder
 
+### Env
+
+Set the following env variables in the deployment env config file (`monitoring-deployment/config.env`):
+
+  ```bash
+  # For chain-head exporter
+
+  # External ETH RPC endpoint (ethereum)
+  # (Optional, default: https://mainnet.infura.io/v3)
+  CERC_ETH_RPC_ENDPOINT=
+
+  # Infura key to be used
+  # (Optional, used with ETH_RPC_ENDPOINT if provided)
+  CERC_INFURA_KEY=
+
+  # External ETH RPC endpoint (filecoin)
+  # (Optional, default: https://api.node.glif.io/rpc/v1)
+  CERC_FIL_RPC_ENDPOINT=
+
+  # Grafana server host URL (used in various links in alerts, etc.)
+  # (Optional, default: http://localhost:3000)
+  GF_SERVER_ROOT_URL=
+  ```
+
 ## Start the stack
 
 Start the deployment:

stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md

@@ -1,10 +1,10 @@
 # Monitoring Watchers
 
-Instructions to setup and run monitoring stack with pre-configured watcher dashboards
+Instructions to setup and run monitoring stack with configured Watchers dashboard
 
 ## Create a deployment
 
-First, create a spec file for the deployment, which will map the stack's ports and volumes to the host:
+After completing [setup](./README.md#setup), create a spec file for the deployment, which will map the stack's ports and volumes to the host:
 
 ```bash
 laconic-so --stack monitoring deploy init --output monitoring-watchers-spec.yml
@@ -35,60 +35,106 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-watchers-spec
 
 ## Configure
 
-### Prometheus Config
+### Prometheus scrape config
 
 Add the following scrape configs to prometheus config file (`monitoring-watchers-deployment/config/monitoring/prometheus/prometheus.yml`) in the deployment folder:
 
   ```yml
   ...
+  - job_name: 'blackbox'
+    ...
+    static_configs:
+      - targets:
+        - <AZIMUTH_GATEWAY_GQL_ENDPOINT>
+  ...
   - job_name: azimuth
+    scrape_interval: 10s
     metrics_path: /metrics
     scheme: http
     static_configs:
       - targets: ['AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_PORT']
         labels:
           instance: 'azimuth'
+          chain: 'ethereum'
       - targets: ['CENSURES_WATCHER_HOST:CENSURES_WATCHER_PORT']
         labels:
           instance: 'censures'
+          chain: 'ethereum'
       - targets: ['CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_PORT']
         labels:
           instance: 'claims'
+          chain: 'ethereum'
       - targets: ['CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_PORT']
         labels:
           instance: 'conditional_star_release'
+          chain: 'ethereum'
       - targets: ['DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_PORT']
         labels:
-          instance: 'delegated_sending_watcher'
+          instance: 'delegated_sending'
+          chain: 'ethereum'
       - targets: ['ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_PORT']
         labels:
           instance: 'ecliptic'
+          chain: 'ethereum'
       - targets: ['LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_PORT']
         labels:
           instance: 'linear_star_release'
+          chain: 'ethereum'
       - targets: ['POLLS_WATCHER_HOST:POLLS_WATCHER_PORT']
         labels:
           instance: 'polls'
+          chain: 'ethereum'
 
   - job_name: sushi
+    scrape_interval: 20s
     metrics_path: /metrics
     scheme: http
     static_configs:
       - targets: ['SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_PORT']
         labels:
           instance: 'sushiswap'
+          chain: 'filecoin'
       - targets: ['MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_PORT']
         labels:
           instance: 'merkl_sushiswap'
+          chain: 'filecoin'
   ```
 
-### Grafana Config
+Add scrape config as done above for any additional watcher to add it to the Watchers dashboard.
 
-In the deployment folder, copy over the pre-configured watcher dashboard JSON files to grafana dashboards config directory:
+### Grafana alerts config
 
-```bash
-cp -r monitoring-watchers-deployment/config/monitoring/grafana/watcher-dashboards/* monitoring-watchers-deployment/config/monitoring/grafana/dashboards/
-```
+Place the pre-configured watcher alerts rules in Grafana provisioning directory:
+
+  ```bash
+  cp monitoring-watchers-deployment/config/monitoring/watcher-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
+  ```
+
+Update the alerting contact points config (`monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/contactpoints.yml`) with desired contact points
+
+Add corresponding routes to the notification policies config (`monitoring-watchers-deployment/monitoring/grafana/provisioning/alerting/policies.yaml`) with appropriate object-matchers:
+
+  ```yml
+  ...
+    routes:
+      - receiver: SlackNotifier
+      object_matchers:
+        # Add matchers below
+        - ['grafana_folder', '=', 'WatcherAlerts']
+  ```
+
+### Env
+
+Set the following env variables in the deployment env config file (`monitoring-watchers-deployment/config.env`):
+
+  ```bash
+  # Infura key to be used
+  CERC_INFURA_KEY=
+
+  # Grafana server host URL to be used
+  # (Optional, default: http://localhost:3000)
+  GF_SERVER_ROOT_URL=
+  ```
 
 ## Start the stack
 

stack_orchestrator/data/stacks/monitoring/stack.yml

@@ -1,7 +1,9 @@
 version: "0.1"
 name: monitoring
 repos:
+  - github.com/cerc-io/watcher-ts@v0.2.79
 containers:
+  - cerc/watcher-ts
 pods:
   - prom-server
   - grafana

stack_orchestrator/data/stacks/node-exporter/README.md

@@ -0,0 +1,17 @@
+# node-exporter
+
+## Start the stack
+
+```bash
+laconic-so --stack node-exporter deploy up
+```
+
+* The host node's metrics can be accessed at `http://localhost:9100/metrics`
+
+## Clean up
+
+Stop the node-exporter running in background:
+
+```bash
+laconic-so --stack node-exporter deploy down
+```

stack_orchestrator/data/stacks/node-exporter/stack.yml

@@ -0,0 +1,6 @@
+version: "0.1"
+name: node-exporter
+repos:
+containers:
+pods:
+  - node-exporter

stack_orchestrator/data/stacks/package-registry/stack.yml

@@ -3,7 +3,7 @@ name: package-registry
 description: "Local Package Registry"
 repos:
   - git.vdb.to/cerc-io/hosting
-  - gitea.com/gitea/act_runner
+  - gitea.com/telackey/act_runner@telackey/entrypoint
 containers:
   - cerc/act-runner
   - cerc/act-runner-task-executor

stack_orchestrator/data/stacks/sushiswap-v3/stack.yml

@@ -2,7 +2,7 @@ version: "1.0"
 name: sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.4
+  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.6
 containers:
   - cerc/watcher-sushiswap-v3
 pods:

stack_orchestrator/data/stacks/webapp-deployer-backend/stack.yml

@@ -0,0 +1,11 @@
+version: "1.0"
+name: webapp-deployer-backend
+description: "Deployer for webapps"
+repos:
+  - git.vdb.to:telackey/webapp-deployment-status-api
+containers:
+  - cerc/webapp-deployer-backend
+pods:
+  - name: webapp-deployer-backend
+    repository: git.vdb.to:telackey/webapp-deployment-status-api
+    path: ./

stack_orchestrator/deploy/deploy.py

@@ -347,8 +347,8 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
             else:
                 if deployment:
                     compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_name}.yml")
-                    pod_pre_start_command = pod["pre_start_command"]
-                    pod_post_start_command = pod["post_start_command"]
+                    pod_pre_start_command = pod.get("pre_start_command")
+                    pod_post_start_command = pod.get("post_start_command")
                     script_dir = compose_dir.parent.joinpath("pods", pod_name, "scripts")
                     if pod_pre_start_command is not None:
                         pre_start_commands.append(os.path.join(script_dir, pod_pre_start_command))
@@ -357,8 +357,8 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
                 else:
                     pod_root_dir = os.path.join(dev_root_path, pod_repository.split("/")[-1], pod["path"])
                     compose_file_name = os.path.join(pod_root_dir, f"docker-compose-{pod_name}.yml")
-                    pod_pre_start_command = pod["pre_start_command"]
-                    pod_post_start_command = pod["post_start_command"]
+                    pod_pre_start_command = pod.get("pre_start_command")
+                    pod_post_start_command = pod.get("post_start_command")
                     if pod_pre_start_command is not None:
                         pre_start_commands.append(os.path.join(pod_root_dir, pod_pre_start_command))
                     if pod_post_start_command is not None:

stack_orchestrator/deploy/deployment_create.py

@@ -54,19 +54,44 @@ def _get_ports(stack):
 
 def _get_named_volumes(stack):
     # Parse the compose files looking for named volumes
-    named_volumes = []
+    named_volumes = {
+        "rw": [],
+        "ro": []
+    }
     parsed_stack = get_parsed_stack_config(stack)
     pods = get_pod_list(parsed_stack)
     yaml = get_yaml()
+
+    def find_vol_usage(parsed_pod_file, vol):
+        ret = {}
+        if "services" in parsed_pod_file:
+            for svc_name, svc in parsed_pod_file["services"].items():
+                if "volumes" in svc:
+                    for svc_volume in svc["volumes"]:
+                        parts = svc_volume.split(":")
+                        if parts[0] == vol:
+                            ret[svc_name] = {
+                                "volume": parts[0],
+                                "mount": parts[1],
+                                "options": parts[2] if len(parts) == 3 else None
+                            }
+        return ret
+
     for pod in pods:
         pod_file_path = get_pod_file_path(parsed_stack, pod)
         parsed_pod_file = yaml.load(open(pod_file_path, "r"))
         if "volumes" in parsed_pod_file:
             volumes = parsed_pod_file["volumes"]
             for volume in volumes.keys():
-                # Volume definition looks like:
-                # 'laconicd-data': None
-                named_volumes.append(volume)
+                for vu in find_vol_usage(parsed_pod_file, volume).values():
+                    read_only = vu["options"] == "ro"
+                    if read_only:
+                        if vu["volume"] not in named_volumes["rw"] and vu["volume"] not in named_volumes["ro"]:
+                            named_volumes["ro"].append(vu["volume"])
+                    else:
+                        if vu["volume"] not in named_volumes["rw"]:
+                            named_volumes["rw"].append(vu["volume"])
+
     return named_volumes
 
 
@@ -98,12 +123,24 @@ def _fixup_pod_file(pod, spec, compose_dir):
                     _create_bind_dir_if_relative(volume, volume_spec, compose_dir)
                     new_volume_spec = {"driver": "local",
                                        "driver_opts": {
-                                          "type": "none",
-                                          "device": volume_spec_fixedup,
-                                          "o": "bind"
+                                           "type": "none",
+                                           "device": volume_spec_fixedup,
+                                           "o": "bind"
                                        }
                                        }
                     pod["volumes"][volume] = new_volume_spec
+
+    # Fix up configmaps
+    if "configmaps" in spec:
+        spec_cfgmaps = spec["configmaps"]
+        if "volumes" in pod:
+            pod_volumes = pod["volumes"]
+            for volume in pod_volumes.keys():
+                if volume in spec_cfgmaps:
+                    volume_cfg = spec_cfgmaps[volume]
+                    # Just make the dir (if necessary)
+                    _create_bind_dir_if_relative(volume, volume_cfg, compose_dir)
+
     # Fix up ports
     if "network" in spec and "ports" in spec["network"]:
         spec_ports = spec["network"]["ports"]
@@ -319,9 +356,18 @@ def init_operation(deploy_command_context, stack, deployer_type, config,
     named_volumes = _get_named_volumes(stack)
     if named_volumes:
         volume_descriptors = {}
-        for named_volume in named_volumes:
+        configmap_descriptors = {}
+        for named_volume in named_volumes["rw"]:
             volume_descriptors[named_volume] = f"./data/{named_volume}"
-        spec_file_content["volumes"] = volume_descriptors
+        for named_volume in named_volumes["ro"]:
+            if "k8s" in deployer_type and "config" in named_volume:
+                configmap_descriptors[named_volume] = f"./data/{named_volume}"
+            else:
+                volume_descriptors[named_volume] = f"./data/{named_volume}"
+        if volume_descriptors:
+            spec_file_content["volumes"] = volume_descriptors
+        if configmap_descriptors:
+            spec_file_content["configmaps"] = configmap_descriptors
 
     if opts.o.debug:
         print(f"Creating spec file for stack: {stack} with content: {spec_file_content}")
@@ -444,7 +490,7 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
     deployer_config_generator = getDeployerConfigGenerator(deployment_type)
     # TODO: make deployment_dir_path a Path above
     deployer_config_generator.generate(deployment_dir_path)
-    call_stack_deploy_create(deployment_context, [network_dir, initial_peers])
+    call_stack_deploy_create(deployment_context, [network_dir, initial_peers, deployment_command_context])
 
 
 # TODO: this code should be in the stack .py files but

stack_orchestrator/deploy/images.py

@@ -31,7 +31,8 @@ def _image_needs_pushed(image: str):
 
 def remote_tag_for_image(image: str, remote_repo_url: str):
     # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
-    (org, image_name_with_version) = image.split("/")
+    major_parts = image.split("/", 2)
+    image_name_with_version = major_parts[1] if 2 == len(major_parts) else major_parts[0]
     (image_name, image_version) = image_name_with_version.split(":")
     if image_version == "local":
         return f"{remote_repo_url}/{image_name}:deploy"

stack_orchestrator/deploy/k8s/cluster_info.py

@@ -13,6 +13,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.
 
+import os
+
 from kubernetes import client
 from typing import Any, List, Set
 
@@ -112,9 +114,10 @@ class ClusterInfo:
             services = pod["services"]
             for service_name in services:
                 service_info = services[service_name]
-                port = int(service_info["ports"][0])
-                if opts.o.debug:
-                    print(f"service port: {port}")
+                if "ports" in service_info:
+                    port = int(service_info["ports"][0])
+                    if opts.o.debug:
+                        print(f"service port: {port}")
         service = client.V1Service(
             metadata=client.V1ObjectMeta(name=f"{self.app_name}-service"),
             spec=client.V1ServiceSpec(
@@ -130,30 +133,70 @@ class ClusterInfo:
 
     def get_pvcs(self):
         result = []
-        volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
+        spec_volumes = self.spec.get_volumes()
+        named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
         if opts.o.debug:
-            print(f"Volumes: {volumes}")
-        for volume_name in volumes:
+            print(f"Spec Volumes: {spec_volumes}")
+            print(f"Named Volumes: {named_volumes}")
+        for volume_name in spec_volumes:
+            if volume_name not in named_volumes:
+                if opts.o.debug:
+                    print(f"{volume_name} not in pod files")
+                continue
             spec = client.V1PersistentVolumeClaimSpec(
                 access_modes=["ReadWriteOnce"],
                 storage_class_name="manual",
                 resources=client.V1ResourceRequirements(
                     requests={"storage": "2Gi"}
                 ),
-                volume_name=volume_name
+                volume_name=f"{self.app_name}-{volume_name}"
             )
             pvc = client.V1PersistentVolumeClaim(
-                metadata=client.V1ObjectMeta(name=volume_name,
-                                             labels={"volume-label": volume_name}),
+                metadata=client.V1ObjectMeta(name=f"{self.app_name}-{volume_name}",
+                                             labels={"volume-label": f"{self.app_name}-{volume_name}"}),
                 spec=spec,
             )
             result.append(pvc)
         return result
 
+    def get_configmaps(self):
+        result = []
+        spec_configmaps = self.spec.get_configmaps()
+        named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
+        for cfg_map_name, cfg_map_path in spec_configmaps.items():
+            if cfg_map_name not in named_volumes:
+                if opts.o.debug:
+                    print(f"{cfg_map_name} not in pod files")
+                continue
+
+            if not cfg_map_path.startswith("/"):
+                cfg_map_path = os.path.join(os.path.dirname(self.spec.file_path), cfg_map_path)
+
+            # Read in all the files at a single-level of the directory.  This mimics the behavior
+            # of `kubectl create configmap foo --from-file=/path/to/dir`
+            data = {}
+            for f in os.listdir(cfg_map_path):
+                full_path = os.path.join(cfg_map_path, f)
+                if os.path.isfile(full_path):
+                    data[f] = open(full_path, 'rt').read()
+
+            spec = client.V1ConfigMap(
+                metadata=client.V1ObjectMeta(name=f"{self.app_name}-{cfg_map_name}",
+                                             labels={"configmap-label": cfg_map_name}),
+                data=data
+            )
+            result.append(spec)
+        return result
+
     def get_pvs(self):
         result = []
-        volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
-        for volume_name in volumes:
+        spec_volumes = self.spec.get_volumes()
+        named_volumes = named_volumes_from_pod_files(self.parsed_pod_yaml_map)
+        for volume_name in spec_volumes:
+            if volume_name not in named_volumes:
+                if opts.o.debug:
+                    print(f"{volume_name} not in pod files")
+                continue
             spec = client.V1PersistentVolumeSpec(
                 storage_class_name="manual",
                 access_modes=["ReadWriteOnce"],
@@ -161,15 +204,15 @@ class ClusterInfo:
                 host_path=client.V1HostPathVolumeSource(path=get_node_pv_mount_path(volume_name))
             )
             pv = client.V1PersistentVolume(
-                metadata=client.V1ObjectMeta(name=volume_name,
-                                             labels={"volume-label": volume_name}),
+                metadata=client.V1ObjectMeta(name=f"{self.app_name}-{volume_name}",
+                                             labels={"volume-label": f"{self.app_name}-{volume_name}"}),
                 spec=spec,
             )
             result.append(pv)
         return result
 
-    # to suit the deployment, and also annotate the container specs to point at said volumes
-    def get_deployment(self):
+    # TODO: put things like image pull policy into an object-scope struct
+    def get_deployment(self, image_pull_policy: str = None):
         containers = []
         for pod_name in self.parsed_pod_yaml_map:
             pod = self.parsed_pod_yaml_map[pod_name]
@@ -178,10 +221,11 @@ class ClusterInfo:
                 container_name = service_name
                 service_info = services[service_name]
                 image = service_info["image"]
-                port = int(service_info["ports"][0])
-                if opts.o.debug:
-                    print(f"image: {image}")
-                    print(f"service port: {port}")
+                if "ports" in service_info:
+                    port = int(service_info["ports"][0])
+                    if opts.o.debug:
+                        print(f"image: {image}")
+                        print(f"service port: {port}")
                 # Re-write the image tag for remote deployment
                 image_to_use = remote_tag_for_image(
                     image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image
@@ -189,17 +233,17 @@ class ClusterInfo:
                 container = client.V1Container(
                     name=container_name,
                     image=image_to_use,
-                    image_pull_policy="Always",
+                    image_pull_policy=image_pull_policy,
                     env=envs_from_environment_variables_map(self.environment_variables.map),
                     ports=[client.V1ContainerPort(container_port=port)],
                     volume_mounts=volume_mounts,
                     resources=client.V1ResourceRequirements(
                         requests={"cpu": "100m", "memory": "200Mi"},
-                        limits={"cpu": "500m", "memory": "500Mi"},
+                        limits={"cpu": "1000m", "memory": "2000Mi"},
                     ),
                 )
                 containers.append(container)
-        volumes = volumes_for_pod_files(self.parsed_pod_yaml_map)
+        volumes = volumes_for_pod_files(self.parsed_pod_yaml_map, self.spec, self.app_name)
         image_pull_secrets = [client.V1LocalObjectReference(name="laconic-registry")]
         template = client.V1PodTemplateSpec(
             metadata=client.V1ObjectMeta(labels={"app": self.app_name}),

stack_orchestrator/deploy/k8s/deploy_k8s.py

@@ -1,5 +1,4 @@
 # Copyright © 2023 Vulcanize
-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
@@ -110,8 +109,22 @@ class K8sDeployer(Deployer):
             if opts.o.debug:
                 print("PVCs created:")
                 print(f"{pvc_resp}")
+
+        # Figure out the ConfigMaps for this deployment
+        config_maps = self.cluster_info.get_configmaps()
+        for cfg_map in config_maps:
+            if opts.o.debug:
+                print(f"Sending this ConfigMap: {cfg_map}")
+            cfg_rsp = self.core_api.create_namespaced_config_map(
+                body=cfg_map,
+                namespace=self.k8s_namespace
+            )
+            if opts.o.debug:
+                print("ConfigMap created:")
+                print(f"{cfg_rsp}")
+
         # Process compose files into a Deployment
-        deployment = self.cluster_info.get_deployment()
+        deployment = self.cluster_info.get_deployment(image_pull_policy=None if self.is_kind() else "Always")
         # Create the k8s objects
         if opts.o.debug:
             print(f"Sending this deployment: {deployment}")
@@ -132,20 +145,24 @@ class K8sDeployer(Deployer):
             print("Service created:")
             print(f"{service_resp}")
 
-        # TODO: disable ingress for kind
-        ingress: client.V1Ingress = self.cluster_info.get_ingress()
+        if not self.is_kind():
+            ingress: client.V1Ingress = self.cluster_info.get_ingress()
 
-        if opts.o.debug:
-            print(f"Sending this ingress: {ingress}")
-        ingress_resp = self.networking_api.create_namespaced_ingress(
-            namespace=self.k8s_namespace,
-            body=ingress
-        )
-        if opts.o.debug:
-            print("Ingress created:")
-            print(f"{ingress_resp}")
+            if ingress:
+                if opts.o.debug:
+                    print(f"Sending this ingress: {ingress}")
+                ingress_resp = self.networking_api.create_namespaced_ingress(
+                    namespace=self.k8s_namespace,
+                    body=ingress
+                )
+                if opts.o.debug:
+                    print("Ingress created:")
+                    print(f"{ingress_resp}")
+            else:
+                if opts.o.debug:
+                    print("No ingress configured")
 
-    def down(self, timeout, volumes):
+    def down(self, timeout, volumes):  # noqa: C901
         self.connect_api()
         # Delete the k8s objects
         # Create the host-path-mounted PVs for this deployment
@@ -175,6 +192,22 @@ class K8sDeployer(Deployer):
                     print(f"{pvc_resp}")
             except client.exceptions.ApiException as e:
                 _check_delete_exception(e)
+
+        # Figure out the ConfigMaps for this deployment
+        cfg_maps = self.cluster_info.get_configmaps()
+        for cfg_map in cfg_maps:
+            if opts.o.debug:
+                print(f"Deleting this ConfigMap: {cfg_map}")
+            try:
+                cfg_map_resp = self.core_api.delete_namespaced_config_map(
+                    name=cfg_map.metadata.name, namespace=self.k8s_namespace
+                )
+                if opts.o.debug:
+                    print("ConfigMap deleted:")
+                    print(f"{cfg_map_resp}")
+            except client.exceptions.ApiException as e:
+                _check_delete_exception(e)
+
         deployment = self.cluster_info.get_deployment()
         if opts.o.debug:
             print(f"Deleting this deployment: {deployment}")
@@ -196,16 +229,20 @@ class K8sDeployer(Deployer):
         except client.exceptions.ApiException as e:
             _check_delete_exception(e)
 
-        # TODO: disable ingress for kind
-        ingress: client.V1Ingress = self.cluster_info.get_ingress()
-        if opts.o.debug:
-            print(f"Deleting this ingress: {ingress}")
-        try:
-            self.networking_api.delete_namespaced_ingress(
-                name=ingress.metadata.name, namespace=self.k8s_namespace
-            )
-        except client.exceptions.ApiException as e:
-            _check_delete_exception(e)
+        if not self.is_kind():
+            ingress: client.V1Ingress = self.cluster_info.get_ingress()
+            if ingress:
+                if opts.o.debug:
+                    print(f"Deleting this ingress: {ingress}")
+                try:
+                    self.networking_api.delete_namespaced_ingress(
+                        name=ingress.metadata.name, namespace=self.k8s_namespace
+                    )
+                except client.exceptions.ApiException as e:
+                    _check_delete_exception(e)
+            else:
+                if opts.o.debug:
+                    print("No ingress to delete")
 
         if self.is_kind():
             # Destroy the kind cluster
@@ -219,7 +256,7 @@ class K8sDeployer(Deployer):
 
         if all_pods.items:
             for p in all_pods.items:
-                if self.cluster_info.app_name in p.metadata.name:
+                if f"{self.cluster_info.app_name}-deployment" in p.metadata.name:
                     pods.append(p)
 
         if not pods:
@@ -266,7 +303,7 @@ class K8sDeployer(Deployer):
         ret = []
 
         for p in pods.items:
-            if self.cluster_info.app_name in p.metadata.name:
+            if f"{self.cluster_info.app_name}-deployment" in p.metadata.name:
                 pod_ip = p.status.pod_ip
                 ports = AttrDict()
                 for c in p.spec.containers:
@@ -299,11 +336,20 @@ class K8sDeployer(Deployer):
 
     def logs(self, services, tail, follow, stream):
         self.connect_api()
-        pods = pods_in_deployment(self.core_api, "test-deployment")
+        pods = pods_in_deployment(self.core_api, self.cluster_info.app_name)
         if len(pods) > 1:
             print("Warning: more than one pod in the deployment")
-        k8s_pod_name = pods[0]
-        log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test")
+        if len(pods) == 0:
+            log_data = "******* Pods not running ********\n"
+        else:
+            k8s_pod_name = pods[0]
+            # If the pod is not yet started, the logs request below will throw an exception
+            try:
+                log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test")
+            except client.exceptions.ApiException as e:
+                if opts.o.debug:
+                    print(f"Error from read_namespaced_pod_log: {e}")
+                log_data = "******* No logs available ********\n"
         return log_stream_from_string(log_data)
 
     def update(self):

stack_orchestrator/deploy/k8s/helpers.py

@@ -21,6 +21,7 @@ from typing import Set, Mapping, List
 
 from stack_orchestrator.opts import opts
 from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names
+from stack_orchestrator.deploy.deployer import DeployerException
 
 
 def _run_command(command: str):
@@ -29,10 +30,13 @@ def _run_command(command: str):
     result = subprocess.run(command, shell=True)
     if opts.o.debug:
         print(f"Result: {result}")
+    return result
 
 
 def create_cluster(name: str, config_file: str):
-    _run_command(f"kind create cluster --name {name} --config {config_file}")
+    result = _run_command(f"kind create cluster --name {name} --config {config_file}")
+    if result.returncode != 0:
+        raise DeployerException(f"kind create cluster failed: {result}")
 
 
 def destroy_cluster(name: str):
@@ -41,12 +45,14 @@ def destroy_cluster(name: str):
 
 def load_images_into_kind(kind_cluster_name: str, image_set: Set[str]):
     for image in image_set:
-        _run_command(f"kind load docker-image {image} --name {kind_cluster_name}")
+        result = _run_command(f"kind load docker-image {image} --name {kind_cluster_name}")
+        if result.returncode != 0:
+            raise DeployerException(f"kind create cluster failed: {result}")
 
 
 def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str):
     pods = []
-    pod_response = core_api.list_namespaced_pod(namespace="default", label_selector="app=test-app")
+    pod_response = core_api.list_namespaced_pod(namespace="default", label_selector=f"app={deployment_name}")
     if opts.o.debug:
         print(f"pod_response: {pod_response}")
     for pod_info in pod_response.items:
@@ -67,7 +73,7 @@ def named_volumes_from_pod_files(parsed_pod_files):
         parsed_pod_file = parsed_pod_files[pod]
         if "volumes" in parsed_pod_file:
             volumes = parsed_pod_file["volumes"]
-            for volume in volumes.keys():
+            for volume, value in volumes.items():
                 # Volume definition looks like:
                 # 'laconicd-data': None
                 named_volumes.append(volume)
@@ -92,22 +98,31 @@ def volume_mounts_for_service(parsed_pod_files, service):
                         volumes = service_obj["volumes"]
                         for mount_string in volumes:
                             # Looks like: test-data:/data
-                            (volume_name, mount_path) = mount_string.split(":")
-                            volume_device = client.V1VolumeMount(mount_path=mount_path, name=volume_name)
+                            parts = mount_string.split(":")
+                            volume_name = parts[0]
+                            mount_path = parts[1]
+                            mount_options = parts[2] if len(parts) == 3 else None
+                            volume_device = client.V1VolumeMount(
+                                mount_path=mount_path, name=volume_name, read_only="ro" == mount_options)
                             result.append(volume_device)
     return result
 
 
-def volumes_for_pod_files(parsed_pod_files):
+def volumes_for_pod_files(parsed_pod_files, spec, app_name):
     result = []
     for pod in parsed_pod_files:
         parsed_pod_file = parsed_pod_files[pod]
         if "volumes" in parsed_pod_file:
             volumes = parsed_pod_file["volumes"]
             for volume_name in volumes.keys():
-                claim = client.V1PersistentVolumeClaimVolumeSource(claim_name=volume_name)
-                volume = client.V1Volume(name=volume_name, persistent_volume_claim=claim)
-                result.append(volume)
+                if volume_name in spec.get_configmaps():
+                    config_map = client.V1ConfigMapVolumeSource(name=f"{app_name}-{volume_name}")
+                    volume = client.V1Volume(name=volume_name, config_map=config_map)
+                    result.append(volume)
+                else:
+                    claim = client.V1PersistentVolumeClaimVolumeSource(claim_name=f"{app_name}-{volume_name}")
+                    volume = client.V1Volume(name=volume_name, persistent_volume_claim=claim)
+                    result.append(volume)
     return result
 
 
@@ -152,7 +167,7 @@ def _generate_kind_mounts(parsed_pod_files, deployment_dir):
                         volume_definitions.append(
                             f"  - hostPath: {_make_absolute_host_path(volume_host_path_map[volume_name], deployment_dir)}\n"
                             f"    containerPath: {get_node_pv_mount_path(volume_name)}"
-                            )
+                        )
     return (
         "" if len(volume_definitions) == 0 else (
             "  extraMounts:\n"

stack_orchestrator/deploy/spec.py

@@ -22,6 +22,7 @@ from stack_orchestrator import constants
 class Spec:
 
     obj: typing.Any
+    file_path: Path
 
     def __init__(self) -> None:
         pass
@@ -29,12 +30,23 @@ class Spec:
     def init_from_file(self, file_path: Path):
         with file_path:
             self.obj = get_yaml().load(open(file_path, "r"))
+            self.file_path = file_path
 
     def get_image_registry(self):
         return (self.obj[constants.image_resigtry_key]
                 if self.obj and constants.image_resigtry_key in self.obj
                 else None)
 
+    def get_volumes(self):
+        return (self.obj["volumes"]
+                if self.obj and "volumes" in self.obj
+                else {})
+
+    def get_configmaps(self):
+        return (self.obj["configmaps"]
+                if self.obj and "configmaps" in self.obj
+                else {})
+
     def get_http_proxy(self):
         return (self.obj[constants.network_key][constants.http_proxy_key]
                 if self.obj and constants.network_key in self.obj

stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py

@@ -26,7 +26,8 @@ from stack_orchestrator.deploy.webapp import deploy_webapp
 from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient,
                                                    build_container_image, push_container_image,
                                                    file_hash, deploy_to_k8s, publish_deployment,
-                                                   hostname_for_deployment_request, generate_hostname_for_app)
+                                                   hostname_for_deployment_request, generate_hostname_for_app,
+                                                   match_owner)
 
 
 def process_app_deployment_request(
@@ -57,19 +58,12 @@ def process_app_deployment_request(
     dns_crn = f"{dns_record_namespace}/{fqdn}"
     dns_record = laconic.get_record(dns_crn)
     if dns_record:
-        dns_record_owners = dns_record.owners
-        dns_request_owners = []
-        if dns_record.request:
-            prev_request = laconic.get_record(dns_record.request, require=True)
-            dns_request_owners = prev_request.owners
+        matched_owner = match_owner(app_deployment_request, dns_record)
+        if not matched_owner and dns_record.attributes.request:
+            matched_owner = match_owner(app_deployment_request, laconic.get_record(dns_record.attributes.request, require=True))
 
-        owner_match = None
-
-        for owner in app_deployment_request.owners:
-            if owner in dns_request_owners or owner in dns_record_owners:
-                owner_match = owner
-        if owner_match:
-            print("Matched DnsRecord ownership to", owner)
+        if matched_owner:
+            print("Matched DnsRecord ownership:", matched_owner)
         else:
             raise Exception("Unable to confirm ownership of DnsRecord %s for request %s" %
                             (dns_record.id, app_deployment_request.id))
@@ -142,18 +136,21 @@ def load_known_requests(filename):
     return {}
 
 
-def dump_known_requests(filename, requests):
+def dump_known_requests(filename, requests, status="SEEN"):
     if not filename:
         return
     known_requests = load_known_requests(filename)
     for r in requests:
-        known_requests[r.id] = r.createTime
-    json.dump(known_requests, open(filename, "w"))
+        known_requests[r.id] = {
+            "createTime": r.createTime,
+            "status": status
+        }
+    with open(filename, "w") as f:
+        json.dump(known_requests, f)
 
 
 @click.command()
 @click.option("--kube-config", help="Provide a config file for a k8s deployment")
-@click.option("--kube-config", help="Provide a config file for a k8s deployment")
 @click.option("--laconic-config", help="Provide a config file for laconicd", required=True)
 @click.option("--image-registry", help="Provide a container image registry url for this k8s cluster")
 @click.option("--deployment-parent-dir", help="Create deployment directories beneath this directory", required=True)
@@ -208,6 +205,7 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
     requests.reverse()
     requests_by_name = {}
     for r in requests:
+        # TODO: Do this _after_ filtering deployments and cancellations to minimize round trips.
         app = laconic.get_record(r.attributes.application)
         if not app:
             print("Skipping request %s, cannot locate app." % r.id)
@@ -237,9 +235,20 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
         if d.attributes.request:
             deployments_by_request[d.attributes.request] = d
 
+    # Find removal requests.
+    cancellation_requests = {}
+    removal_requests = laconic.app_deployment_removal_requests()
+    for r in removal_requests:
+        if r.attributes.request:
+            cancellation_requests[r.attributes.request] = r
+
     requests_to_execute = []
     for r in requests_by_name.values():
-        if r.id not in deployments_by_request:
+        if r.id in cancellation_requests and match_owner(cancellation_requests[r.id], r):
+            print(f"Found deployment cancellation request for {r.id} at {cancellation_requests[r.id].id}")
+        elif r.id in deployments_by_request:
+            print(f"Found satisfied request for {r.id} at {deployments_by_request[r.id].id}")
+        else:
             if r.id not in previous_requests:
                 print(f"Request {r.id} needs to processed.")
                 requests_to_execute.append(r)
@@ -247,13 +256,13 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
                 print(
                     f"Skipping unsatisfied request {r.id} because we have seen it before."
                 )
-        else:
-            print(f"Found satisfied request {r.id} at {deployments_by_request[r.id].names[0]}")
 
     print("Found %d unsatisfied request(s) to process." % len(requests_to_execute))
 
     if not dry_run:
         for r in requests_to_execute:
+            dump_known_requests(state_file, [r], "DEPLOYING")
+            status = "ERROR"
             try:
                 process_app_deployment_request(
                     ctx,
@@ -266,5 +275,6 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
                     kube_config,
                     image_registry
                 )
+                status = "DEPLOYED"
             finally:
-                dump_known_requests(state_file, [r])
+                dump_known_requests(state_file, [r], status)

stack_orchestrator/deploy/webapp/run_webapp.py

@@ -27,7 +27,7 @@ from dotenv import dotenv_values
 from stack_orchestrator import constants
 from stack_orchestrator.deploy.deployer_factory import getDeployer
 
-WEBAPP_PORT = 3000
+WEBAPP_PORT = 80
 
 
 @click.command()

stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py

@@ -0,0 +1,189 @@
+# Copyright ©2023 Vulcanize
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+
+import click
+
+from stack_orchestrator.deploy.webapp.util import LaconicRegistryClient, match_owner
+
+
+def process_app_removal_request(ctx,
+                                laconic: LaconicRegistryClient,
+                                app_removal_request,
+                                deployment_parent_dir,
+                                delete_volumes,
+                                delete_names):
+    deployment_record = laconic.get_record(app_removal_request.attributes.deployment, require=True)
+    dns_record = laconic.get_record(deployment_record.attributes.dns, require=True)
+    deployment_dir = os.path.join(deployment_parent_dir, dns_record.attributes.name)
+
+    if not os.path.exists(deployment_dir):
+        raise Exception("Deployment directory %s does not exist." % deployment_dir)
+
+    # Check if the removal request is from the owner of the DnsRecord or deployment record.
+    matched_owner = match_owner(app_removal_request, deployment_record, dns_record)
+
+    # Or of the original deployment request.
+    if not matched_owner and deployment_record.attributes.request:
+        matched_owner = match_owner(app_removal_request, laconic.get_record(deployment_record.attributes.request, require=True))
+
+    if matched_owner:
+        print("Matched deployment ownership:", matched_owner)
+    else:
+        raise Exception("Unable to confirm ownership of deployment %s for removal request %s" %
+                        (deployment_record.id, app_removal_request.id))
+
+    # TODO(telackey): Call the function directly.  The easiest way to build the correct click context is to
+    # exec the process, but it would be better to refactor so we could just call down_operation with the
+    # necessary parameters
+    down_command = [sys.argv[0], "deployment", "--dir", deployment_dir, "down"]
+    if delete_volumes:
+        down_command.append("--delete-volumes")
+    result = subprocess.run(down_command)
+    result.check_returncode()
+
+    removal_record = {
+        "record": {
+            "type": "ApplicationDeploymentRemovalRecord",
+            "version": "1.0.0",
+            "request": app_removal_request.id,
+            "deployment": deployment_record.id,
+        }
+    }
+    laconic.publish(removal_record)
+
+    if delete_names:
+        if deployment_record.names:
+            for name in deployment_record.names:
+                laconic.delete_name(name)
+
+        if dns_record.names:
+            for name in dns_record.names:
+                laconic.delete_name(name)
+
+    if delete_volumes:
+        shutil.rmtree(deployment_dir)
+
+
+def load_known_requests(filename):
+    if filename and os.path.exists(filename):
+        return json.load(open(filename, "r"))
+    return {}
+
+
+def dump_known_requests(filename, requests):
+    if not filename:
+        return
+    known_requests = load_known_requests(filename)
+    for r in requests:
+        known_requests[r.id] = r.createTime
+    json.dump(known_requests, open(filename, "w"))
+
+
+@click.command()
+@click.option("--laconic-config", help="Provide a config file for laconicd", required=True)
+@click.option("--deployment-parent-dir", help="Create deployment directories beneath this directory", required=True)
+@click.option("--request-id", help="The ApplicationDeploymentRemovalRequest to process")
+@click.option("--discover", help="Discover and process all pending ApplicationDeploymentRemovalRequests",
+              is_flag=True, default=False)
+@click.option("--state-file", help="File to store state about previously seen requests.")
+@click.option("--only-update-state", help="Only update the state file, don't process any requests anything.", is_flag=True)
+@click.option("--delete-names/--preserve-names", help="Delete all names associated with removed deployments.", default=True)
+@click.option("--delete-volumes/--preserve-volumes", default=True, help="delete data volumes")
+@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
+@click.pass_context
+def command(ctx, laconic_config, deployment_parent_dir,
+            request_id, discover, state_file, only_update_state,
+            delete_names, delete_volumes, dry_run):
+    if request_id and discover:
+        print("Cannot specify both --request-id and --discover", file=sys.stderr)
+        sys.exit(2)
+
+    if not request_id and not discover:
+        print("Must specify either --request-id or --discover", file=sys.stderr)
+        sys.exit(2)
+
+    if only_update_state and not state_file:
+        print("--only-update-state requires --state-file", file=sys.stderr)
+        sys.exit(2)
+
+    laconic = LaconicRegistryClient(laconic_config)
+
+    # Find deployment removal requests.
+    # single request
+    if request_id:
+        requests = [laconic.get_record(request_id, require=True)]
+        # TODO: assert record type
+    # all requests
+    elif discover:
+        requests = laconic.app_deployment_removal_requests()
+
+    if only_update_state:
+        if not dry_run:
+            dump_known_requests(state_file, requests)
+        return
+
+    previous_requests = load_known_requests(state_file)
+    requests.sort(key=lambda r: r.createTime)
+
+    # Find deployments.
+    deployments = {}
+    for d in laconic.app_deployments(all=True):
+        deployments[d.id] = d
+
+    # Find removal requests.
+    removals_by_deployment = {}
+    removals_by_request = {}
+    for r in laconic.app_deployment_removals():
+        if r.attributes.deployment:
+            # TODO: should we handle CRNs?
+            removals_by_deployment[r.attributes.deployment] = r
+
+    requests_to_execute = []
+    for r in requests:
+        if not r.attributes.deployment:
+            print(f"Skipping removal request {r.id} since it was a cancellation.")
+        elif r.id in removals_by_request:
+            print(f"Found satisfied request for {r.id} at {removals_by_request[r.id].id}")
+        elif r.attributes.deployment in removals_by_deployment:
+            print(
+                f"Found removal record for indicated deployment {r.attributes.deployment} at "
+                f"{removals_by_deployment[r.attributes.deployment].id}")
+        else:
+            if r.id not in previous_requests:
+                print(f"Request {r.id} needs to processed.")
+                requests_to_execute.append(r)
+            else:
+                print(f"Skipping unsatisfied request {r.id} because we have seen it before.")
+
+    print("Found %d unsatisfied request(s) to process." % len(requests_to_execute))
+
+    if not dry_run:
+        for r in requests_to_execute:
+            try:
+                process_app_removal_request(
+                    ctx,
+                    laconic,
+                    r,
+                    os.path.abspath(deployment_parent_dir),
+                    delete_volumes,
+                    delete_names
+                )
+            finally:
+                dump_known_requests(state_file, [r])

stack_orchestrator/deploy/webapp/util.py

@@ -49,6 +49,14 @@ def cmd(*vargs):
         raise err
 
 
+def match_owner(recordA, *records):
+    for owner in recordA.owners:
+        for otherRecord in records:
+            if owner in otherRecord.owners:
+                return owner
+    return None
+
+
 class LaconicRegistryClient:
     def __init__(self, config_file):
         self.config_file = config_file
@@ -146,11 +154,17 @@ class LaconicRegistryClient:
             raise Exception("Cannot locate record:", name_or_id)
         return None
 
-    def app_deployment_requests(self):
-        return self.list_records({"type": "ApplicationDeploymentRequest"}, True)
+    def app_deployment_requests(self, all=True):
+        return self.list_records({"type": "ApplicationDeploymentRequest"}, all)
 
-    def app_deployments(self):
-        return self.list_records({"type": "ApplicationDeploymentRecord"})
+    def app_deployments(self, all=True):
+        return self.list_records({"type": "ApplicationDeploymentRecord"}, all)
+
+    def app_deployment_removal_requests(self, all=True):
+        return self.list_records({"type": "ApplicationDeploymentRemovalRequest"}, all)
+
+    def app_deployment_removals(self, all=True):
+        return self.list_records({"type": "ApplicationDeploymentRemovalRecord"}, all)
 
     def publish(self, record, names=[]):
         tmpdir = tempfile.mkdtemp()
@@ -165,16 +179,39 @@ class LaconicRegistryClient:
                 cmd("laconic", "-c", self.config_file, "cns", "record", "publish", "--filename", record_fname)
             )["id"]
             for name in names:
-                cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, new_record_id)
+                self.set_name(name, new_record_id)
             return new_record_id
         finally:
             cmd("rm", "-rf", tmpdir)
 
+    def set_name(self, name, record_id):
+        cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, record_id)
+
+    def delete_name(self, name):
+        cmd("laconic", "-c", self.config_file, "cns", "name", "delete", name)
+
 
 def file_hash(filename):
     return hashlib.sha1(open(filename).read().encode()).hexdigest()
 
 
+def determine_base_container(clone_dir, app_type="webapp"):
+    if not app_type or not app_type.startswith("webapp"):
+        raise Exception(f"Unsupported app_type {app_type}")
+
+    base_container = "cerc/webapp-base"
+    if app_type == "webapp/next":
+        base_container = "cerc/nextjs-base"
+    elif app_type == "webapp":
+        pkg_json_path = os.path.join(clone_dir, "package.json")
+        if os.path.exists(pkg_json_path):
+            pkg_json = json.load(open(pkg_json_path))
+            if "next" in pkg_json.get("dependencies", {}):
+                base_container = "cerc/nextjs-base"
+
+    return base_container
+
+
 def build_container_image(app_record, tag, extra_build_args=[]):
     tmpdir = tempfile.mkdtemp()
 
@@ -187,15 +224,24 @@ def build_container_image(app_record, tag, extra_build_args=[]):
         print(f"Cloning repository {repo} to {clone_dir} ...")
         if ref:
             # TODO: Determing branch or hash, and use depth 1 if we can.
-            result = subprocess.run(["git", "clone", repo, clone_dir])
-            result.check_returncode()
-            subprocess.check_call(["git", "checkout", ref], cwd=clone_dir)
+            git_env = dict(os.environ.copy())
+            # Never prompt
+            git_env["GIT_TERMINAL_PROMPT"] = "0"
+            subprocess.check_call(["git", "clone", repo, clone_dir], env=git_env)
+            subprocess.check_call(["git", "checkout", ref], cwd=clone_dir, env=git_env)
         else:
             result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir])
             result.check_returncode()
 
+        base_container = determine_base_container(clone_dir, app_record.attributes.app_type)
+
         print("Building webapp ...")
-        build_command = [sys.argv[0], "build-webapp", "--source-repo", clone_dir, "--tag", tag]
+        build_command = [
+            sys.argv[0], "build-webapp",
+            "--source-repo", clone_dir,
+            "--tag", tag,
+            "--base-container", base_container
+        ]
         if extra_build_args:
             build_command.append("--extra-build-args")
             build_command.append(" ".join(extra_build_args))

stack_orchestrator/main.py

@@ -20,7 +20,10 @@ from stack_orchestrator.repos import setup_repositories
 from stack_orchestrator.build import build_containers
 from stack_orchestrator.build import build_npms
 from stack_orchestrator.build import build_webapp
-from stack_orchestrator.deploy.webapp import run_webapp, deploy_webapp, deploy_webapp_from_registry
+from stack_orchestrator.deploy.webapp import (run_webapp,
+                                              deploy_webapp,
+                                              deploy_webapp_from_registry,
+                                              undeploy_webapp_from_registry)
 from stack_orchestrator.deploy import deploy
 from stack_orchestrator import version
 from stack_orchestrator.deploy import deployment
@@ -54,6 +57,7 @@ cli.add_command(build_webapp.command, "build-webapp")
 cli.add_command(run_webapp.command, "run-webapp")
 cli.add_command(deploy_webapp.command, "deploy-webapp")
 cli.add_command(deploy_webapp_from_registry.command, "deploy-webapp-from-registry")
+cli.add_command(undeploy_webapp_from_registry.command, "undeploy-webapp-from-registry")
 cli.add_command(deploy.command, "deploy")  # deploy is an alias for deploy-system
 cli.add_command(deploy.command, "deploy-system")
 cli.add_command(deployment.command, "deployment")

tests/k8s-deploy/run-deploy-test.sh

@@ -1,14 +1,59 @@
 #!/usr/bin/env bash
 set -e
 if [ -n "$CERC_SCRIPT_DEBUG" ]; then
-  set -x
+    set -x
+    # Dump environment variables for debugging
+    echo "Environment variables:"
+    env
 fi
+
+# Helper functions: TODO move into a separate file
+wait_for_pods_started () {
+    for i in {1..50}
+    do
+        local ps_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir ps )
+
+        if [[ "$ps_output" == *"Running containers:"* ]]; then
+            # if ready, return
+            return
+        else
+            # if not ready, wait
+            sleep 5
+        fi
+    done
+    # Timed out, error exit
+    echo "waiting for pods to start: FAILED"
+    delete_cluster_exit
+}
+
+wait_for_log_output () {
+    for i in {1..50}
+    do
+
+        local log_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+
+        if [[ ! -z "$log_output" ]]; then
+            # if ready, return
+            return
+        else
+            # if not ready, wait
+            sleep 5
+        fi
+    done
+    # Timed out, error exit
+    echo "waiting for pods log content: FAILED"
+    delete_cluster_exit
+}
+
+
+delete_cluster_exit () {
+    $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
+    exit 1
+}
+
 # Note: eventually this test should be folded into ../deploy/
 # but keeping it separate for now for convenience
 TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
-# Dump environment variables for debugging
-echo "Environment variables:"
-env
 # Set a non-default repo dir
 export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir
 echo "Testing this package: $TEST_TARGET_SO"
@@ -18,7 +63,9 @@ echo "Version reported is: ${reported_version_string}"
 echo "Cloning repositories into: $CERC_REPO_BASE_DIR"
 rm -rf $CERC_REPO_BASE_DIR
 mkdir -p $CERC_REPO_BASE_DIR
-# Test basic stack-orchestrator deploy
+$TEST_TARGET_SO --stack test setup-repositories
+$TEST_TARGET_SO --stack test build-containers
+# Test basic stack-orchestrator deploy to k8s
 test_deployment_dir=$CERC_REPO_BASE_DIR/test-deployment-dir
 test_deployment_spec=$CERC_REPO_BASE_DIR/test-deployment-spec.yml
 $TEST_TARGET_SO --stack test deploy --deploy-to k8s-kind init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED
@@ -53,23 +100,37 @@ fi
 echo "deploy create output file test: passed"
 # Try to start the deployment
 $TEST_TARGET_SO deployment --dir $test_deployment_dir start
-# TODO: add a check to see if the container is up
-# Sleep because k8s not up yet
-sleep 30
+wait_for_pods_started
 # Check logs command works
+wait_for_log_output
 log_output_3=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
 if [[ "$log_output_3" == *"Filesystem is fresh"* ]]; then
     echo "deployment logs test: passed"
 else
     echo "deployment logs test: FAILED"
-    exit 1
+    delete_cluster_exit
 fi
 # Check the config variable CERC_TEST_PARAM_1 was passed correctly
 if [[ "$log_output_3" == *"Test-param-1: PASSED"* ]]; then
     echo "deployment config test: passed"
 else
     echo "deployment config test: FAILED"
-    exit 1
+    delete_cluster_exit
+fi
+# Stop then start again and check the volume was preserved
+$TEST_TARGET_SO deployment --dir $test_deployment_dir stop
+# Sleep a bit just in case
+# sleep for longer to check if that's why the subsequent create cluster fails
+sleep 20
+$TEST_TARGET_SO deployment --dir $test_deployment_dir start
+wait_for_pods_started
+wait_for_log_output
+log_output_4=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_4" == *"Filesystem is old"* ]]; then
+    echo "Retain volumes test: passed"
+else
+    echo "Retain volumes test: FAILED"
+    delete_cluster_exit
 fi
 # Stop and clean up
 $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes

tests/scripts/install-kind.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# TODO: handle ARM
+curl --silent -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
+chmod +x ./kind
+mv ./kind /usr/local/bin

tests/scripts/install-kubectl.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# TODO: handle ARM
+curl --silent -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+chmod +x ./kubectl
+mv ./kubectl /usr/local/bin

tests/webapp-test/run-webapp-test.sh

@@ -30,14 +30,14 @@ CHECK="SPECIAL_01234567890_TEST_STRING"
 
 set +e
 
-CONTAINER_ID=$(docker run -p 3000:3000 -d -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG cerc/test-progressive-web-app:local)
+CONTAINER_ID=$(docker run -p 3000:80 -d -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG cerc/test-progressive-web-app:local)
 sleep 3
 wget -t 7 -O test.before -m http://localhost:3000
 
 docker logs $CONTAINER_ID
 docker remove -f $CONTAINER_ID
 
-CONTAINER_ID=$(docker run -p 3000:3000 -e CERC_WEBAPP_DEBUG=$CHECK -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG -d cerc/test-progressive-web-app:local)
+CONTAINER_ID=$(docker run -p 3000:80 -e CERC_WEBAPP_DEBUG=$CHECK -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG -d cerc/test-progressive-web-app:local)
 sleep 3
 wget -t 7 -O test.after -m http://localhost:3000