birbit stack

Merge branch 'main' of github.com:cerc-io/stack-orchestrator
Set k8s deploy test to manual trigger
2024-01-26 17:44:31 +00:00 · 2024-01-17 08:27:38 -07:00 · 2024-01-17 08:27:09 -07:00 · 2024-01-17 08:26:51 -07:00 · 2024-01-17 14:02:10 +05:30 · 2024-01-16 21:15:21 -07:00
73 changed files with 33329 additions and 150 deletions
@@ -1,23 +1,16 @@
 name: K8s Deploy Test

 on:
-  pull_request:
-    branches: '*'
  push:
-    branches:
-      - main
-      - ci-test
-    paths-ignore:
-      - '.gitea/workflows/triggers/*'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
+    branches: '*'
+    paths:
+      - '!**'
+      - '.gitea/workflows/triggers/fixturenet-laconicd-test'

 jobs:
  test:
-    name: "Run deploy test suite"
-    runs-on: ubuntu-latest
+    name: "Run deploy test suite on kind/k8s"
+    runs-on: ubuntu-22.04-with-syn-ethdb
    steps:
      - name: "Clone project repository"
        uses: actions/checkout@v3
@@ -41,15 +34,9 @@ jobs:
        run: ./scripts/create_build_tag_file.sh
      - name: "Build local shiv package"
        run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
-      - name: "Install Go"
-        uses: actions/setup-go@v4
-        with:
-          go-version: '1.21'
-      - name: "Install Kind"
-        run: go install sigs.k8s.io/kind@v0.20.0
-      - name: "Debug Kind"
-        run: kind create cluster --retain && docker logs kind-control-plane
+      - name: "Install kind"
+        run: ./tests/scripts/install-kind.sh
+      - name: "Install Kubectl"
+        run: ./tests/scripts/install-kubectl.sh
+      - name: "Run k8s deployment test"
+        run: ./tests/k8s-deploy/run-deploy-test.sh
@@ -41,6 +41,8 @@ jobs:
        run: ./scripts/create_build_tag_file.sh
      - name: "Build local shiv package"
        run: ./scripts/build_shiv_package.sh
+      - name: "Install wget" # 20240109 - Only needed until the executors are updated.
+        run: apt update && apt install -y wget
      - name: Start dockerd # Also needed until we can incorporate into the executor
        run: |
          dockerd -H $DOCKER_HOST --userland-proxy=false &
@@ -0,0 +1 @@
+Change this file to trigger running the test-k8s-deploy CI job
@@ -2,7 +2,7 @@ services:
  laconicd:
    restart: unless-stopped
    image: cerc/laconicd:local
-    command: ["sh", "/docker-entrypoint-scripts.d/create-fixturenet.sh"]
+    entrypoint: ["sh", "/docker-entrypoint-scripts.d/create-fixturenet.sh"]
    volumes:
      # The cosmos-sdk node's database directory:
      - laconicd-data:/root/.laconicd
@@ -15,12 +15,12 @@ services:
      - "6060"
      - "26657"
      - "26656"
-      - "9473"
+      - "9473:9473"
      - "8545"
      - "8546"
      - "9090"
      - "9091"
-      - "1317"
+      - "1317:1317"
  cli:
    image: cerc/laconic-registry-cli:local
    volumes:
@@ -0,0 +1,8 @@
+version: "3.2"
+
+services:
+  geojson:
+    image: cerc/geojson:local
+    restart: always
+    ports:
+      - 0.0.0.0:8080:8080
@@ -0,0 +1,23 @@
+version: "3.7"
+
+services:
+  grafana:
+    image: grafana/grafana:10.2.2
+    restart: always
+    environment:
+      GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL}
+    volumes:
+      - ../config/monitoring/grafana/provisioning:/etc/grafana/provisioning
+      - ../config/monitoring/grafana/dashboards:/etc/grafana/dashboards
+      - grafana_storage:/var/lib/grafana
+    ports:
+      - "3000"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "3000"]
+      interval: 30s
+      timeout: 5s
+      retries: 10
+      start_period: 3s
+
+volumes:
+  grafana_storage:
@@ -0,0 +1,15 @@
+version: '3.8'
+
+services:
+  node-exporter:
+    image: prom/node-exporter:latest
+    restart: unless-stopped
+    command:
+      - '--web.listen-address=:9100'
+      - '--path.rootfs=/host'
+      - '--collector.systemd'
+      - '--collector.processes'
+    network_mode: host
+    pid: host
+    volumes:
+      - '/:/host:ro,rslave'
@@ -0,0 +1,22 @@
+version: "3.2"
+
+services:
+  osmosis-front-end:
+    image: cerc/osmosis-front-end-urbit:local
+    restart: on-failure
+    environment:
+      - NEXT_PUBLIC_WEB_API_BASE_URL=${CERC_WEB_API_BASE_URL}
+      - ASSET_LIST_COMMIT_HASH=a326bcefc51372b4912be5a2a2fa84a5d142a438
+      - NEXT_PUBLIC_BASEPATH=/apps/osmosis
+      - NEXT_PUBLIC_URBIT_DEPLOYMENT=true
+    working_dir: /app/packages/web
+    command: ["./build-app-for-urbit.sh"]
+    volumes:
+      - ../config/osmosis/build-app-for-urbit.sh:/app/packages/web/build-app-for-urbit.sh
+      - ../config/osmosis/.env.production:/app/packages/web/.env.production
+      - urbit_app_builds:/app-builds
+      - ../config/osmosis/urbit-files/mar:/app/packages/web/mar
+      - ../config/osmosis/urbit-files/desk.docket-0:/app/packages/web/desk.docket-0
+
+volumes:
+  urbit_app_builds:
@@ -6,10 +6,12 @@ services:
    restart: on-failure
    environment:
      - NEXT_PUBLIC_WEB_API_BASE_URL=${CERC_WEB_API_BASE_URL}
+      - ASSET_LIST_COMMIT_HASH=a326bcefc51372b4912be5a2a2fa84a5d142a438
    working_dir: /app/packages/web
    command: ["./build-app.sh"]
    volumes:
      - ../config/osmosis/build-app.sh:/app/packages/web/build-app.sh
+      - ../config/osmosis/.env.production:/app/packages/web/.env.production
      - app_builds:/app-builds

  nginx:
@@ -0,0 +1,57 @@
+version: "3.7"
+
+services:
+  prometheus:
+    image: prom/prometheus:v2.49.1
+    restart: always
+    volumes:
+      - ../config/monitoring/prometheus:/etc/prometheus
+      - prometheus_data:/prometheus
+    ports:
+      - "9090"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "9090"]
+      interval: 30s
+      timeout: 5s
+      retries: 10
+      start_period: 3s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  blackbox:
+    image: prom/blackbox-exporter:latest
+    restart: always
+    volumes:
+      - ../config/monitoring/blackbox.yml:/etc/blackbox_exporter/config.yml
+    ports:
+      - '9115'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  chain-head-exporter:
+    image: cerc/watcher-ts:local
+    restart: always
+    working_dir: /app/packages/cli
+    environment:
+      ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      FIL_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT}
+      ETH_RPC_API_KEY: ${CERC_INFURA_KEY}
+      PORT: ${CERC_METRICS_PORT}
+    command: ["sh", "-c", "yarn export-metrics:chain-heads"]
+    ports:
+      - '5000'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  postgres-exporter:
+    image: quay.io/prometheuscommunity/postgres-exporter
+    restart: always
+    volumes:
+      - ../config/monitoring/postgres-exporter.yml:/postgres_exporter.yml
+    ports:
+      - '9187'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+volumes:
+  prometheus_data:
@@ -0,0 +1,7 @@
+modules:
+  http_2xx:
+    prober: http
+    timeout: 5s
+    http:
+      valid_status_codes: [] #default to 2xx
+      method: GET
@@ -0,0 +1,943 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "node.js prometheus client basic metrics",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "gnetId": 11159,
+  "graphTooltip": 0,
+  "id": 15,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 10,
+        "x": 0,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 6,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "irate(process_cpu_user_seconds_total{instance=~\"$instance\"}[2m]) * 100",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "User CPU - {{instance}}",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "irate(process_cpu_system_seconds_total{instance=~\"$instance\"}[2m]) * 100",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Sys CPU - {{instance}}",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Process CPU Usage",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "percent",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 9,
+        "x": 10,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 8,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_eventloop_lag_seconds{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "{{instance}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Event Loop Lag",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "s",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "text",
+            "mode": "fixed"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 5,
+        "x": 19,
+        "y": 0
+      },
+      "id": 2,
+      "interval": "",
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "name",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.2.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(nodejs_version_info{instance=~\"$instance\"}) by (version)",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{version}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Node.js Version",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "#F2495C",
+            "mode": "fixed"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 5,
+        "x": 19,
+        "y": 3
+      },
+      "id": 4,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.2.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(changes(process_start_time_seconds{instance=~\"$instance\"}[1m]))",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "{{instance}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Process Restart Times",
+      "type": "stat"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 16,
+        "x": 0,
+        "y": 7
+      },
+      "hiddenSeries": false,
+      "id": 7,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "rightSide": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "process_resident_memory_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Process Memory - {{instance}}",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_heap_size_total_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Heap Total - {{instance}}",
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_heap_size_used_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Heap Used - {{instance}}",
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_external_memory_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "External Memory - {{instance}}",
+          "refId": "D"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Process Memory Usage",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 8,
+        "x": 16,
+        "y": 7
+      },
+      "hiddenSeries": false,
+      "id": 9,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_active_handles_total{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Active Handler - {{instance}}",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_active_requests_total{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Active Request - {{instance}}",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Active Handlers/Requests Total",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 14
+      },
+      "hiddenSeries": false,
+      "id": 10,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_heap_space_size_total_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Heap Total - {{instance}} - {{space}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Heap Total Detail",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 8,
+        "y": 14
+      },
+      "hiddenSeries": false,
+      "id": 11,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_heap_space_size_used_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Heap Used - {{instance}} - {{space}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Heap Used Detail",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 16,
+        "y": 14
+      },
+      "hiddenSeries": false,
+      "id": 12,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_heap_space_size_available_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Heap Used - {{instance}} - {{space}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Heap Available Detail",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    }
+  ],
+  "refresh": "10s",
+  "schemaVersion": 38,
+  "tags": [
+    "nodejs"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": true,
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
+        "definition": "label_values(nodejs_version_info, instance)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "instance",
+        "multi": true,
+        "name": "instance",
+        "options": [],
+        "query": "label_values(nodejs_version_info, instance)",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-15m",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "",
+  "title": "NodeJS Application Dashboard",
+  "uid": "PTSqcpJWk",
+  "version": 3,
+  "weekStart": ""
+}
@@ -0,0 +1,14 @@
+# https://www.clever-cloud.com/blog/features/2021/12/03/slack-alerts-for-grafana/
+
+apiVersion: 1
+
+contactPoints:
+  - orgId: 1
+    name: SlackNotifier
+    receivers:
+      - uid: a71b06e3-58b6-41fe-af65-fbbb29653951
+        type: slack
+        settings:
+          # Slack hook URL (see https://api.slack.com/messaging/webhooks)
+          url: <YOUR_SLACK_HOOK_URL>
+        disableResolveMessage: false
@@ -0,0 +1,15 @@
+# https://grafana.com/docs/grafana/latest/alerting/alerting-rules/create-notification-policy/
+
+apiVersion: 1
+
+policies:
+  - orgId: 1
+    receiver: grafana-default-email
+    group_by:
+      - grafana_folder
+      - alertname
+    routes:
+      - receiver: SlackNotifier
+        object_matchers:
+          # Add matchers below
+          # - ['grafana_folder', '=', 'MyAlerts']
@@ -0,0 +1,10 @@
+apiVersion: 1
+
+providers:
+  - name: dashboards
+    type: file
+    updateIntervalSeconds: 10
+    allowUiUpdates: true
+    options:
+      path: /etc/grafana/dashboards
+      foldersFromFilesStructure: true
@@ -0,0 +1,16 @@
+apiVersion: 1
+
+datasources:
+  - id: 1
+    orgId: 1
+    name: Prometheus
+    type: prometheus
+    typeName: Prometheus
+    typeLogoUrl: public/app/plugins/datasource/prometheus/img/prometheus_logo.svg
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    jsonData:
+      httpMethod: POST
+    version: 1
+    editable: true
@@ -0,0 +1,8 @@
+auth_modules:
+  foo:
+    type: userpass
+    userpass:
+      username: username
+      password: password
+    options:
+      sslmode: disable
@@ -0,0 +1,67 @@
+global:
+  scrape_interval:     10s
+  evaluation_interval: 15s
+
+rule_files:
+  # - "first.rules"
+  # - "second.rules"
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: node
+    static_configs:
+      # Add node-exporter targets to be monitored below
+      # - targets: ['example-host:9100']
+      #   labels:
+      #     instance: 'my-host'
+
+  - job_name: 'blackbox'
+    scrape_interval: 10s
+    metrics_path: /probe
+    params:
+      module: [http_2xx]
+    static_configs:
+      # Add URLs to be monitored below
+      - targets:
+        # - https://github.com
+    relabel_configs:
+      - source_labels: [__address__]
+        regex: (.*)(:80)?
+        target_label: __param_target
+      - source_labels: [__param_target]
+        regex: (.*)
+        target_label: instance
+        replacement: ${1}
+      - source_labels: []
+        regex: .*
+        target_label: __address__
+        replacement: blackbox:9115
+
+  - job_name: chain_heads
+    scrape_interval: 10s
+    metrics_path: /metrics
+    scheme: http
+    static_configs:
+      - targets: ['chain-head-exporter:5000']
+
+  - job_name: 'postgres'
+    scrape_interval: 30s
+    scrape_timeout: 30s
+    static_configs:
+      # Add DB targets below
+      # - targets: [example-server:5432]
+        # labels:
+          # instance: 'example-label'
+    metrics_path: /probe
+    params:
+      auth_module: [foo]
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: postgres-exporter:9187
@@ -0,0 +1,933 @@
+# https://grafana.com/docs/grafana/latest/alerting/alerting-rules/create-grafana-managed-rule/
+
+apiVersion: 1
+
+groups:
+  - orgId: 1
+    name: watcher
+    folder: WatcherAlerts
+    interval: 30s
+    rules:
+      # Azimuth
+      - uid: azimuth_diff_external
+        title: azimuth_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: censures_diff_external
+        title: censures_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: claims_diff_external
+        title: claims_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: conditional_star_release_diff_external
+        title: conditional_star_release_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: delegated_sending_diff_external
+        title: delegated_sending_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: ecliptic_diff_external
+        title: ecliptic_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: linear_star_release_diff_external
+        title: linear_star_release_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="linear_star_release", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: polls_diff_external
+        title: polls_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+
+      # Sushi
+      - uid: sushiswap_diff_external
+        title: sushiswap_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: merkl_sushiswap_diff_external
+        title: merkl_sushiswap_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
@@ -0,0 +1,3 @@
+NEXT_PUBLIC_COINGECKO_URL=https://api.coingecko.com
+
+NEXT_PUBLIC_ENABLE_FEATURES=true
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+# Check and exit if a deployment already exists (on restarts)
+if [ -d /app-builds/osmosis/build ]; then
+  echo "Build already exists, remove volume to rebuild"
+  exit 0
+fi
+
+yarn build:static
+./build-urbit.sh
+
+# Move build to app-builds
+mkdir -p /app-builds/osmosis
+cp -r ./out /app-builds/osmosis/build
+
+cp -r mar /app-builds/osmosis/
+cp desk.docket-0 /app-builds/osmosis/
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash

 set -e
 if [ -n "$CERC_SCRIPT_DEBUG" ]; then
@@ -0,0 +1,10 @@
+:~  title+'Osmosis'
+    info+'Osmosis DEX Frontend, built and maintained by Laconic'
+    color+0xcd.75df
+    image+'https://altcoinsbox.com/wp-content/uploads/2023/03/osmosis-logo.png'
+    base+'osmosis'
+    glob-http+['REPLACE_WITH_GLOB_URL' REPLACE_WITH_GLOB_HASH]
+    version+[0 0 2]
+    website+'https://osmosis.zone/'
+    license+'MIT'
+==
@@ -0,0 +1,12 @@
+|_  dat=@
++  grow
+  |%
+  ++  mime  [/image/x-icon (as-octs:mimes:html dat)]
+  --
++  grab
+  |%
+  ++  mime  |=([p=mite q=octs] q.q)
+  ++  noun  @
+  --
++  grad  %mime
+--
@@ -0,0 +1,12 @@
+|_  dat=@
++  grow
+  |%
+  ++  mime  [/image/jpeg (as-octs:mimes:html dat)]
+  --
++  grab
+  |%
+  ++  mime  |=([p=mite q=octs] q.q)
+  ++  noun  @
+  --
++  grad  %mime
+--
@@ -0,0 +1,18 @@
+::
+::::  /hoon/map/mar
+  ::  Mark for js source maps
+/?    310
+::
+=,  eyre
+|_  mud=@
++  grow
+  |%
+  ++  mime  [/application/octet-stream (as-octs:mimes:html (@t mud))]
+  --
++  grab
+  |%                                                    ::  convert from
+  ++  mime  |=([p=mite q=octs] (@t q.q))
+  ++  noun  cord                                        ::  clam from %noun
+  --
++  grad  %mime
+--
@@ -0,0 +1,12 @@
+|_  dat=@
++  grow
+  |%
+  ++  mime  [/image/webp (as-octs:mimes:html dat)]
+  --
++  grab
+  |%
+  ++  mime  |=([p=mite q=octs] q.q)
+  ++  noun  @
+  --
++  grad  %mime
+--
@@ -84,8 +84,6 @@
  subgraphEventsOrder = true
  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
  blockDelayInMilliSecs = 30000
-  prefetchBlocksInMem = false
-  prefetchBlockCount = 10

  # Boolean to switch between modes of processing events when starting the server.
  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
@@ -84,8 +84,6 @@
  subgraphEventsOrder = true
  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
  blockDelayInMilliSecs = 30000
-  prefetchBlocksInMem = false
-  prefetchBlockCount = 10

  # Boolean to switch between modes of processing events when starting the server.
  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
@@ -2,4 +2,6 @@
 # Build a local version of the task executor for act-runner
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-docker build -t cerc/act-runner-task-executor:local -f ${CERC_REPO_BASE_DIR}/hosting/act-runner/Dockerfile.task-executor ${build_command_args} ${SCRIPT_DIR}
+
+cd ${CERC_REPO_BASE_DIR}/hosting/act-runner
+docker build -t cerc/act-runner-task-executor:local -f Dockerfile.task-executor ${build_command_args} .
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build the birbit image
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+docker build -t cerc/birbit:local -f ${CERC_REPO_BASE_DIR}/birbit/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/birbit
@@ -0,0 +1,11 @@
+FROM node:14
+
+WORKDIR /app
+
+COPY . .
+
+RUN npm install && npm run build
+
+EXPOSE 8080
+
+CMD ["npm", "run", "serve"]
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Build the geojson image
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+docker build -t cerc/geojson:local -f ${CERC_CONTAINER_BASE_DIR}/cerc-geojson/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/geojson.io
+#docker build -t cerc/geojson:local -f ${CERC_REPO_BASE_DIR}/geojson.io/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/geojson.io
@@ -1,6 +1,6 @@
 # Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile
 # [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
-ARG VARIANT=20-bullseye
+ARG VARIANT=20-bullseye-slim
 FROM node:${VARIANT}

 ARG USERNAME=node
@@ -30,7 +30,7 @@ RUN \

 # [Optional] Uncomment this section to install additional OS packages.
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends jq gettext-base moreutils
+    && apt-get -y install --no-install-recommends jq gettext-base

 # [Optional] Uncomment if you want to install more global node modules
 # RUN su node -c "npm install -g <your-package-list-here>"
@@ -21,6 +21,11 @@ WORK_DIR="${1:-/app}"

 cd "${WORK_DIR}" || exit 1

+# If this file doesn't exist at all, we'll get errors below.
+if [ ! -f "next.config.js" ]; then
+  touch next.config.js
+fi
+
 if [ ! -f "next.config.dist" ]; then
  cp next.config.js next.config.dist
 fi
@@ -104,7 +109,8 @@ CUR_NEXT_VERSION="`jq -r '.dependencies.next' package.json`"

 if [ "$CERC_NEXT_VERSION" != "keep" ] && [ "$CUR_NEXT_VERSION" != "$CERC_NEXT_VERSION" ]; then
  echo "Changing 'next' version specifier from '$CUR_NEXT_VERSION' to '$CERC_NEXT_VERSION' (set with '--extra-build-args \"--build-arg CERC_NEXT_VERSION=$CERC_NEXT_VERSION\"')"
-  cat package.json | jq ".dependencies.next = \"$CERC_NEXT_VERSION\"" | sponge package.json
+  cat package.json | jq ".dependencies.next = \"$CERC_NEXT_VERSION\"" > package.json.$$
+  mv package.json.$$ package.json
 fi

 $CERC_BUILD_TOOL install || exit 1
@@ -128,7 +134,8 @@ to use for the build with:
 ###############################################################################

 EOF
-  cat package.json | jq ".dependencies.next = \"^$CERC_MIN_NEXTVER\"" | sponge package.json
+  cat package.json | jq ".dependencies.next = \"^$CERC_MIN_NEXTVER\"" > package.json.$$
+  mv package.json.$$ package.json
  $CERC_BUILD_TOOL install || exit 1
 fi

@@ -42,7 +42,7 @@ if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then
    while [ $count -lt $CERC_MAX_GENERATE_TIME ] && [ "$generate_done" == "false" ]; do
      sleep 1
      count=$((count + 1))
-      grep 'rendered as static HTML' gen.out > /dev/null
+      grep 'rendered as static' gen.out > /dev/null
      if [ $? -eq 0 ]; then
        generate_done="true"
      fi
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build the osmosis front end image
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+docker build -t cerc/osmosis-front-end-urbit:local -f ${CERC_REPO_BASE_DIR}/osmosis-frontend/docker/Dockerfile.static ${build_command_args} ${CERC_REPO_BASE_DIR}/osmosis-frontend
@@ -1,7 +0,0 @@
-FROM python:3.13.0a2-alpine3.18
-
-RUN apk --update --no-cache add alpine-sdk jq bash curl wget
-
-WORKDIR /app
-
-ENTRYPOINT [ "bash" ]
@@ -1,9 +0,0 @@
-#!/usr/bin/env bash
-# Build the urbit-globs-host image
-
-source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
-
-# See: https://stackoverflow.com/a/246128/1701505
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-
-docker build -t cerc/urbit-globs-host:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${SCRIPT_DIR}
@@ -3,7 +3,7 @@ name: act-runner
 description: "Local act-runner"
 repos:
  - git.vdb.to/cerc-io/hosting
-  - gitea.com/gitea/act_runner
+  - gitea.com/telackey/act_runner@telackey/entrypoint
 containers:
  - cerc/act-runner
  - cerc/act-runner-task-executor
@@ -40,20 +40,36 @@ network:
  ports:
    watcher-db:
     - 0.0.0.0:15432:5432
+    azimuth-watcher-job-runner:
+     - 0.0.0.0:9000:9000
    azimuth-watcher-server:
     - 0.0.0.0:3001:3001
+    censures-watcher-job-runner:
+     - 0.0.0.0:9002:9002
    censures-watcher-server:
     - 0.0.0.0:3002:3002
+    claims-watcher-job-runner:
+     - 0.0.0.0:9004:9004
    claims-watcher-server:
     - 0.0.0.0:3003:3003
+    conditional-star-release-watcher-job-runner:
+     - 0.0.0.0:9006:9006
    conditional-star-release-watcher-server:
     - 0.0.0.0:3004:3004
+    delegated-sending-watcher-job-runner:
+     - 0.0.0.0:9008:9008
    delegated-sending-watcher-server:
     - 0.0.0.0:3005:3005
+    ecliptic-watcher-job-runner:
+     - 0.0.0.0:9010:9010
    ecliptic-watcher-server:
     - 0.0.0.0:3006:3006
+    linear-star-release-watcher-job-runner:
+     - 0.0.0.0:9012:9012
    linear-star-release-watcher-server:
     - 0.0.0.0:3007:3007
+    polls-watcher-job-runner:
+     - 0.0.0.0:9014:9014
    polls-watcher-server:
     - 0.0.0.0:3008:3008
    gateway-server:
@@ -1,7 +1,7 @@
 version: "1.0"
 name: azimuth
 repos:
-  - github.com/cerc-io/azimuth-watcher-ts@v0.1.2
+  - github.com/cerc-io/azimuth-watcher-ts@v0.1.3
 containers:
  - cerc/watcher-azimuth
 pods:
@@ -0,0 +1,3 @@
+# birbit
+
+takes geo-tagged images, publishes the metadata on chain, then generates a geojson front end of that geo data
@@ -0,0 +1,19 @@
+version: "0.1"
+name: birbit
+repos:
+  #- github.com/zramsay/birbit
+  - github.com/mapbox/geojson.io
+containers:
+  #- cerc/birbit
+  - cerc/geojson
+pods:
+    # configurable process that reads from chain and generates a view for the front end
+    # for MVP is basically does nothing, b/c FE view is  reading all records
+    # imagine a 'watcher' for a specific view
+    #- birbit
+    # front end, gets its file from kubo;
+    # https://github.com/mapbox/geojson.io/blob/main/API.md#datadatatextx-url
+    # deployed via kubernetes
+  - geojson
+    # hosts all the dot geojson files that are required to generated views
+  - kubo
@@ -2,7 +2,7 @@ version: "1.0"
 name: merkl-sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.4
+  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.6
 containers:
  - cerc/watcher-merkl-sushiswap-v3
 pods:
@@ -0,0 +1,180 @@
+# monitoring
+
+* Instructions to setup and run a Prometheus server and Grafana dashboard
+* Comes with the following built-in exporters / dashboards:
+  * Chain Head Exporter - for tracking chain heads given external ETH RPC endpoints
+  * Watchers dashboard
+  * [Prometheus Blackbox](https://grafana.com/grafana/dashboards/7587-prometheus-blackbox-exporter/) - for tracking HTTP endpoints
+  * [NodeJS Application Dashboard](https://grafana.com/grafana/dashboards/11159-nodejs-application-dashboard/) - for default NodeJS metrics
+  * [PostgreSQL Database](https://grafana.com/grafana/dashboards/9628-postgresql-database/) - for monitoring Postgres dbs
+  * [Node Exporter Full](https://grafana.com/grafana/dashboards/1860-node-exporter-full/) - for monitoring system metrics
+* See [monitoring-watchers.md](./monitoring-watchers.md) for an example usage of the stack with pre-configured dashboards for watchers
+
+## Setup
+
+Clone required repositories:
+
+```bash
+laconic-so --stack monitoring setup-repositories --git-ssh --pull
+```
+
+Build the container images:
+
+```bash
+laconic-so --stack monitoring build-containers
+```
+
+## Create a deployment
+
+First, create a spec file for the deployment, which will map the stack's ports and volumes to the host:
+
+```bash
+laconic-so --stack monitoring deploy init --output monitoring-spec.yml
+```
+
+### Ports
+
+Edit `network` in spec file to map container ports to same ports in host:
+
+```
+...
+network:
+  ports:
+    prometheus:
+      - '9090:9090'
+    grafana:
+      - '3000:3000'
+...
+```
+
+### Data volumes
+
+Container data volumes are bind-mounted to specified paths in the host filesystem.
+The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory. The default mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`.
+
+---
+
+Once you've made any needed changes to the spec file, create a deployment from it:
+
+```bash
+laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --deployment-dir monitoring-deployment
+```
+
+## Configure
+
+### Prometheus Config
+
+* Add desired scrape configs to prometheus config file (`monitoring-deployment/config/monitoring/prometheus/prometheus.yml`) in the deployment folder; for example:
+
+  ```yml
+  ...
+  - job_name: <JOB_NAME>
+    metrics_path: /metrics/path
+    scheme: http
+    static_configs:
+      - targets: ['<METRICS_ENDPOINT_HOST>:<METRICS_ENDPOINT_PORT>']
+  ```
+
+* Node exporter: update the `node` job to add any node-exporter targets to be monitored:
+
+  ```yml
+  ...
+  - job_name: 'node'
+    ...
+    static_configs:
+      # Add node-exporter targets to be monitored below
+      - targets: [example-host:9100]
+        labels:
+          instance: 'my-host'
+  ```
+
+* Blackbox (in-stack exporter): update the `blackbox` job to add any endpoints to be monitored on the Blackbox dashboard:
+
+  ```yml
+  ...
+  - job_name: 'blackbox'
+    ...
+    static_configs:
+      # Add URLs to be monitored below
+      - targets:
+        - <HTTP_ENDPOINT_1>
+        - <HTTP_ENDPOINT_2>
+  ```
+
+* Postgres (in-stack exporter):
+  * Update the `postgres` job to add Postgres db targets to be monitored:
+
+    ```yml
+    ...
+    - job_name: 'postgres'
+      ...
+      static_configs:
+        # Add DB targets below
+        - targets: [example-server:5432]
+          labels:
+            instance: 'example-db'
+    ```
+  * Add database credentials to be used in `auth_modules` in the postgres-exporter config file (`monitoring-deployment/config/monitoring/postgres-exporter.yml`)
+
+Note: Use `host.docker.internal` as host to access ports on the host machine
+
+### Grafana Config
+
+Place the dashboard json files in grafana dashboards config directory (`monitoring-deployment/config/monitoring/grafana/dashboards`) in the deployment folder
+
+### Env
+
+Set the following env variables in the deployment env config file (`monitoring-deployment/config.env`):
+
+  ```bash
+  # For chain-head exporter
+
+  # External ETH RPC endpoint (ethereum)
+  # (Optional, default: https://mainnet.infura.io/v3)
+  CERC_ETH_RPC_ENDPOINT=
+
+  # Infura key to be used
+  # (Optional, used with ETH_RPC_ENDPOINT if provided)
+  CERC_INFURA_KEY=
+
+  # External ETH RPC endpoint (filecoin)
+  # (Optional, default: https://api.node.glif.io/rpc/v1)
+  CERC_FIL_RPC_ENDPOINT=
+
+  # Grafana server host URL (used in various links in alerts, etc.)
+  # (Optional, default: http://localhost:3000)
+  GF_SERVER_ROOT_URL=
+  ```
+
+## Start the stack
+
+Start the deployment:
+
+```bash
+laconic-so deployment --dir monitoring-deployment start
+```
+
+* List and check the health status of all the containers using `docker ps` and wait for them to be `healthy`
+
+* Grafana should now be visible at http://localhost:3000 with configured dashboards
+
+## Clean up
+
+To stop monitoring services running in the background, while preserving data:
+
+```bash
+# Only stop the docker containers
+laconic-so deployment --dir monitoring-deployment stop
+
+# Run 'start' to restart the deployment
+```
+
+To stop monitoring services and also delete data:
+
+```bash
+# Stop the docker containers
+laconic-so deployment --dir monitoring-deployment stop --delete-volumes
+
+# Remove deployment directory (deployment will have to be recreated for a re-run)
+rm -rf monitoring-deployment
+```
@@ -0,0 +1,170 @@
+# Monitoring Watchers
+
+Instructions to setup and run monitoring stack with configured Watchers dashboard
+
+## Create a deployment
+
+After completing [setup](./README.md#setup), create a spec file for the deployment, which will map the stack's ports and volumes to the host:
+
+```bash
+laconic-so --stack monitoring deploy init --output monitoring-watchers-spec.yml
+```
+
+### Ports
+
+Edit `network` in spec file to map container ports to same ports in host:
+
+```
+...
+network:
+  ports:
+    prometheus:
+      - '9090:9090'
+    grafana:
+      - '3000:3000'
+...
+```
+
+---
+
+Once you've made any needed changes to the spec file, create a deployment from it:
+
+```bash
+laconic-so --stack monitoring deploy create --spec-file monitoring-watchers-spec.yml --deployment-dir monitoring-watchers-deployment
+```
+
+## Configure
+
+### Prometheus scrape config
+
+Add the following scrape configs to prometheus config file (`monitoring-watchers-deployment/config/monitoring/prometheus/prometheus.yml`) in the deployment folder:
+
+  ```yml
+  ...
+  - job_name: 'blackbox'
+    ...
+    static_configs:
+      - targets:
+        - <AZIMUTH_GATEWAY_GQL_ENDPOINT>
+  ...
+  - job_name: azimuth
+    scrape_interval: 10s
+    metrics_path: /metrics
+    scheme: http
+    static_configs:
+      - targets: ['AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_PORT']
+        labels:
+          instance: 'azimuth'
+          chain: 'ethereum'
+      - targets: ['CENSURES_WATCHER_HOST:CENSURES_WATCHER_PORT']
+        labels:
+          instance: 'censures'
+          chain: 'ethereum'
+      - targets: ['CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_PORT']
+        labels:
+          instance: 'claims'
+          chain: 'ethereum'
+      - targets: ['CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_PORT']
+        labels:
+          instance: 'conditional_star_release'
+          chain: 'ethereum'
+      - targets: ['DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_PORT']
+        labels:
+          instance: 'delegated_sending'
+          chain: 'ethereum'
+      - targets: ['ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_PORT']
+        labels:
+          instance: 'ecliptic'
+          chain: 'ethereum'
+      - targets: ['LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_PORT']
+        labels:
+          instance: 'linear_star_release'
+          chain: 'ethereum'
+      - targets: ['POLLS_WATCHER_HOST:POLLS_WATCHER_PORT']
+        labels:
+          instance: 'polls'
+          chain: 'ethereum'
+
+  - job_name: sushi
+    scrape_interval: 20s
+    metrics_path: /metrics
+    scheme: http
+    static_configs:
+      - targets: ['SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_PORT']
+        labels:
+          instance: 'sushiswap'
+          chain: 'filecoin'
+      - targets: ['MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_PORT']
+        labels:
+          instance: 'merkl_sushiswap'
+          chain: 'filecoin'
+  ```
+
+Add scrape config as done above for any additional watcher to add it to the Watchers dashboard.
+
+### Grafana alerts config
+
+Place the pre-configured watcher alerts rules in Grafana provisioning directory:
+
+  ```bash
+  cp monitoring-watchers-deployment/config/monitoring/watcher-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
+  ```
+
+Update the alerting contact points config (`monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/contactpoints.yml`) with desired contact points
+
+Add corresponding routes to the notification policies config (`monitoring-watchers-deployment/monitoring/grafana/provisioning/alerting/policies.yaml`) with appropriate object-matchers:
+
+  ```yml
+  ...
+    routes:
+      - receiver: SlackNotifier
+      object_matchers:
+        # Add matchers below
+        - ['grafana_folder', '=', 'WatcherAlerts']
+  ```
+
+### Env
+
+Set the following env variables in the deployment env config file (`monitoring-watchers-deployment/config.env`):
+
+  ```bash
+  # Infura key to be used
+  CERC_INFURA_KEY=
+
+  # Grafana server host URL to be used
+  # (Optional, default: http://localhost:3000)
+  GF_SERVER_ROOT_URL=
+  ```
+
+## Start the stack
+
+Start the deployment:
+
+```bash
+laconic-so deployment --dir monitoring-watchers-deployment start
+```
+
+* List and check the health status of all the containers using `docker ps` and wait for them to be `healthy`
+
+* Grafana should now be visible at http://localhost:3000 with configured dashboards
+
+## Clean up
+
+To stop monitoring services running in the background, while preserving data:
+
+```bash
+# Only stop the docker containers
+laconic-so deployment --dir monitoring-watchers-deployment stop
+
+# Run 'start' to restart the deployment
+```
+
+To stop monitoring services and also delete data:
+
+```bash
+# Stop the docker containers
+laconic-so deployment --dir monitoring-watchers-deployment stop --delete-volumes
+
+# Remove deployment directory (deployment will have to be recreated for a re-run)
+rm -rf monitoring-watchers-deployment
+```
@@ -0,0 +1,9 @@
+version: "0.1"
+name: monitoring
+repos:
+  - github.com/cerc-io/watcher-ts@v0.2.79
+containers:
+  - cerc/watcher-ts
+pods:
+  - prom-server
+  - grafana
@@ -0,0 +1,17 @@
+# node-exporter
+
+## Start the stack
+
+```bash
+laconic-so --stack node-exporter deploy up
+```
+
+* The host node's metrics can be accessed at `http://localhost:9100/metrics`
+
+## Clean up
+
+Stop the node-exporter running in background:
+
+```bash
+laconic-so --stack node-exporter deploy down
+```
@@ -0,0 +1,6 @@
+version: "0.1"
+name: node-exporter
+repos:
+containers:
+pods:
+  - node-exporter
@@ -0,0 +1,158 @@
+# Self-hosted Osmosis Front End On Urbit
+
+Instructions to build and deploy:
+- Urbit
+- An ipfs node
+- The osmosis front end
+
+## Setup
+
+Clone required repositories:
+
+```bash
+laconic-so --stack osmosis-urbit-app setup-repositories --pull
+
+# If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories and re-run the command
+```
+
+Build the container images:
+
+```bash
+laconic-so --stack osmosis-urbit-app build-containers
+```
+
+## Create a deployment
+
+First, create a spec file for the deployment, which will map the stack's ports and volumes to the host:
+
+```bash
+laconic-so --stack osmosis-urbit-app deploy init --output osmosis-urbit-app-spec.yml
+```
+
+### Ports
+
+Edit `network` in spec file to map container ports to same ports in host:
+
+```yml
+...
+network:
+  ports:
+    urbit-fake-ship:
+      - '8080:80'
+    proxy-server:
+      - '4000:4000'
+    ipfs:
+      - '8081:8080'
+      - '5001:5001'
+```
+
+Note: Skip the `ipfs` ports if need to use an externally running IPFS node
+
+### Data volumes
+
+Container data volumes are bind-mounted to specified paths in the host filesystem.
+The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory. The default mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`.
+
+---
+
+Once you've made any needed changes to the spec file, create a deployment from it:
+
+```bash
+laconic-so --stack osmosis-urbit-app deploy create --spec-file osmosis-urbit-app-spec.yml --deployment-dir osmosis-urbit-app-deployment
+```
+
+## Set env variables
+
+Inside the deployment directory, open the file `config.env` and set the following env variables:
+
+(Note: Use the following config as is if running the stack locally)
+
+  ```bash
+  # App to be installed (Do not change)
+  CERC_URBIT_APP=osmosis
+
+  # Osmosis API base URL
+  # Set this to proxy server endpoint for osmosis app
+  # (Eg. http://localhost:4000 - in case stack is being run locally with proxy enabled)
+  # (Eg. https://abc.xyz.com - in case https://abc.xyz.com is pointed to the proxy endpoint)
+  CERC_WEB_API_BASE_URL=http://localhost:4000
+
+  # Optional
+
+  # Whether to enable app installation on Urbit
+  # (just builds and uploads the glob file if disabled) (Default: true)
+  CERC_ENABLE_APP_INSTALL=
+
+  # Whether to run the proxy server
+  # (Disable only if proxy not required to be run) (Default: true)
+  CERC_ENABLE_PROXY=
+
+  # Proxy server configuration
+  # Used only if proxy is enabled
+
+  # Upstream API URL
+  CERC_PROXY_UPSTREAM=https://app.osmosis.zone
+
+  # IPFS configuration
+
+  # IFPS endpoint to host the glob file on
+  # (Default: http://ipfs:5001 pointing to in-stack IPFS node)
+  CERC_IPFS_GLOB_HOST_ENDPOINT=
+
+  # IFPS endpoint to fetch the glob file from
+  # (Default: http://ipfs:8080 pointing to in-stack IPFS node)
+  CERC_IPFS_SERVER_ENDPOINT=
+  ```
+
+## Start the stack
+
+Start the deployment:
+
+```bash
+laconic-so deployment --dir osmosis-urbit-app-deployment start
+```
+
+* List and check the health status of all the containers using `docker ps` and wait for them to be `healthy`
+
+* Run the following to get login password for Urbit web interface:
+
+  ```bash
+  laconic-so deployment --dir osmosis-urbit-app-deployment exec urbit-fake-ship "curl -s --data '{\"source\":{\"dojo\":\"+code\"},\"sink\":{\"stdout\":null}}' http://localhost:12321"
+
+  # Expected output: "<PASSWORD>\n"%
+  ```
+
+* Open the Urbit web UI at http://localhost:8080 and use the `PASSWORD` from previous step to login
+
+* The osmosis app is not available when starting stack for the first time. Check `urbit-fake-ship` logs to see that app has installed:
+
+  ```bash
+  laconic-so deployment --dir osmosis-urbit-app-deployment logs -f
+
+  # Expected output:
+  # laconic-3ccf7ee79bdae874-urbit-fake-ship-1    | docket: fetching %http glob for %osmosis desk
+  # laconic-3ccf7ee79bdae874-urbit-fake-ship-1    | ">="">="osmosis app installed
+  ```
+
+* The osmosis app will be now visible at http://localhost:8080
+
+## Clean up
+
+To stop all osmosis-urbit-app services running in the background, while preserving data:
+
+```bash
+# Only stop the docker containers
+laconic-so deployment --dir osmosis-urbit-app-deployment stop
+
+# Run 'start' to restart the deployment
+```
+
+To stop all osmosis-urbit-app services and also delete data:
+
+```bash
+# Stop the docker containers
+laconic-so deployment --dir osmosis-urbit-app-deployment stop --delete-volumes
+
+# Remove deployment directory (deployment will have to be recreated for a re-run)
+rm -r osmosis-urbit-app-deployment
+```
@@ -0,0 +1,13 @@
+version: "0.1"
+name: osmosis-urbit-app
+repos:
+  - github.com/cerc-io/osmosis-frontend@laconic
+  - github.com/cerc-io/watcher-ts@v0.2.78
+containers:
+  - cerc/osmosis-front-end-urbit
+  - cerc/watcher-ts
+pods:
+  - osmosis-front-end-urbit
+  - proxy-server
+  - fixturenet-urbit
+  - kubo
@@ -1,17 +1,17 @@
 # self-hosted osmosis

 Instructions to build and deploy:
- self-hosted gitea
- an ipfs node
- the osmosis front end
- a laconicd chain
+- Self-hosted gitea
+- An ipfs node
+- The osmosis front end
+- A laconicd chain

 ## Setup

 Clone required repositories:

 ```bash
-laconic-so --stack osmosis setup-repositories
+laconic-so --stack osmosis setup-repositories --pull

 # If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories and re-run the command
 ```
@@ -13,7 +13,7 @@ containers:
  - cerc/watcher-ts
  - cerc/osmosis-front-end
 pods:
-  - kubo
+  # - kubo
  - proxy-server
  - osmosis-front-end
  # TODO: Fix pod config for deployment command
@@ -3,7 +3,7 @@ name: package-registry
 description: "Local Package Registry"
 repos:
  - git.vdb.to/cerc-io/hosting
-  - gitea.com/gitea/act_runner
+  - gitea.com/telackey/act_runner@telackey/entrypoint
 containers:
  - cerc/act-runner
  - cerc/act-runner-task-executor
@@ -2,7 +2,7 @@ version: "1.0"
 name: sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.4
+  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.6
 containers:
  - cerc/watcher-sushiswap-v3
 pods:
@@ -14,7 +14,7 @@ Clone required repositories:
 ```bash
 laconic-so --stack uniswap-urbit-app setup-repositories --pull

-# If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories mentioned below and re-run the command
+# If this throws an error as a result of being already checked out to a branch/tag in a repo, remove the repositories and re-run the command
 ```

 Build the container images:
@@ -35,7 +35,7 @@ laconic-so --stack uniswap-urbit-app deploy init --output uniswap-urbit-app-spec

 Edit `network` in spec file to map container ports to same ports in host:

-```
+```yml
 ...
 network:
  ports:
@@ -134,13 +134,14 @@ laconic-so deployment --dir uniswap-urbit-app-deployment start

 * Open the Urbit web UI at http://localhost:8080 and use the `PASSWORD` from previous step to login

-* The uniswap app is not available when starting stack for the first time. Check `urbit-fake-ship` logs to see that app has installed
-  ```
+* The uniswap app is not available when starting stack for the first time. Check `urbit-fake-ship` logs to see that app has installed:
+
+  ```bash
  laconic-so deployment --dir uniswap-urbit-app-deployment logs -f

  # Expected output:
  # laconic-3ccf7ee79bdae874-urbit-fake-ship-1    | docket: fetching %http glob for %uniswap desk
-  # laconic-3ccf7ee79bdae874-urbit-fake-ship-1    | ">="">="Uniswap app installed
+  # laconic-3ccf7ee79bdae874-urbit-fake-ship-1    | ">="">="uniswap app installed
  ```

 * The uniswap app will be now visible at http://localhost:8080
@@ -6,7 +6,6 @@ repos:
 containers:
  - cerc/uniswap-interface
  - cerc/watcher-ts
-  - cerc/urbit-globs-host
 pods:
  - uniswap-interface
  - proxy-server
@@ -444,7 +444,7 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
    deployer_config_generator = getDeployerConfigGenerator(deployment_type)
    # TODO: make deployment_dir_path a Path above
    deployer_config_generator.generate(deployment_dir_path)
-    call_stack_deploy_create(deployment_context, [network_dir, initial_peers])
+    call_stack_deploy_create(deployment_context, [network_dir, initial_peers, deployment_command_context])


 # TODO: this code should be in the stack .py files but
@@ -168,8 +168,8 @@ class ClusterInfo:
            result.append(pv)
        return result

-    # to suit the deployment, and also annotate the container specs to point at said volumes
-    def get_deployment(self):
+    # TODO: put things like image pull policy into an object-scope struct
+    def get_deployment(self, image_pull_policy: str = None):
        containers = []
        for pod_name in self.parsed_pod_yaml_map:
            pod = self.parsed_pod_yaml_map[pod_name]
@@ -189,7 +189,7 @@ class ClusterInfo:
                container = client.V1Container(
                    name=container_name,
                    image=image_to_use,
-                    image_pull_policy="Always",
+                    image_pull_policy=image_pull_policy,
                    env=envs_from_environment_variables_map(self.environment_variables.map),
                    ports=[client.V1ContainerPort(container_port=port)],
                    volume_mounts=volume_mounts,
@@ -200,9 +200,10 @@ class ClusterInfo:
                )
                containers.append(container)
        volumes = volumes_for_pod_files(self.parsed_pod_yaml_map)
+        image_pull_secrets = [client.V1LocalObjectReference(name="laconic-registry")]
        template = client.V1PodTemplateSpec(
            metadata=client.V1ObjectMeta(labels={"app": self.app_name}),
-            spec=client.V1PodSpec(containers=containers, volumes=volumes),
+            spec=client.V1PodSpec(containers=containers, image_pull_secrets=image_pull_secrets, volumes=volumes),
        )
        spec = client.V1DeploymentSpec(
            replicas=1, template=template, selector={
@@ -111,7 +111,7 @@ class K8sDeployer(Deployer):
                print("PVCs created:")
                print(f"{pvc_resp}")
        # Process compose files into a Deployment
-        deployment = self.cluster_info.get_deployment()
+        deployment = self.cluster_info.get_deployment(image_pull_policy=None if self.is_kind() else "Always")
        # Create the k8s objects
        if opts.o.debug:
            print(f"Sending this deployment: {deployment}")
@@ -132,18 +132,18 @@ class K8sDeployer(Deployer):
            print("Service created:")
            print(f"{service_resp}")

-        # TODO: disable ingress for kind
-        ingress: client.V1Ingress = self.cluster_info.get_ingress()
+        if not self.is_kind():
+            ingress: client.V1Ingress = self.cluster_info.get_ingress()

-        if opts.o.debug:
-            print(f"Sending this ingress: {ingress}")
-        ingress_resp = self.networking_api.create_namespaced_ingress(
-            namespace=self.k8s_namespace,
-            body=ingress
-        )
-        if opts.o.debug:
-            print("Ingress created:")
-            print(f"{ingress_resp}")
+            if opts.o.debug:
+                print(f"Sending this ingress: {ingress}")
+            ingress_resp = self.networking_api.create_namespaced_ingress(
+                namespace=self.k8s_namespace,
+                body=ingress
+            )
+            if opts.o.debug:
+                print("Ingress created:")
+                print(f"{ingress_resp}")

    def down(self, timeout, volumes):
        self.connect_api()
@@ -196,16 +196,16 @@ class K8sDeployer(Deployer):
        except client.exceptions.ApiException as e:
            _check_delete_exception(e)

-        # TODO: disable ingress for kind
-        ingress: client.V1Ingress = self.cluster_info.get_ingress()
-        if opts.o.debug:
-            print(f"Deleting this ingress: {ingress}")
-        try:
-            self.networking_api.delete_namespaced_ingress(
-                name=ingress.metadata.name, namespace=self.k8s_namespace
-            )
-        except client.exceptions.ApiException as e:
-            _check_delete_exception(e)
+        if not self.is_kind():
+            ingress: client.V1Ingress = self.cluster_info.get_ingress()
+            if opts.o.debug:
+                print(f"Deleting this ingress: {ingress}")
+            try:
+                self.networking_api.delete_namespaced_ingress(
+                    name=ingress.metadata.name, namespace=self.k8s_namespace
+                )
+            except client.exceptions.ApiException as e:
+                _check_delete_exception(e)

        if self.is_kind():
            # Destroy the kind cluster
@@ -219,7 +219,7 @@ class K8sDeployer(Deployer):

        if all_pods.items:
            for p in all_pods.items:
-                if self.cluster_info.app_name in p.metadata.name:
+                if f"{self.cluster_info.app_name}-deployment" in p.metadata.name:
                    pods.append(p)

        if not pods:
@@ -266,7 +266,7 @@ class K8sDeployer(Deployer):
        ret = []

        for p in pods.items:
-            if self.cluster_info.app_name in p.metadata.name:
+            if f"{self.cluster_info.app_name}-deployment" in p.metadata.name:
                pod_ip = p.status.pod_ip
                ports = AttrDict()
                for c in p.spec.containers:
@@ -299,11 +299,20 @@ class K8sDeployer(Deployer):

    def logs(self, services, tail, follow, stream):
        self.connect_api()
-        pods = pods_in_deployment(self.core_api, "test-deployment")
+        pods = pods_in_deployment(self.core_api, self.cluster_info.app_name)
        if len(pods) > 1:
            print("Warning: more than one pod in the deployment")
-        k8s_pod_name = pods[0]
-        log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test")
+        if len(pods) == 0:
+            log_data = "******* Pods not running ********\n"
+        else:
+            k8s_pod_name = pods[0]
+            # If the pod is not yet started, the logs request below will throw an exception
+            try:
+                log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test")
+            except client.exceptions.ApiException as e:
+                if opts.o.debug:
+                    print(f"Error from read_namespaced_pod_log: {e}")
+                log_data = "******* No logs available ********\n"
        return log_stream_from_string(log_data)

    def update(self):
@@ -21,6 +21,7 @@ from typing import Set, Mapping, List

 from stack_orchestrator.opts import opts
 from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names
+from stack_orchestrator.deploy.deployer import DeployerException


 def _run_command(command: str):
@@ -29,10 +30,13 @@ def _run_command(command: str):
    result = subprocess.run(command, shell=True)
    if opts.o.debug:
        print(f"Result: {result}")
+    return result


 def create_cluster(name: str, config_file: str):
-    _run_command(f"kind create cluster --name {name} --config {config_file}")
+    result = _run_command(f"kind create cluster --name {name} --config {config_file}")
+    if result.returncode != 0:
+        raise DeployerException(f"kind create cluster failed: {result}")


 def destroy_cluster(name: str):
@@ -41,12 +45,14 @@ def destroy_cluster(name: str):

 def load_images_into_kind(kind_cluster_name: str, image_set: Set[str]):
    for image in image_set:
-        _run_command(f"kind load docker-image {image} --name {kind_cluster_name}")
+        result = _run_command(f"kind load docker-image {image} --name {kind_cluster_name}")
+        if result.returncode != 0:
+            raise DeployerException(f"kind create cluster failed: {result}")


 def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str):
    pods = []
-    pod_response = core_api.list_namespaced_pod(namespace="default", label_selector="app=test-app")
+    pod_response = core_api.list_namespaced_pod(namespace="default", label_selector=f"app={deployment_name}")
    if opts.o.debug:
        print(f"pod_response: {pod_response}")
    for pod_info in pod_response.items:
@@ -1,5 +1,4 @@
 # Copyright ©2023 Vulcanize
-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
@@ -13,6 +12,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.

+import hashlib
 import json
 import os
 import shlex
@@ -26,7 +26,8 @@ from stack_orchestrator.deploy.webapp import deploy_webapp
 from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient,
                                                   build_container_image, push_container_image,
                                                   file_hash, deploy_to_k8s, publish_deployment,
-                                                   hostname_for_deployment_request, generate_hostname_for_app)
+                                                   hostname_for_deployment_request, generate_hostname_for_app,
+                                                   match_owner)


 def process_app_deployment_request(
@@ -51,26 +52,18 @@ def process_app_deployment_request(
        raise Exception("Only unqualified hostnames allowed at this time.")

    fqdn = f"{requested_name}.{dns_suffix}"
-    container_tag = "%s:local" % app.attributes.name.replace("@", "")

    # 3. check ownership of existing dnsrecord vs this request
    # TODO: Support foreign DNS
    dns_crn = f"{dns_record_namespace}/{fqdn}"
    dns_record = laconic.get_record(dns_crn)
    if dns_record:
-        dns_record_owners = dns_record.owners
-        dns_request_owners = []
-        if dns_record.request:
-            prev_request = laconic.get_record(dns_record.request, require=True)
-            dns_request_owners = prev_request.owners
+        matched_owner = match_owner(app_deployment_request, dns_record)
+        if not matched_owner and dns_record.request:
+            matched_owner = match_owner(app_deployment_request, laconic.get_record(dns_record.request, require=True))

-        owner_match = None
-
-        for owner in app_deployment_request.owners:
-            if owner in dns_request_owners or owner in dns_record_owners:
-                owner_match = owner
-        if owner_match:
-            print("Matched DnsRecord ownership to", owner)
+        if matched_owner:
+            print("Matched DnsRecord ownership:", matched_owner)
        else:
            raise Exception("Unable to confirm ownership of DnsRecord %s for request %s" %
                            (dns_record.id, app_deployment_request.id))
@@ -94,13 +87,14 @@ def process_app_deployment_request(
    deployment_record = laconic.get_record(app_deployment_crn)
    deployment_dir = os.path.join(deployment_parent_dir, fqdn)
    deployment_config_file = os.path.join(deployment_dir, "config.env")
+    deployment_container_tag = "laconic-webapp/%s:local" % hashlib.md5(deployment_dir.encode()).hexdigest()
    #   b. check for deployment directory (create if necessary)
    if not os.path.exists(deployment_dir):
        if deployment_record:
-            raise ("Deployment record %s exists, but not deployment dir %s. Please remove name." %
-                   (app_deployment_crn, deployment_dir))
+            raise Exception("Deployment record %s exists, but not deployment dir %s. Please remove name." %
+                            (app_deployment_crn, deployment_dir))
        print("deploy_webapp", deployment_dir)
-        deploy_webapp.create_deployment(ctx, deployment_dir, container_tag,
+        deploy_webapp.create_deployment(ctx, deployment_dir, deployment_container_tag,
                                        f"https://{fqdn}", kube_config, image_registry, env_filename)
    elif env_filename:
        shutil.copyfile(env_filename, deployment_config_file)
@@ -108,7 +102,7 @@ def process_app_deployment_request(
    needs_k8s_deploy = False
    # 6. build container (if needed)
    if not deployment_record or deployment_record.attributes.application != app.id:
-        build_container_image(app, container_tag)
+        build_container_image(app, deployment_container_tag)
        push_container_image(deployment_dir)
        needs_k8s_deploy = True

@@ -153,7 +147,6 @@ def dump_known_requests(filename, requests):

@click.command()
@click.option("--kube-config", help="Provide a config file for a k8s deployment")
-@click.option("--kube-config", help="Provide a config file for a k8s deployment")
@click.option("--laconic-config", help="Provide a config file for laconicd", required=True)
@click.option("--image-registry", help="Provide a container image registry url for this k8s cluster")
@click.option("--deployment-parent-dir", help="Create deployment directories beneath this directory", required=True)
@@ -237,9 +230,20 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
        if d.attributes.request:
            deployments_by_request[d.attributes.request] = d

+    # Find removal requests.
+    cancellation_requests = {}
+    removal_requests = laconic.app_deployment_removal_requests()
+    for r in removal_requests:
+        if r.attributes.request:
+            cancellation_requests[r.attributes.request] = r
+
    requests_to_execute = []
    for r in requests_by_name.values():
-        if r.id not in deployments_by_request:
+        if r.id in cancellation_requests and match_owner(cancellation_requests[r.id], r):
+            print(f"Found deployment cancellation request for {r.id} at {cancellation_requests[r.id].id}")
+        elif r.id in deployments_by_request:
+            print(f"Found satisfied request for {r.id} at {deployments_by_request[r.id].id}")
+        else:
            if r.id not in previous_requests:
                print(f"Request {r.id} needs to processed.")
                requests_to_execute.append(r)
@@ -247,8 +251,6 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
                print(
                    f"Skipping unsatisfied request {r.id} because we have seen it before."
                )
-        else:
-            print(f"Found satisfied request {r.id} at {deployments_by_request[r.id].names[0]}")

    print("Found %d unsatisfied request(s) to process." % len(requests_to_execute))

@@ -262,7 +264,7 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
                    record_namespace_deployments,
                    record_namespace_dns,
                    dns_suffix,
-                    deployment_parent_dir,
+                    os.path.abspath(deployment_parent_dir),
                    kube_config,
                    image_registry
                )
@@ -0,0 +1,189 @@
+# Copyright ©2023 Vulcanize
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+
+import click
+
+from stack_orchestrator.deploy.webapp.util import LaconicRegistryClient, match_owner
+
+
+def process_app_removal_request(ctx,
+                                laconic: LaconicRegistryClient,
+                                app_removal_request,
+                                deployment_parent_dir,
+                                delete_volumes,
+                                delete_names):
+    deployment_record = laconic.get_record(app_removal_request.attributes.deployment, require=True)
+    dns_record = laconic.get_record(deployment_record.attributes.dns, require=True)
+    deployment_dir = os.path.join(deployment_parent_dir, dns_record.attributes.name)
+
+    if not os.path.exists(deployment_dir):
+        raise Exception("Deployment directory %s does not exist." % deployment_dir)
+
+    # Check if the removal request is from the owner of the DnsRecord or deployment record.
+    matched_owner = match_owner(app_removal_request, deployment_record, dns_record)
+
+    # Or of the original deployment request.
+    if not matched_owner and deployment_record.request:
+        matched_owner = match_owner(app_removal_request, laconic.get_record(deployment_record.request, require=True))
+
+    if matched_owner:
+        print("Matched deployment ownership:", matched_owner)
+    else:
+        raise Exception("Unable to confirm ownership of deployment %s for removal request %s" %
+                        (deployment_record.id, app_removal_request.id))
+
+    # TODO(telackey): Call the function directly.  The easiest way to build the correct click context is to
+    # exec the process, but it would be better to refactor so we could just call down_operation with the
+    # necessary parameters
+    down_command = [sys.argv[0], "deployment", "--dir", deployment_dir, "down"]
+    if delete_volumes:
+        down_command.append("--delete-volumes")
+    result = subprocess.run(down_command)
+    result.check_returncode()
+
+    removal_record = {
+        "record": {
+            "type": "ApplicationDeploymentRemovalRecord",
+            "version": "1.0.0",
+            "request": app_removal_request.id,
+            "deployment": deployment_record.id,
+        }
+    }
+    laconic.publish(removal_record)
+
+    if delete_names:
+        if deployment_record.names:
+            for name in deployment_record.names:
+                laconic.delete_name(name)
+
+        if dns_record.names:
+            for name in dns_record.names:
+                laconic.delete_name(name)
+
+    if delete_volumes:
+        shutil.rmtree(deployment_dir)
+
+
+def load_known_requests(filename):
+    if filename and os.path.exists(filename):
+        return json.load(open(filename, "r"))
+    return {}
+
+
+def dump_known_requests(filename, requests):
+    if not filename:
+        return
+    known_requests = load_known_requests(filename)
+    for r in requests:
+        known_requests[r.id] = r.createTime
+    json.dump(known_requests, open(filename, "w"))
+
+
+@click.command()
+@click.option("--laconic-config", help="Provide a config file for laconicd", required=True)
+@click.option("--deployment-parent-dir", help="Create deployment directories beneath this directory", required=True)
+@click.option("--request-id", help="The ApplicationDeploymentRemovalRequest to process")
+@click.option("--discover", help="Discover and process all pending ApplicationDeploymentRemovalRequests",
+              is_flag=True, default=False)
+@click.option("--state-file", help="File to store state about previously seen requests.")
+@click.option("--only-update-state", help="Only update the state file, don't process any requests anything.", is_flag=True)
+@click.option("--delete-names/--preserve-names", help="Delete all names associated with removed deployments.", default=True)
+@click.option("--delete-volumes/--preserve-volumes", default=True, help="delete data volumes")
+@click.option("--dry-run", help="Don't do anything, just report what would be done.", is_flag=True)
+@click.pass_context
+def command(ctx, laconic_config, deployment_parent_dir,
+            request_id, discover, state_file, only_update_state,
+            delete_names, delete_volumes, dry_run):
+    if request_id and discover:
+        print("Cannot specify both --request-id and --discover", file=sys.stderr)
+        sys.exit(2)
+
+    if not request_id and not discover:
+        print("Must specify either --request-id or --discover", file=sys.stderr)
+        sys.exit(2)
+
+    if only_update_state and not state_file:
+        print("--only-update-state requires --state-file", file=sys.stderr)
+        sys.exit(2)
+
+    laconic = LaconicRegistryClient(laconic_config)
+
+    # Find deployment removal requests.
+    # single request
+    if request_id:
+        requests = [laconic.get_record(request_id, require=True)]
+        # TODO: assert record type
+    # all requests
+    elif discover:
+        requests = laconic.app_deployment_removal_requests()
+
+    if only_update_state:
+        if not dry_run:
+            dump_known_requests(state_file, requests)
+        return
+
+    previous_requests = load_known_requests(state_file)
+    requests.sort(key=lambda r: r.createTime)
+
+    # Find deployments.
+    deployments = {}
+    for d in laconic.app_deployments(all=True):
+        deployments[d.id] = d
+
+    # Find removal requests.
+    removals_by_deployment = {}
+    removals_by_request = {}
+    for r in laconic.app_deployment_removals():
+        if r.attributes.deployment:
+            # TODO: should we handle CRNs?
+            removals_by_deployment[r.attributes.deployment] = r
+
+    requests_to_execute = []
+    for r in requests:
+        if not r.attributes.deployment:
+            print(f"Skipping removal request {r.id} since it was a cancellation.")
+        elif r.id in removals_by_request:
+            print(f"Found satisfied request for {r.id} at {removals_by_request[r.id].id}")
+        elif r.attributes.deployment in removals_by_deployment:
+            print(
+                f"Found removal record for indicated deployment {r.attributes.deployment} at "
+                f"{removals_by_deployment[r.attributes.deployment].id}")
+        else:
+            if r.id not in previous_requests:
+                print(f"Request {r.id} needs to processed.")
+                requests_to_execute.append(r)
+            else:
+                print(f"Skipping unsatisfied request {r.id} because we have seen it before.")
+
+    print("Found %d unsatisfied request(s) to process." % len(requests_to_execute))
+
+    if not dry_run:
+        for r in requests_to_execute:
+            try:
+                process_app_removal_request(
+                    ctx,
+                    laconic,
+                    r,
+                    os.path.abspath(deployment_parent_dir),
+                    delete_volumes,
+                    delete_names
+                )
+            finally:
+                dump_known_requests(state_file, [r])
@@ -49,6 +49,14 @@ def cmd(*vargs):
        raise err


+def match_owner(recordA, *records):
+    for owner in recordA.owners:
+        for otherRecord in records:
+            if owner in otherRecord.owners:
+                return owner
+    return None
+
+
 class LaconicRegistryClient:
    def __init__(self, config_file):
        self.config_file = config_file
@@ -146,11 +154,17 @@ class LaconicRegistryClient:
            raise Exception("Cannot locate record:", name_or_id)
        return None

-    def app_deployment_requests(self):
-        return self.list_records({"type": "ApplicationDeploymentRequest"}, True)
+    def app_deployment_requests(self, all=True):
+        return self.list_records({"type": "ApplicationDeploymentRequest"}, all)

-    def app_deployments(self):
-        return self.list_records({"type": "ApplicationDeploymentRecord"})
+    def app_deployments(self, all=True):
+        return self.list_records({"type": "ApplicationDeploymentRecord"}, all)
+
+    def app_deployment_removal_requests(self, all=True):
+        return self.list_records({"type": "ApplicationDeploymentRemovalRequest"}, all)
+
+    def app_deployment_removals(self, all=True):
+        return self.list_records({"type": "ApplicationDeploymentRemovalRecord"}, all)

    def publish(self, record, names=[]):
        tmpdir = tempfile.mkdtemp()
@@ -165,11 +179,17 @@ class LaconicRegistryClient:
                cmd("laconic", "-c", self.config_file, "cns", "record", "publish", "--filename", record_fname)
            )["id"]
            for name in names:
-                cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, new_record_id)
+                self.set_name(name, new_record_id)
            return new_record_id
        finally:
            cmd("rm", "-rf", tmpdir)

+    def set_name(self, name, record_id):
+        cmd("laconic", "-c", self.config_file, "cns", "name", "set", name, record_id)
+
+    def delete_name(self, name):
+        cmd("laconic", "-c", self.config_file, "cns", "name", "delete", name)
+

 def file_hash(filename):
    return hashlib.sha1(open(filename).read().encode()).hexdigest()
@@ -187,9 +207,11 @@ def build_container_image(app_record, tag, extra_build_args=[]):
        print(f"Cloning repository {repo} to {clone_dir} ...")
        if ref:
            # TODO: Determing branch or hash, and use depth 1 if we can.
-            result = subprocess.run(["git", "clone", repo, clone_dir])
-            result.check_returncode()
-            subprocess.check_call(["git", "checkout", ref], cwd=clone_dir)
+            git_env = dict(os.environ.copy())
+            # Never prompt
+            git_env["GIT_TERMINAL_PROMPT"] = "0"
+            subprocess.check_call(["git", "clone", repo, clone_dir], env=git_env)
+            subprocess.check_call(["git", "checkout", ref], cwd=clone_dir, env=git_env)
        else:
            result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir])
            result.check_returncode()
@@ -20,7 +20,10 @@ from stack_orchestrator.repos import setup_repositories
 from stack_orchestrator.build import build_containers
 from stack_orchestrator.build import build_npms
 from stack_orchestrator.build import build_webapp
-from stack_orchestrator.deploy.webapp import run_webapp, deploy_webapp, deploy_webapp_from_registry
+from stack_orchestrator.deploy.webapp import (run_webapp,
+                                              deploy_webapp,
+                                              deploy_webapp_from_registry,
+                                              undeploy_webapp_from_registry)
 from stack_orchestrator.deploy import deploy
 from stack_orchestrator import version
 from stack_orchestrator.deploy import deployment
@@ -54,6 +57,7 @@ cli.add_command(build_webapp.command, "build-webapp")
 cli.add_command(run_webapp.command, "run-webapp")
 cli.add_command(deploy_webapp.command, "deploy-webapp")
 cli.add_command(deploy_webapp_from_registry.command, "deploy-webapp-from-registry")
+cli.add_command(undeploy_webapp_from_registry.command, "undeploy-webapp-from-registry")
 cli.add_command(deploy.command, "deploy")  # deploy is an alias for deploy-system
 cli.add_command(deploy.command, "deploy-system")
 cli.add_command(deployment.command, "deployment")
@@ -1,14 +1,59 @@
 #!/usr/bin/env bash
 set -e
 if [ -n "$CERC_SCRIPT_DEBUG" ]; then
-  set -x
+    set -x
+    # Dump environment variables for debugging
+    echo "Environment variables:"
+    env
 fi
+
+# Helper functions: TODO move into a separate file
+wait_for_pods_started () {
+    for i in {1..5}
+    do
+        local ps_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir ps )
+
+        if [[ "$ps_output" == *"Running containers:"* ]]; then
+            # if ready, return
+            return
+        else
+            # if not ready, wait
+            sleep 5
+        fi
+    done
+    # Timed out, error exit
+    echo "waiting for pods to start: FAILED"
+    delete_cluster_exit
+}
+
+wait_for_log_output () {
+    for i in {1..5}
+    do
+
+        local log_output=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+
+        if [[ ! -z "$log_output" ]]; then
+            # if ready, return
+            return
+        else
+            # if not ready, wait
+            sleep 5
+        fi
+    done
+    # Timed out, error exit
+    echo "waiting for pods log content: FAILED"
+    delete_cluster_exit
+}
+
+
+delete_cluster_exit () {
+    $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
+    exit 1
+}
+
 # Note: eventually this test should be folded into ../deploy/
 # but keeping it separate for now for convenience
 TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
-# Dump environment variables for debugging
-echo "Environment variables:"
-env
 # Set a non-default repo dir
 export CERC_REPO_BASE_DIR=~/stack-orchestrator-test/repo-base-dir
 echo "Testing this package: $TEST_TARGET_SO"
@@ -18,7 +63,9 @@ echo "Version reported is: ${reported_version_string}"
 echo "Cloning repositories into: $CERC_REPO_BASE_DIR"
 rm -rf $CERC_REPO_BASE_DIR
 mkdir -p $CERC_REPO_BASE_DIR
-# Test basic stack-orchestrator deploy
+$TEST_TARGET_SO --stack test setup-repositories
+$TEST_TARGET_SO --stack test build-containers
+# Test basic stack-orchestrator deploy to k8s
 test_deployment_dir=$CERC_REPO_BASE_DIR/test-deployment-dir
 test_deployment_spec=$CERC_REPO_BASE_DIR/test-deployment-spec.yml
 $TEST_TARGET_SO --stack test deploy --deploy-to k8s-kind init --output $test_deployment_spec --config CERC_TEST_PARAM_1=PASSED
@@ -53,23 +100,36 @@ fi
 echo "deploy create output file test: passed"
 # Try to start the deployment
 $TEST_TARGET_SO deployment --dir $test_deployment_dir start
-# TODO: add a check to see if the container is up
-# Sleep because k8s not up yet
-sleep 30
+wait_for_pods_started
 # Check logs command works
+wait_for_log_output
 log_output_3=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
 if [[ "$log_output_3" == *"Filesystem is fresh"* ]]; then
    echo "deployment logs test: passed"
 else
    echo "deployment logs test: FAILED"
-    exit 1
+    delete_cluster_exit
 fi
 # Check the config variable CERC_TEST_PARAM_1 was passed correctly
 if [[ "$log_output_3" == *"Test-param-1: PASSED"* ]]; then
    echo "deployment config test: passed"
 else
    echo "deployment config test: FAILED"
-    exit 1
+    delete_cluster_exit
+fi
+# Stop then start again and check the volume was preserved
+$TEST_TARGET_SO deployment --dir $test_deployment_dir stop
+# Sleep a bit just in case
+sleep 2
+$TEST_TARGET_SO deployment --dir $test_deployment_dir start
+wait_for_pods_started
+wait_for_log_output
+log_output_4=$( $TEST_TARGET_SO deployment --dir $test_deployment_dir logs )
+if [[ "$log_output_4" == *"Filesystem is old"* ]]; then
+    echo "Retain volumes test: passed"
+else
+    echo "Retain volumes test: FAILED"
+    delete_cluster_exit
 fi
 # Stop and clean up
 $TEST_TARGET_SO deployment --dir $test_deployment_dir stop --delete-volumes
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# TODO: handle ARM
+curl --silent -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
+chmod +x ./kind
+mv ./kind /usr/local/bin
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# TODO: handle ARM
+curl --silent -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+chmod +x ./kubectl
+mv ./kubectl /usr/local/bin
Author	SHA1	Message	Date
zramsay	cc4396d593	birbit stack	2024-01-26 17:44:31 +00:00
dboreham	f826f50c4d	Merge branch 'main' of github.com:cerc-io/stack-orchestrator	2024-01-17 08:27:38 -07:00
dboreham	b83465767d	Set k8s deploy test to manual trigger	2024-01-17 08:27:09 -07:00
dboreham	50509203d1	Set k8s deploy test to manual trigger	2024-01-17 08:26:51 -07:00
prathamesh0	282e175566	Remove unnecessary hyperlinks and pin image versions (#706 ) * Remove invalid dashboard and panel ids from alert rules * Pin grafana and prometheus versions * Configure custom grafana server URL	2024-01-17 14:02:10 +05:30
dboreham	635aa7037b	Build test container	2024-01-16 21:15:21 -07:00
dboreham	9877cfaf85	Update for new runner	2024-01-16 20:08:32 -07:00
dboreham	c642e5d490	Try different runner	2024-01-16 17:09:58 -07:00
dboreham	02c49d66f5	Add debug output to check container	2024-01-16 17:07:03 -07:00
dboreham	90cebdb7a6	Add CI script for k8s deployment test (#705 )	2024-01-16 16:16:07 -07:00
dboreham	1f9653e6f7	Fix kind mode and add k8s deployment test (#704 ) * Fix kind mode and add k8s deployment test * Fix lint errors	2024-01-16 15:55:58 -07:00
telackey	0587813dd0	Create next.config.js if it is missing. (#701 ) * Create next.config.js if it is missing. * Add comment.	2024-01-15 12:12:59 -06:00
telackey	4b3ea7c30f	Update independent act-runner stack to use custom act as well. (#702 ) * Update independent act-runner stack to use custom act as well. * Remove branches which are not needed or already merged.	2024-01-15 12:10:48 -06:00
nabarun	db8aec52aa	Pin commit hash of asset list repo in osmosis frontend app (#703 )	2024-01-15 16:56:06 +05:30
telackey	b83030f63b	Use custom act with gitea. (#700 ) * Use custom act with gitea. * Make sure wget is available * Fix repo url	2024-01-09 22:53:43 -06:00
prathamesh0	a3eb3c0bb0	Setup basic alerting for watchers in monitoring stack (#698 ) * Provision Grafana alert contactpoints and policies for Slack * Add watcher alert rules * Update watcher monitoring instructions * Add listening port flag to node exporter command * Add reference links	2024-01-08 17:25:30 +05:30
nabarun	eae2af7ccc	Upgrade watcher versions (#699 )	2024-01-08 11:51:54 +05:30
telackey	837e443800	Support application removal requests. (#697 ) * Support application removal request. * Git should never prompt when deploying a webapp	2023-12-21 18:05:40 -06:00
prathamesh0	a57b0cdd26	Add a stack for prom node exporter and its dashboard in monitoring stack (#696 ) * Add a stack for Prometheus node exporter * Add node exporter dashboard to monitoring stack	2023-12-21 15:15:03 +05:30
prathamesh0	38622fb33c	[WIP] Use templating for watcher dashboard and add Postgres exporter (#695 ) * Add Postgres exporter and it's dashboard * Use templating for watcher dashboard * Add subgraph related panels to watcher dashboard * Remove individual watcher dashboards and update instructions	2023-12-21 13:41:36 +05:30
prathamesh0	4a1a46facc	Update monitoring stack with additional dashboards and watcher metrics (#693 ) * Include retry jobs and update default refresh intervals * Add prometheus blackbox exporter and it's dashboard * Add NodeJS application dashboard * Allow UI updates * Update watcher dashboards for upstream and external chain heads * Update watcher dashboards with watcher config metrics * Upgrade sushiswap and azimuth watchers * Removed fixed title size values * Update instructions * Update instructions for env config * Update instructions with setup	2023-12-21 09:26:37 +05:30
Zach	42b92f7e23	use square logo for an urbit tile (#689 ) * use square logo for an urbit tile * bump version, improve info text	2023-12-19 09:50:14 -05:00
nabarun	def192edab	Update new environment values for Osmosis frontend app (#694 ) * Update new env values for Osmosis frontend app * Use .env.production instead of local	2023-12-18 17:49:45 +05:30
dboreham	d8357df345	Add image pull secret to pods (#692 )	2023-12-15 14:27:45 -07:00
telackey	997496b8a5	Update script for new nextjs build output. (#691 )	2023-12-14 19:47:30 -06:00
telackey	61f2884505	Reduce base image size (first round of improvements) (#690 )	2023-12-14 17:46:03 -06:00
telackey	27a14737f8	Make the container tag based on the deployment path. (#688 )	2023-12-14 09:49:21 -06:00
prathamesh0andnabarun	b9b758bfdd	Add a stack for running Osmosis frontend app on Urbit (#683 ) * Deploy osmosis on Urbit fake ship * Remove Urbit configuration from existing osmosis stack * Add a separate stack for Osmosis front end on Urbit * Run script for renaming build files with bash * Add environment variables required in urbit osmosis build * Fix BASEPATH in compose file * Remove ipfs-glob-host from network config in osmosis readme * Use laconic branch for osmosis frontend --------- Co-authored-by: Nabarun <nabarun@deepstacksoft.com>	2023-12-14 17:28:10 +05:30
prathamesh0	9ba410b095	Add a stack for monitoring watchers with prometheus and grafana (#687 ) * Setup Prometheus and Grafana for monitoring stack * Add dashboard for azimuth watchers * Add a dashboard for sushiswap watcher * Persist prometheus server data * Additional metrics in watcher dashboards * Update dashboards and add for merkl sushiswap watcher * Add dashboards for remaining azimuth watchers * Separate out preconfigured watcher dashboards and add instructions * Keep the empty dashboards dir	2023-12-14 16:59:00 +05:30
				`@@ -0,0 +1 @@`
				`Change this file to trigger running the test-k8s-deploy CI job`