update laconicd test (#710)

* Remove legacy docker config

* Trigger test run

.gitea/workflows/fixturenet-laconicd-test.yml

@@ -7,10 +7,6 @@ on:
       - '!**'
       - '.gitea/workflows/triggers/fixturenet-laconicd-test'
 
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
 
 jobs:
   test:
@@ -47,9 +43,5 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
       - name: "Run fixturenet-laconicd tests"
         run: ./tests/fixturenet-laconicd/run-test.sh

.gitea/workflows/test-k8s-deploy.yml

@@ -1,23 +1,16 @@
 name: K8s Deploy Test
 
 on:
-  pull_request:
-    branches: '*'
   push:
-    branches:
-      - main
-      - ci-test
-    paths-ignore:
-      - '.gitea/workflows/triggers/*'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
+    branches: '*'
+    paths:
+      - '!**'
+      - '.gitea/workflows/triggers/fixturenet-laconicd-test'
 
 jobs:
   test:
-    name: "Run deploy test suite"
-    runs-on: ubuntu-latest
+    name: "Run deploy test suite on kind/k8s"
+    runs-on: ubuntu-22.04-with-syn-ethdb
     steps:
       - name: "Clone project repository"
         uses: actions/checkout@v3
@@ -41,15 +34,9 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
-      - name: "Install Go"
-        uses: actions/setup-go@v4
-        with:
-          go-version: '1.21'
-      - name: "Install Kind"
-        run: go install sigs.k8s.io/kind@v0.20.0
-      - name: "Debug Kind"
-        run: kind create cluster --retain && docker logs kind-control-plane
+      - name: "Install kind"
+        run: ./tests/scripts/install-kind.sh
+      - name: "Install Kubectl"
+        run: ./tests/scripts/install-kubectl.sh
+      - name: "Run k8s deployment test"
+        run: ./tests/k8s-deploy/run-deploy-test.sh

.gitea/workflows/test-webapp.yml

@@ -41,6 +41,8 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
+      - name: "Install wget" # 20240109 - Only needed until the executors are updated.
+        run: apt update && apt install -y wget
       - name: Start dockerd # Also needed until we can incorporate into the executor
         run: |
           dockerd -H $DOCKER_HOST --userland-proxy=false &

.gitea/workflows/triggers/fixturenet-laconicd-test

@@ -1,2 +1,3 @@
 Change this file to trigger running the fixturenet-laconicd-test CI job
-
+Trigger
+Trigger

.gitea/workflows/triggers/test-k8s-deploy

@@ -0,0 +1 @@
+Change this file to trigger running the test-k8s-deploy CI job

docs/webapp.md

@@ -0,0 +1,64 @@
+### Building and Running Webapps
+
+It is possible to build and run Next.js webapps using the `build-webapp` and `run-webapp` subcommands.
+
+To make it easier to build once and deploy into different environments and with different configuration,
+compilation and static page generation are separated in the `build-webapp` and `run-webapp` steps.
+
+This offers much more flexibilty than standard Next.js build methods, since any environment variables accessed
+via `process.env`, whether for pages or for API, will have values drawn from their runtime deployment environment,
+not their build environment. 
+
+## Building
+
+Building usually requires no additional configuration.  By default, the Next.js version specified in `package.json`
+is used, and either `yarn` or `npm` will be used automatically depending on which lock files are present.  These
+can be overidden with the build arguments `CERC_NEXT_VERSION` and `CERC_BUILD_TOOL` respectively.  For example: `--extra-build-args "--build-arg CERC_NEXT_VERSION=13.4.12"`
+
+**Example**:
+```
+$ cd ~/cerc
+$ git clone git@git.vdb.to:cerc-io/test-progressive-web-app.git
+$ laconic-so build-webapp --source-repo ~/cerc/test-progressive-web-app
+...
+
+Built host container for ~/cerc/test-progressive-web-app with tag:
+
+    cerc/test-progressive-web-app:local
+
+To test locally run:
+
+    laconic-so run-webapp --image cerc/test-progressive-web-app:local --env-file /path/to/environment.env
+
+```
+
+## Running
+
+With `run-webapp` a new container will be launched on the local machine, with runtime configuration provided by `--env-file` (if specified) and published on an available port.  Multiple instances can be launched with different configuration.
+
+**Example**:
+```
+# Production env
+$ laconic-so run-webapp --image cerc/test-progressive-web-app:local --env-file /path/to/environment/production.env
+
+Image: cerc/test-progressive-web-app:local
+ID: 4c6e893bf436b3e91a2b92ce37e30e499685131705700bd92a90d2eb14eefd05
+URL: http://localhost:32768
+
+# Dev env
+$ laconic-so run-webapp --image cerc/test-progressive-web-app:local --env-file /path/to/environment/dev.env
+
+Image: cerc/test-progressive-web-app:local
+ID: 9ab96494f563aafb6c057d88df58f9eca81b90f8721a4e068493a289a976051c
+URL: http://localhost:32769
+```
+
+## Deploying
+
+Use the subcommand `deploy-webapp create` to make a deployment directory that can be subsequently deployed to a Kubernetes cluster.
+Example commands are shown below, assuming that the webapp container image `cerc/test-progressive-web-app:local` has already been built:
+```
+$ laconic-so deploy-webapp create --kube-config ~/kubectl/k8s-kubeconfig.yaml --image-registry registry.digitalocean.com/laconic-registry --deployment-dir webapp-k8s-deployment --image cerc/test-progressive-web-app:local --url https://test-pwa-app.hosting.laconic.com/ --env-file test-webapp.env
+$ laconic-so deployment --dir webapp-k8s-deployment push-images
+$ laconic-so deployment --dir webapp-k8s-deployment start
+```

stack_orchestrator/build/build_containers.py

@@ -27,7 +27,7 @@ import subprocess
 import click
 import importlib.resources
 from pathlib import Path
-from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config
+from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external
 from stack_orchestrator.base import get_npm_registry_url
 
 # TODO: find a place for this
@@ -58,7 +58,8 @@ def make_container_build_env(dev_root_path: str,
     return container_build_env
 
 
-def process_container(container,
+def process_container(stack: str,
+                      container,
                       container_build_dir: str,
                       container_build_env: dict,
                       dev_root_path: str,
@@ -69,12 +70,29 @@ def process_container(container,
                       ):
     if not quiet:
         print(f"Building: {container}")
-    build_dir = os.path.join(container_build_dir, container.replace("/", "-"))
-    build_script_filename = os.path.join(build_dir, "build.sh")
+
+    default_container_tag = f"{container}:local"
+    container_build_env.update({"CERC_DEFAULT_CONTAINER_IMAGE_TAG": default_container_tag})
+
+    # Check if this is in an external stack
+    if stack_is_external(stack):
+        container_parent_dir = Path(stack).joinpath("container-build")
+        temp_build_dir = container_parent_dir.joinpath(container.replace("/", "-"))
+        temp_build_script_filename = temp_build_dir.joinpath("build.sh")
+        # Now check if the container exists in the external stack.
+        if not temp_build_script_filename.exists():
+            # If not, revert to building an internal container
+            container_parent_dir = container_build_dir
+    else:
+        container_parent_dir = container_build_dir
+
+    build_dir = container_parent_dir.joinpath(container.replace("/", "-"))
+    build_script_filename = build_dir.joinpath("build.sh")
+
     if verbose:
         print(f"Build script filename: {build_script_filename}")
     if os.path.exists(build_script_filename):
-        build_command = build_script_filename
+        build_command = build_script_filename.as_posix()
     else:
         if verbose:
             print(f"No script file found: {build_script_filename}, using default build script")
@@ -84,7 +102,7 @@ def process_container(container,
         repo_full_path = os.path.join(dev_root_path, repo_dir)
         repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir
         build_command = os.path.join(container_build_dir,
-                                     "default-build.sh") + f" {container}:local {repo_dir_or_build_dir}"
+                                     "default-build.sh") + f" {default_container_tag} {repo_dir_or_build_dir}"
     if not dry_run:
         if verbose:
             print(f"Executing: {build_command} with environment: {container_build_env}")
@@ -158,7 +176,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
 
     for container in containers_in_scope:
         if include_exclude_check(container, include, exclude):
-            process_container(container, container_build_dir, container_build_env,
+            process_container(stack, container, container_build_dir, container_build_env,
                               dev_root_path, quiet, verbose, dry_run, continue_on_error)
         else:
             if verbose:

stack_orchestrator/build/build_webapp.py

@@ -32,8 +32,9 @@ from stack_orchestrator.build import build_containers
 @click.option('--source-repo', help="directory containing the webapp to build", required=True)
 @click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild")
 @click.option("--extra-build-args", help="Supply extra arguments to build")
+@click.option("--tag", help="Container tag (default: cerc/<app_name>:local)")
 @click.pass_context
-def command(ctx, base_container, source_repo, force_rebuild, extra_build_args):
+def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, tag):
     '''build the specified webapp container'''
 
     quiet = ctx.obj.quiet
@@ -60,7 +61,7 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args):
     container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug,
                                                                     force_rebuild, extra_build_args)
 
-    build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet,
+    build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet,
                                        verbose, dry_run, continue_on_error)
 
 
@@ -70,8 +71,11 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args):
     container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = os.path.join(container_build_dir,
                                                                     base_container.replace("/", "-"),
                                                                     "Dockerfile.webapp")
-    webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1]
-    container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local"
+    if not tag:
+        webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1]
+        container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local"
+    else:
+        container_build_env["CERC_CONTAINER_BUILD_TAG"] = tag
 
-    build_containers.process_container(base_container, container_build_dir, container_build_env, dev_root_path, quiet,
+    build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet,
                                        verbose, dry_run, continue_on_error)

stack_orchestrator/constants.py

@@ -0,0 +1,32 @@
+# Copyright © 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+cluster_name_prefix = "laconic-"
+stack_file_name = "stack.yml"
+spec_file_name = "spec.yml"
+config_file_name = "config.env"
+deployment_file_name = "deployment.yml"
+compose_dir_name = "compose"
+compose_deploy_type = "compose"
+k8s_kind_deploy_type = "k8s-kind"
+k8s_deploy_type = "k8s"
+cluster_id_key = "cluster-id"
+kube_config_key = "kube-config"
+deploy_to_key = "deploy-to"
+network_key = "network"
+http_proxy_key = "http-proxy"
+image_resigtry_key = "image-registry"
+kind_config_filename = "kind-config.yml"
+kube_config_filename = "kubeconfig.yml"

stack_orchestrator/data/compose/docker-compose-fixturenet-laconic-console.yml

@@ -4,6 +4,6 @@ services:
     image: cerc/laconic-console-host:local
     environment:
       - CERC_WEBAPP_FILES_DIR=${CERC_WEBAPP_FILES_DIR:-/usr/local/share/.config/yarn/global/node_modules/@cerc-io/console-app/dist/production}
-      - LACONIC_HOSTED_ENDPOINT=${LACONIC_HOSTED_ENDPOINT:-http://localhost}
+      - LACONIC_HOSTED_ENDPOINT=${LACONIC_HOSTED_ENDPOINT:-http://localhost:9473}
     ports:
       - "80"

stack_orchestrator/data/compose/docker-compose-fixturenet-laconicd.yml

@@ -5,7 +5,7 @@ services:
     command: ["sh", "/docker-entrypoint-scripts.d/create-fixturenet.sh"]
     volumes:
       # The cosmos-sdk node's database directory:
-      - laconicd-data:/root/.laconicd/data
+      - laconicd-data:/root/.laconicd
       # TODO: look at folding these scripts into the container
       - ../config/fixturenet-laconicd/create-fixturenet.sh:/docker-entrypoint-scripts.d/create-fixturenet.sh
       - ../config/fixturenet-laconicd/export-mykey.sh:/docker-entrypoint-scripts.d/export-mykey.sh

stack_orchestrator/data/compose/docker-compose-fixturenet-urbit.yml

@@ -0,0 +1,36 @@
+version: '3.7'
+
+services:
+  # Runs an Urbit fake ship and attempts an app installation using given data
+  # Uploads the app glob to given IPFS endpoint
+  # From urbit_app_builds volume:
+  # - takes app build from ${CERC_URBIT_APP}/build (waits for it to appear)
+  # - takes additional mark files from ${CERC_URBIT_APP}/mar
+  # - takes the docket file from ${CERC_URBIT_APP}/desk.docket-0
+  urbit-fake-ship:
+    restart: unless-stopped
+    image: tloncorp/vere
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_URBIT_APP: ${CERC_URBIT_APP}
+      CERC_ENABLE_APP_INSTALL: ${CERC_ENABLE_APP_INSTALL:-true}
+      CERC_IPFS_GLOB_HOST_ENDPOINT: ${CERC_IPFS_GLOB_HOST_ENDPOINT:-http://ipfs:5001}
+      CERC_IPFS_SERVER_ENDPOINT: ${CERC_IPFS_SERVER_ENDPOINT:-http://ipfs:8080}
+    entrypoint: ["bash", "-c", "./run-urbit-ship.sh && ./deploy-app.sh && tail -f /dev/null"]
+    volumes:
+      - urbit_data:/urbit
+      - urbit_app_builds:/app-builds
+      - ../config/urbit/run-urbit-ship.sh:/urbit/run-urbit-ship.sh
+      - ../config/urbit/deploy-app.sh:/urbit/deploy-app.sh
+    ports:
+      - "80"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "80"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+
+volumes:
+  urbit_data:
+  urbit_app_builds:

stack_orchestrator/data/compose/docker-compose-grafana.yml

@@ -0,0 +1,23 @@
+version: "3.7"
+
+services:
+  grafana:
+    image: grafana/grafana:10.2.2
+    restart: always
+    environment:
+      GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL}
+    volumes:
+      - ../config/monitoring/grafana/provisioning:/etc/grafana/provisioning
+      - ../config/monitoring/grafana/dashboards:/etc/grafana/dashboards
+      - grafana_storage:/var/lib/grafana
+    ports:
+      - "3000"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "3000"]
+      interval: 30s
+      timeout: 5s
+      retries: 10
+      start_period: 3s
+
+volumes:
+  grafana_storage:

stack_orchestrator/data/compose/docker-compose-kubo.yml

@@ -1,13 +1,24 @@
 version: "3.2"
+
 # See: https://docs.ipfs.tech/install/run-ipfs-inside-docker/#set-up
 services:
   ipfs:
-    image: ipfs/kubo:master-2023-02-20-714a968
+    image: ipfs/kubo:v0.24.0
     restart: always
     volumes:
-      - ./ipfs/import:/import
-      - ./ipfs/data:/data/ipfs
+      - ipfs-import:/import
+      - ipfs-data:/data/ipfs
     ports:
-      - "0.0.0.0:8080:8080"
-      - "0.0.0.0:4001:4001"
+      - "4001"
+      - "8080"
       - "0.0.0.0:5001:5001"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5001"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+
+volumes:
+  ipfs-import:
+  ipfs-data:

stack_orchestrator/data/compose/docker-compose-node-exporter.yml

@@ -0,0 +1,15 @@
+version: '3.8'
+
+services:
+  node-exporter:
+    image: prom/node-exporter:latest
+    restart: unless-stopped
+    command:
+      - '--web.listen-address=:9100'
+      - '--path.rootfs=/host'
+      - '--collector.systemd'
+      - '--collector.processes'
+    network_mode: host
+    pid: host
+    volumes:
+      - '/:/host:ro,rslave'

stack_orchestrator/data/compose/docker-compose-osmosis-front-end-urbit.yml

@@ -0,0 +1,22 @@
+version: "3.2"
+
+services:
+  osmosis-front-end:
+    image: cerc/osmosis-front-end-urbit:local
+    restart: on-failure
+    environment:
+      - NEXT_PUBLIC_WEB_API_BASE_URL=${CERC_WEB_API_BASE_URL}
+      - ASSET_LIST_COMMIT_HASH=a326bcefc51372b4912be5a2a2fa84a5d142a438
+      - NEXT_PUBLIC_BASEPATH=/apps/osmosis
+      - NEXT_PUBLIC_URBIT_DEPLOYMENT=true
+    working_dir: /app/packages/web
+    command: ["./build-app-for-urbit.sh"]
+    volumes:
+      - ../config/osmosis/build-app-for-urbit.sh:/app/packages/web/build-app-for-urbit.sh
+      - ../config/osmosis/.env.production:/app/packages/web/.env.production
+      - urbit_app_builds:/app-builds
+      - ../config/osmosis/urbit-files/mar:/app/packages/web/mar
+      - ../config/osmosis/urbit-files/desk.docket-0:/app/packages/web/desk.docket-0
+
+volumes:
+  urbit_app_builds:

stack_orchestrator/data/compose/docker-compose-osmosis-front-end.yml

@@ -0,0 +1,36 @@
+version: "3.2"
+
+services:
+  osmosis-front-end:
+    image: cerc/osmosis-front-end:local
+    restart: on-failure
+    environment:
+      - NEXT_PUBLIC_WEB_API_BASE_URL=${CERC_WEB_API_BASE_URL}
+      - ASSET_LIST_COMMIT_HASH=a326bcefc51372b4912be5a2a2fa84a5d142a438
+    working_dir: /app/packages/web
+    command: ["./build-app.sh"]
+    volumes:
+      - ../config/osmosis/build-app.sh:/app/packages/web/build-app.sh
+      - ../config/osmosis/.env.production:/app/packages/web/.env.production
+      - app_builds:/app-builds
+
+  nginx:
+    image: nginx:1.23-alpine
+    restart: always
+    depends_on:
+      osmosis-front-end:
+        condition: service_completed_successfully
+    volumes:
+      - ../config/osmosis/nginx:/etc/nginx/conf.d
+      - app_builds:/usr/share/nginx
+    ports:
+      - "80"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "80"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+
+volumes:
+  app_builds:

stack_orchestrator/data/compose/docker-compose-prom-server.yml

@@ -0,0 +1,57 @@
+version: "3.7"
+
+services:
+  prometheus:
+    image: prom/prometheus:v2.49.1
+    restart: always
+    volumes:
+      - ../config/monitoring/prometheus:/etc/prometheus
+      - prometheus_data:/prometheus
+    ports:
+      - "9090"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "9090"]
+      interval: 30s
+      timeout: 5s
+      retries: 10
+      start_period: 3s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  blackbox:
+    image: prom/blackbox-exporter:latest
+    restart: always
+    volumes:
+      - ../config/monitoring/blackbox.yml:/etc/blackbox_exporter/config.yml
+    ports:
+      - '9115'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  chain-head-exporter:
+    image: cerc/watcher-ts:local
+    restart: always
+    working_dir: /app/packages/cli
+    environment:
+      ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      FIL_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT}
+      ETH_RPC_API_KEY: ${CERC_INFURA_KEY}
+      PORT: ${CERC_METRICS_PORT}
+    command: ["sh", "-c", "yarn export-metrics:chain-heads"]
+    ports:
+      - '5000'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  postgres-exporter:
+    image: quay.io/prometheuscommunity/postgres-exporter
+    restart: always
+    volumes:
+      - ../config/monitoring/postgres-exporter.yml:/postgres_exporter.yml
+    ports:
+      - '9187'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+volumes:
+  prometheus_data:

stack_orchestrator/data/compose/docker-compose-proxy-server.yml

@@ -0,0 +1,22 @@
+version: "3.2"
+
+services:
+  proxy-server:
+    image: cerc/watcher-ts:local
+    restart: on-failure
+    working_dir: /app/packages/cli
+    environment:
+      ENABLE_PROXY: ${CERC_ENABLE_PROXY:-true}
+      PROXY_UPSTREAM: ${CERC_PROXY_UPSTREAM}
+      PROXY_ORIGIN_HEADER: ${CERC_PROXY_ORIGIN_HEADER}
+    command: ["sh", "-c", "./run.sh"]
+    volumes:
+      - ../config/proxy-server/run.sh:/app/packages/cli/run.sh
+    ports:
+      - "4000"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "4000"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s

stack_orchestrator/data/compose/docker-compose-uniswap-interface.yml

@@ -0,0 +1,18 @@
+version: "3.2"
+
+services:
+  uniswap-interface:
+    image: cerc/uniswap-interface:local
+    restart: on-failure
+    environment:
+      - REACT_APP_INFURA_KEY=${CERC_INFURA_KEY}
+      - REACT_APP_AWS_API_ENDPOINT=${CERC_UNISWAP_GQL}
+    command: ["./build-app.sh"]
+    volumes:
+      - ../config/uniswap-interface/build-app.sh:/app/build-app.sh
+      - urbit_app_builds:/app-builds
+      - ../config/uniswap-interface/urbit-files/mar:/app/mar
+      - ../config/uniswap-interface/urbit-files/desk.docket-0:/app/desk.docket-0
+
+volumes:
+  urbit_app_builds:

stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml

@@ -10,6 +10,7 @@ services:
       - POSTGRES_MULTIPLE_DATABASES=azimuth-watcher,azimuth-watcher-job-queue,censures-watcher,censures-watcher-job-queue,claims-watcher,claims-watcher-job-queue,conditional-star-release-watcher,conditional-star-release-watcher-job-queue,delegated-sending-watcher,delegated-sending-watcher-job-queue,ecliptic-watcher,ecliptic-watcher-job-queue,linear-star-release-watcher,linear-star-release-watcher-job-queue,polls-watcher,polls-watcher-job-queue
       - POSTGRES_EXTENSION=azimuth-watcher-job-queue:pgcrypto,censures-watcher-job-queue:pgcrypto,claims-watcher-job-queue:pgcrypto,conditional-star-release-watcher-job-queue:pgcrypto,delegated-sending-watcher-job-queue:pgcrypto,ecliptic-watcher-job-queue:pgcrypto,linear-star-release-watcher-job-queue:pgcrypto,polls-watcher-job-queue:pgcrypto,
       - POSTGRES_PASSWORD=password
+    command: ["postgres", "-c", "max_connections=200"]
     volumes:
       - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
       - watcher_db_data:/var/lib/postgresql/data
@@ -22,6 +23,38 @@ services:
       retries: 15
       start_period: 10s
 
+  # Starts the azimuth-watcher job runner
+  azimuth-watcher-job-runner:
+    image: cerc/watcher-azimuth:local
+    restart: unless-stopped
+    depends_on:
+      watcher-db:
+        condition: service_healthy
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_HISTORICAL_BLOCK_RANGE: 500
+      CONTRACT_ADDRESS: 0x223c067F8CF28ae173EE5CafEa60cA44C335fecB
+      CONTRACT_NAME: Azimuth
+      STARTING_BLOCK: 6784880
+    working_dir: /app/packages/azimuth-watcher
+    command: "./start-job-runner.sh"
+    volumes:
+      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/azimuth-watcher/environments/watcher-config-template.toml
+      - ../config/watcher-azimuth/merge-toml.js:/app/packages/azimuth-watcher/merge-toml.js
+      - ../config/watcher-azimuth/start-job-runner.sh:/app/packages/azimuth-watcher/start-job-runner.sh
+    ports:
+      - "9000"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "9000"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
   # Starts the azimuth-watcher server
   azimuth-watcher-server:
     image: cerc/watcher-azimuth:local
@@ -29,8 +62,8 @@ services:
     depends_on:
       watcher-db:
         condition: service_healthy
-    env_file:
-      - ../config/watcher-azimuth/watcher-params.env
+      azimuth-watcher-job-runner:
+        condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
       CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
@@ -52,6 +85,37 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
+  # Starts the censures-watcher job runner
+  censures-watcher-job-runner:
+    image: cerc/watcher-azimuth:local
+    restart: unless-stopped
+    depends_on:
+      watcher-db:
+        condition: service_healthy
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CONTRACT_ADDRESS: 0x325f68d32BdEe6Ed86E7235ff2480e2A433D6189
+      CONTRACT_NAME: Censures
+      STARTING_BLOCK: 6784954
+    working_dir: /app/packages/censures-watcher
+    command: "./start-job-runner.sh"
+    volumes:
+      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/censures-watcher/environments/watcher-config-template.toml
+      - ../config/watcher-azimuth/merge-toml.js:/app/packages/censures-watcher/merge-toml.js
+      - ../config/watcher-azimuth/start-job-runner.sh:/app/packages/censures-watcher/start-job-runner.sh
+    ports:
+      - "9002"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "9002"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
   # Starts the censures-watcher server
   censures-watcher-server:
     image: cerc/watcher-azimuth:local
@@ -59,8 +123,8 @@ services:
     depends_on:
       watcher-db:
         condition: service_healthy
-    env_file:
-      - ../config/watcher-azimuth/watcher-params.env
+      censures-watcher-job-runner:
+        condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
       CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
@@ -82,6 +146,37 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
+  # Starts the claims-watcher job runner
+  claims-watcher-job-runner:
+    image: cerc/watcher-azimuth:local
+    restart: unless-stopped
+    depends_on:
+      watcher-db:
+        condition: service_healthy
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CONTRACT_ADDRESS: 0xe7e7f69b34D7d9Bd8d61Fb22C33b22708947971A
+      CONTRACT_NAME: Claims
+      STARTING_BLOCK: 6784941
+    working_dir: /app/packages/claims-watcher
+    command: "./start-job-runner.sh"
+    volumes:
+      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/claims-watcher/environments/watcher-config-template.toml
+      - ../config/watcher-azimuth/merge-toml.js:/app/packages/claims-watcher/merge-toml.js
+      - ../config/watcher-azimuth/start-job-runner.sh:/app/packages/claims-watcher/start-job-runner.sh
+    ports:
+      - "9004"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "9004"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
   # Starts the claims-watcher server
   claims-watcher-server:
     image: cerc/watcher-azimuth:local
@@ -89,8 +184,8 @@ services:
     depends_on:
       watcher-db:
         condition: service_healthy
-    env_file:
-      - ../config/watcher-azimuth/watcher-params.env
+      claims-watcher-job-runner:
+        condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
       CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
@@ -112,6 +207,37 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
+  # Starts the conditional-star-release-watcher job runner
+  conditional-star-release-watcher-job-runner:
+    image: cerc/watcher-azimuth:local
+    restart: unless-stopped
+    depends_on:
+      watcher-db:
+        condition: service_healthy
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CONTRACT_ADDRESS: 0x8C241098C3D3498Fe1261421633FD57986D74AeA
+      CONTRACT_NAME: ConditionalStarRelease
+      STARTING_BLOCK: 6828004
+    working_dir: /app/packages/conditional-star-release-watcher
+    command: "./start-job-runner.sh"
+    volumes:
+      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/conditional-star-release-watcher/environments/watcher-config-template.toml
+      - ../config/watcher-azimuth/merge-toml.js:/app/packages/conditional-star-release-watcher/merge-toml.js
+      - ../config/watcher-azimuth/start-job-runner.sh:/app/packages/conditional-star-release-watcher/start-job-runner.sh
+    ports:
+      - "9006"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "9006"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
   # Starts the conditional-star-release-watcher server
   conditional-star-release-watcher-server:
     image: cerc/watcher-azimuth:local
@@ -119,8 +245,8 @@ services:
     depends_on:
       watcher-db:
         condition: service_healthy
-    env_file:
-      - ../config/watcher-azimuth/watcher-params.env
+      conditional-star-release-watcher-job-runner:
+        condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
       CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
@@ -142,6 +268,37 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
+  # Starts the delegated-sending-watcher job runner
+  delegated-sending-watcher-job-runner:
+    image: cerc/watcher-azimuth:local
+    restart: unless-stopped
+    depends_on:
+      watcher-db:
+        condition: service_healthy
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CONTRACT_ADDRESS: 0xf6b461fE1aD4bd2ce25B23Fe0aff2ac19B3dFA76
+      CONTRACT_NAME: DelegatedSending
+      STARTING_BLOCK: 6784956
+    working_dir: /app/packages/delegated-sending-watcher
+    command: "./start-job-runner.sh"
+    volumes:
+      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/delegated-sending-watcher/environments/watcher-config-template.toml
+      - ../config/watcher-azimuth/merge-toml.js:/app/packages/delegated-sending-watcher/merge-toml.js
+      - ../config/watcher-azimuth/start-job-runner.sh:/app/packages/delegated-sending-watcher/start-job-runner.sh
+    ports:
+      - "9008"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "9008"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
   # Starts the delegated-sending-watcher server
   delegated-sending-watcher-server:
     image: cerc/watcher-azimuth:local
@@ -149,8 +306,8 @@ services:
     depends_on:
       watcher-db:
         condition: service_healthy
-    env_file:
-      - ../config/watcher-azimuth/watcher-params.env
+      delegated-sending-watcher-job-runner:
+        condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
       CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
@@ -172,6 +329,37 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
+  # Starts the ecliptic-watcher job runner
+  ecliptic-watcher-job-runner:
+    image: cerc/watcher-azimuth:local
+    restart: unless-stopped
+    depends_on:
+      watcher-db:
+        condition: service_healthy
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CONTRACT_ADDRESS: 0x33EeCbf908478C10614626A9D304bfe18B78DD73
+      CONTRACT_NAME: Ecliptic
+      STARTING_BLOCK: 13692129
+    working_dir: /app/packages/ecliptic-watcher
+    command: "./start-job-runner.sh"
+    volumes:
+      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/ecliptic-watcher/environments/watcher-config-template.toml
+      - ../config/watcher-azimuth/merge-toml.js:/app/packages/ecliptic-watcher/merge-toml.js
+      - ../config/watcher-azimuth/start-job-runner.sh:/app/packages/ecliptic-watcher/start-job-runner.sh
+    ports:
+      - "9010"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "9010"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
   # Starts the ecliptic-watcher server
   ecliptic-watcher-server:
     image: cerc/watcher-azimuth:local
@@ -179,8 +367,8 @@ services:
     depends_on:
       watcher-db:
         condition: service_healthy
-    env_file:
-      - ../config/watcher-azimuth/watcher-params.env
+      ecliptic-watcher-job-runner:
+        condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
       CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
@@ -202,6 +390,37 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
+  # Starts the linear-star-release-watcher job runner
+  linear-star-release-watcher-job-runner:
+    image: cerc/watcher-azimuth:local
+    restart: unless-stopped
+    depends_on:
+      watcher-db:
+        condition: service_healthy
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CONTRACT_ADDRESS: 0x86cd9cd0992F04231751E3761De45cEceA5d1801
+      CONTRACT_NAME: LinearStarRelease
+      STARTING_BLOCK: 6784943
+    working_dir: /app/packages/linear-star-release-watcher
+    command: "./start-job-runner.sh"
+    volumes:
+      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/linear-star-release-watcher/environments/watcher-config-template.toml
+      - ../config/watcher-azimuth/merge-toml.js:/app/packages/linear-star-release-watcher/merge-toml.js
+      - ../config/watcher-azimuth/start-job-runner.sh:/app/packages/linear-star-release-watcher/start-job-runner.sh
+    ports:
+      - "9012"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "9012"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
   # Starts the linear-star-release-watcher server
   linear-star-release-watcher-server:
     image: cerc/watcher-azimuth:local
@@ -209,8 +428,8 @@ services:
     depends_on:
       watcher-db:
         condition: service_healthy
-    env_file:
-      - ../config/watcher-azimuth/watcher-params.env
+      linear-star-release-watcher-job-runner:
+        condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
       CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
@@ -232,6 +451,37 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
+  # Starts the polls-watcher job runner
+  polls-watcher-job-runner:
+    image: cerc/watcher-azimuth:local
+    restart: unless-stopped
+    depends_on:
+      watcher-db:
+        condition: service_healthy
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CONTRACT_ADDRESS: 0x7fEcaB617c868Bb5996d99D95200D2Fa708218e4
+      CONTRACT_NAME: Polls
+      STARTING_BLOCK: 6784912
+    working_dir: /app/packages/polls-watcher
+    command: "./start-job-runner.sh"
+    volumes:
+      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/polls-watcher/environments/watcher-config-template.toml
+      - ../config/watcher-azimuth/merge-toml.js:/app/packages/polls-watcher/merge-toml.js
+      - ../config/watcher-azimuth/start-job-runner.sh:/app/packages/polls-watcher/start-job-runner.sh
+    ports:
+      - "9014"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "9014"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
   # Starts the polls-watcher server
   polls-watcher-server:
     image: cerc/watcher-azimuth:local
@@ -239,8 +489,8 @@ services:
     depends_on:
       watcher-db:
         condition: service_healthy
-    env_file:
-      - ../config/watcher-azimuth/watcher-params.env
+      polls-watcher-job-runner:
+        condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
       CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}

stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml

@@ -0,0 +1,76 @@
+version: '3.2'
+
+services:
+  merkl-sushiswap-v3-watcher-db:
+    restart: unless-stopped
+    image: postgres:14-alpine
+    environment:
+      - POSTGRES_USER=vdbm
+      - POSTGRES_MULTIPLE_DATABASES=merkl-sushiswap-v3-watcher,merkl-sushiswap-v3-watcher-job-queue
+      - POSTGRES_EXTENSION=merkl-sushiswap-v3-watcher-job-queue:pgcrypto
+      - POSTGRES_PASSWORD=password
+    volumes:
+      - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
+      - merkl_sushiswap_v3_watcher_db_data:/var/lib/postgresql/data
+    ports:
+      - "5432"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+
+  merkl-sushiswap-v3-watcher-job-runner:
+    restart: unless-stopped
+    depends_on:
+      merkl-sushiswap-v3-watcher-db:
+        condition: service_healthy
+    image: cerc/watcher-merkl-sushiswap-v3:local
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+    command: ["bash", "./start-job-runner.sh"]
+    volumes:
+      - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
+      - ../config/watcher-merkl-sushiswap-v3/start-job-runner.sh:/app/start-job-runner.sh
+    ports:
+      - "9002:9000"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  merkl-sushiswap-v3-watcher-server:
+    restart: unless-stopped
+    depends_on:
+      merkl-sushiswap-v3-watcher-db:
+        condition: service_healthy
+      merkl-sushiswap-v3-watcher-job-runner:
+        condition: service_healthy
+    image: cerc/watcher-merkl-sushiswap-v3:local
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+    command: ["bash", "./start-server.sh"]
+    volumes:
+      - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
+      - ../config/watcher-merkl-sushiswap-v3/start-server.sh:/app/start-server.sh
+    ports:
+      - "127.0.0.1:3007:3008"
+      - "9003:9001"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+volumes:
+  merkl_sushiswap_v3_watcher_db_data:

stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml

@@ -0,0 +1,76 @@
+version: '3.2'
+
+services:
+  sushiswap-v3-watcher-db:
+    restart: unless-stopped
+    image: postgres:14-alpine
+    environment:
+      - POSTGRES_USER=vdbm
+      - POSTGRES_MULTIPLE_DATABASES=sushiswap-v3-watcher,sushiswap-v3-watcher-job-queue
+      - POSTGRES_EXTENSION=sushiswap-v3-watcher-job-queue:pgcrypto
+      - POSTGRES_PASSWORD=password
+    volumes:
+      - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
+      - sushiswap_v3_watcher_db_data:/var/lib/postgresql/data
+    ports:
+      - "5432"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+
+  sushiswap-v3-watcher-job-runner:
+    restart: unless-stopped
+    depends_on:
+      sushiswap-v3-watcher-db:
+        condition: service_healthy
+    image: cerc/watcher-sushiswap-v3:local
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+    command: ["bash", "./start-job-runner.sh"]
+    volumes:
+      - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
+      - ../config/watcher-sushiswap-v3/start-job-runner.sh:/app/start-job-runner.sh
+    ports:
+      - "9000:9000"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  sushiswap-v3-watcher-server:
+    restart: unless-stopped
+    depends_on:
+      sushiswap-v3-watcher-db:
+        condition: service_healthy
+      sushiswap-v3-watcher-job-runner:
+        condition: service_healthy
+    image: cerc/watcher-sushiswap-v3:local
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+    command: ["bash", "./start-server.sh"]
+    volumes:
+      - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
+      - ../config/watcher-sushiswap-v3/start-server.sh:/app/start-server.sh
+    ports:
+      - "127.0.0.1:3008:3008"
+      - "9001:9001"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+volumes:
+  sushiswap_v3_watcher_db_data:

stack_orchestrator/data/compose/docker-compose-webapp-template.yml

@@ -0,0 +1,8 @@
+services:
+  webapp:
+    image: cerc/webapp-container:local
+    restart: always
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+    ports:
+      - "3000"

stack_orchestrator/data/config/fixturenet-laconicd/create-fixturenet.sh

@@ -14,104 +14,111 @@ LOGLEVEL="info"
 TRACE="--trace"
 # TRACE=""
 
-# validate dependencies are installed
-command -v jq > /dev/null 2>&1 || { echo >&2 "jq not installed. More info: https://stedolan.github.io/jq/download/"; exit 1; }
+if [ "$1" == "clean" ] || [ ! -d "$HOME/.laconicd/data/blockstore.db" ]; then
+  # validate dependencies are installed
+  command -v jq > /dev/null 2>&1 || { echo >&2 "jq not installed. More info: https://stedolan.github.io/jq/download/"; exit 1; }
 
-# remove existing daemon and client
-rm -rf ~/.laconic*
+  # remove existing daemon and client
+  rm -rf $HOME/.laconicd/*
+  rm -rf $HOME/.laconic/*
 
-make install
-
-laconicd config keyring-backend $KEYRING
-laconicd config chain-id $CHAINID
-
-# if $KEY exists it should be deleted
-laconicd keys add $KEY --keyring-backend $KEYRING --algo $KEYALGO
-
-# Set moniker and chain-id for Ethermint (Moniker can be anything, chain-id must be an integer)
-laconicd init $MONIKER --chain-id $CHAINID
-
-# Change parameter token denominations to aphoton
-cat $HOME/.laconicd/config/genesis.json | jq '.app_state["staking"]["params"]["bond_denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-cat $HOME/.laconicd/config/genesis.json | jq '.app_state["crisis"]["constant_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-cat $HOME/.laconicd/config/genesis.json | jq '.app_state["gov"]["deposit_params"]["min_deposit"][0]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-cat $HOME/.laconicd/config/genesis.json | jq '.app_state["mint"]["params"]["mint_denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-# Custom modules
-cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["record_rent"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_commit_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_reveal_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_minimum_bid"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-
-if [[ "$TEST_REGISTRY_EXPIRY" == "true" ]]; then
-  echo "Setting timers for expiry tests."
-
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["record_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_grace_period"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-fi
-
-if [[ "$TEST_AUCTION_ENABLED" == "true" ]]; then
-  echo "Enabling auction and setting timers."
-
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_enabled"]=true' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_grace_period"]="300s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_commits_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_reveals_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-fi
-
-# increase block time (?)
-cat $HOME/.laconicd/config/genesis.json | jq '.consensus_params["block"]["time_iota_ms"]="1000"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-
-# Set gas limit in genesis
-cat $HOME/.laconicd/config/genesis.json | jq '.consensus_params["block"]["max_gas"]="10000000"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-
-# disable produce empty block
-if [[ "$OSTYPE" == "darwin"* ]]; then
-    sed -i '' 's/create_empty_blocks = true/create_empty_blocks = false/g' $HOME/.laconicd/config/config.toml
-  else
-    sed -i 's/create_empty_blocks = true/create_empty_blocks = false/g' $HOME/.laconicd/config/config.toml
-fi
-
-if [[ $1 == "pending" ]]; then
-  if [[ "$OSTYPE" == "darwin"* ]]; then
-      sed -i '' 's/create_empty_blocks_interval = "0s"/create_empty_blocks_interval = "30s"/g' $HOME/.laconicd/config/config.toml
-      sed -i '' 's/timeout_propose = "3s"/timeout_propose = "30s"/g' $HOME/.laconicd/config/config.toml
-      sed -i '' 's/timeout_propose_delta = "500ms"/timeout_propose_delta = "5s"/g' $HOME/.laconicd/config/config.toml
-      sed -i '' 's/timeout_prevote = "1s"/timeout_prevote = "10s"/g' $HOME/.laconicd/config/config.toml
-      sed -i '' 's/timeout_prevote_delta = "500ms"/timeout_prevote_delta = "5s"/g' $HOME/.laconicd/config/config.toml
-      sed -i '' 's/timeout_precommit = "1s"/timeout_precommit = "10s"/g' $HOME/.laconicd/config/config.toml
-      sed -i '' 's/timeout_precommit_delta = "500ms"/timeout_precommit_delta = "5s"/g' $HOME/.laconicd/config/config.toml
-      sed -i '' 's/timeout_commit = "5s"/timeout_commit = "150s"/g' $HOME/.laconicd/config/config.toml
-      sed -i '' 's/timeout_broadcast_tx_commit = "10s"/timeout_broadcast_tx_commit = "150s"/g' $HOME/.laconicd/config/config.toml
-  else
-      sed -i 's/create_empty_blocks_interval = "0s"/create_empty_blocks_interval = "30s"/g' $HOME/.laconicd/config/config.toml
-      sed -i 's/timeout_propose = "3s"/timeout_propose = "30s"/g' $HOME/.laconicd/config/config.toml
-      sed -i 's/timeout_propose_delta = "500ms"/timeout_propose_delta = "5s"/g' $HOME/.laconicd/config/config.toml
-      sed -i 's/timeout_prevote = "1s"/timeout_prevote = "10s"/g' $HOME/.laconicd/config/config.toml
-      sed -i 's/timeout_prevote_delta = "500ms"/timeout_prevote_delta = "5s"/g' $HOME/.laconicd/config/config.toml
-      sed -i 's/timeout_precommit = "1s"/timeout_precommit = "10s"/g' $HOME/.laconicd/config/config.toml
-      sed -i 's/timeout_precommit_delta = "500ms"/timeout_precommit_delta = "5s"/g' $HOME/.laconicd/config/config.toml
-      sed -i 's/timeout_commit = "5s"/timeout_commit = "150s"/g' $HOME/.laconicd/config/config.toml
-      sed -i 's/timeout_broadcast_tx_commit = "10s"/timeout_broadcast_tx_commit = "150s"/g' $HOME/.laconicd/config/config.toml
+  if [ -n "`which make`" ]; then
+    make install
   fi
-fi
 
-# Allocate genesis accounts (cosmos formatted addresses)
-laconicd add-genesis-account $KEY 100000000000000000000000000aphoton --keyring-backend $KEYRING
+  laconicd config keyring-backend $KEYRING
+  laconicd config chain-id $CHAINID
 
-# Sign genesis transaction
-laconicd gentx $KEY 1000000000000000000000aphoton --keyring-backend $KEYRING --chain-id $CHAINID
+  # if $KEY exists it should be deleted
+  laconicd keys add $KEY --keyring-backend $KEYRING --algo $KEYALGO
 
-# Collect genesis tx
-laconicd collect-gentxs
+  # Set moniker and chain-id for Ethermint (Moniker can be anything, chain-id must be an integer)
+  laconicd init $MONIKER --chain-id $CHAINID
 
-# Run this to ensure everything worked and that the genesis file is setup correctly
-laconicd validate-genesis
+  # Change parameter token denominations to aphoton
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["staking"]["params"]["bond_denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["crisis"]["constant_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["gov"]["deposit_params"]["min_deposit"][0]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["mint"]["params"]["mint_denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  # Custom modules
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["record_rent"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_commit_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_reveal_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_minimum_bid"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
 
-if [[ $1 == "pending" ]]; then
-  echo "pending mode is on, please wait for the first block committed."
+  if [[ "$TEST_REGISTRY_EXPIRY" == "true" ]]; then
+    echo "Setting timers for expiry tests."
+
+    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["record_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_grace_period"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  fi
+
+  if [[ "$TEST_AUCTION_ENABLED" == "true" ]]; then
+    echo "Enabling auction and setting timers."
+
+    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_enabled"]=true' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_grace_period"]="300s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_commits_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_reveals_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  fi
+
+  # increase block time (?)
+  cat $HOME/.laconicd/config/genesis.json | jq '.consensus_params["block"]["time_iota_ms"]="1000"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+
+  # Set gas limit in genesis
+  cat $HOME/.laconicd/config/genesis.json | jq '.consensus_params["block"]["max_gas"]="10000000"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+
+  # disable produce empty block
+  if [[ "$OSTYPE" == "darwin"* ]]; then
+      sed -i '' 's/create_empty_blocks = true/create_empty_blocks = false/g' $HOME/.laconicd/config/config.toml
+    else
+      sed -i 's/create_empty_blocks = true/create_empty_blocks = false/g' $HOME/.laconicd/config/config.toml
+  fi
+
+  if [[ $1 == "pending" ]]; then
+    if [[ "$OSTYPE" == "darwin"* ]]; then
+        sed -i '' 's/create_empty_blocks_interval = "0s"/create_empty_blocks_interval = "30s"/g' $HOME/.laconicd/config/config.toml
+        sed -i '' 's/timeout_propose = "3s"/timeout_propose = "30s"/g' $HOME/.laconicd/config/config.toml
+        sed -i '' 's/timeout_propose_delta = "500ms"/timeout_propose_delta = "5s"/g' $HOME/.laconicd/config/config.toml
+        sed -i '' 's/timeout_prevote = "1s"/timeout_prevote = "10s"/g' $HOME/.laconicd/config/config.toml
+        sed -i '' 's/timeout_prevote_delta = "500ms"/timeout_prevote_delta = "5s"/g' $HOME/.laconicd/config/config.toml
+        sed -i '' 's/timeout_precommit = "1s"/timeout_precommit = "10s"/g' $HOME/.laconicd/config/config.toml
+        sed -i '' 's/timeout_precommit_delta = "500ms"/timeout_precommit_delta = "5s"/g' $HOME/.laconicd/config/config.toml
+        sed -i '' 's/timeout_commit = "5s"/timeout_commit = "150s"/g' $HOME/.laconicd/config/config.toml
+        sed -i '' 's/timeout_broadcast_tx_commit = "10s"/timeout_broadcast_tx_commit = "150s"/g' $HOME/.laconicd/config/config.toml
+    else
+        sed -i 's/create_empty_blocks_interval = "0s"/create_empty_blocks_interval = "30s"/g' $HOME/.laconicd/config/config.toml
+        sed -i 's/timeout_propose = "3s"/timeout_propose = "30s"/g' $HOME/.laconicd/config/config.toml
+        sed -i 's/timeout_propose_delta = "500ms"/timeout_propose_delta = "5s"/g' $HOME/.laconicd/config/config.toml
+        sed -i 's/timeout_prevote = "1s"/timeout_prevote = "10s"/g' $HOME/.laconicd/config/config.toml
+        sed -i 's/timeout_prevote_delta = "500ms"/timeout_prevote_delta = "5s"/g' $HOME/.laconicd/config/config.toml
+        sed -i 's/timeout_precommit = "1s"/timeout_precommit = "10s"/g' $HOME/.laconicd/config/config.toml
+        sed -i 's/timeout_precommit_delta = "500ms"/timeout_precommit_delta = "5s"/g' $HOME/.laconicd/config/config.toml
+        sed -i 's/timeout_commit = "5s"/timeout_commit = "150s"/g' $HOME/.laconicd/config/config.toml
+        sed -i 's/timeout_broadcast_tx_commit = "10s"/timeout_broadcast_tx_commit = "150s"/g' $HOME/.laconicd/config/config.toml
+    fi
+  fi
+
+  # Allocate genesis accounts (cosmos formatted addresses)
+  laconicd add-genesis-account $KEY 100000000000000000000000000aphoton --keyring-backend $KEYRING
+
+  # Sign genesis transaction
+  laconicd gentx $KEY 1000000000000000000000aphoton --keyring-backend $KEYRING --chain-id $CHAINID
+
+  # Collect genesis tx
+  laconicd collect-gentxs
+
+  # Run this to ensure everything worked and that the genesis file is setup correctly
+  laconicd validate-genesis
+
+  if [[ $1 == "pending" ]]; then
+    echo "pending mode is on, please wait for the first block committed."
+  fi
+else
+  echo "Using existing database at $HOME/.laconicd.  To replace, run '`basename $0` clean'"
 fi
 
 # Start the node (remove the --pruning=nothing flag if historical queries are not needed)

stack_orchestrator/data/config/monitoring/blackbox.yml

@@ -0,0 +1,7 @@
+modules:
+  http_2xx:
+    prober: http
+    timeout: 5s
+    http:
+      valid_status_codes: [] #default to 2xx
+      method: GET

stack_orchestrator/data/config/monitoring/grafana/dashboards/nodejs-app-dashboard.json

@@ -0,0 +1,943 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "node.js prometheus client basic metrics",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "gnetId": 11159,
+  "graphTooltip": 0,
+  "id": 15,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 10,
+        "x": 0,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 6,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "irate(process_cpu_user_seconds_total{instance=~\"$instance\"}[2m]) * 100",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "User CPU - {{instance}}",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "irate(process_cpu_system_seconds_total{instance=~\"$instance\"}[2m]) * 100",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Sys CPU - {{instance}}",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Process CPU Usage",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "percent",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 9,
+        "x": 10,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 8,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_eventloop_lag_seconds{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "{{instance}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Event Loop Lag",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "s",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "text",
+            "mode": "fixed"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 5,
+        "x": 19,
+        "y": 0
+      },
+      "id": 2,
+      "interval": "",
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "name",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.2.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(nodejs_version_info{instance=~\"$instance\"}) by (version)",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{version}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Node.js Version",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "#F2495C",
+            "mode": "fixed"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 5,
+        "x": 19,
+        "y": 3
+      },
+      "id": 4,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.2.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(changes(process_start_time_seconds{instance=~\"$instance\"}[1m]))",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "{{instance}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Process Restart Times",
+      "type": "stat"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 16,
+        "x": 0,
+        "y": 7
+      },
+      "hiddenSeries": false,
+      "id": 7,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "rightSide": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "process_resident_memory_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Process Memory - {{instance}}",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_heap_size_total_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Heap Total - {{instance}}",
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_heap_size_used_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Heap Used - {{instance}}",
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_external_memory_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "External Memory - {{instance}}",
+          "refId": "D"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Process Memory Usage",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 8,
+        "x": 16,
+        "y": 7
+      },
+      "hiddenSeries": false,
+      "id": 9,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_active_handles_total{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Active Handler - {{instance}}",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_active_requests_total{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Active Request - {{instance}}",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Active Handlers/Requests Total",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 14
+      },
+      "hiddenSeries": false,
+      "id": 10,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_heap_space_size_total_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Heap Total - {{instance}} - {{space}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Heap Total Detail",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 8,
+        "y": 14
+      },
+      "hiddenSeries": false,
+      "id": 11,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_heap_space_size_used_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Heap Used - {{instance}} - {{space}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Heap Used Detail",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 16,
+        "y": 14
+      },
+      "hiddenSeries": false,
+      "id": 12,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_heap_space_size_available_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Heap Used - {{instance}} - {{space}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Heap Available Detail",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    }
+  ],
+  "refresh": "10s",
+  "schemaVersion": 38,
+  "tags": [
+    "nodejs"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": true,
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
+        "definition": "label_values(nodejs_version_info, instance)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "instance",
+        "multi": true,
+        "name": "instance",
+        "options": [],
+        "query": "label_values(nodejs_version_info, instance)",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-15m",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "",
+  "title": "NodeJS Application Dashboard",
+  "uid": "PTSqcpJWk",
+  "version": 3,
+  "weekStart": ""
+}

stack_orchestrator/data/config/monitoring/grafana/provisioning/alerting/contactpoints.yml

@@ -0,0 +1,14 @@
+# https://www.clever-cloud.com/blog/features/2021/12/03/slack-alerts-for-grafana/
+
+apiVersion: 1
+
+contactPoints:
+  - orgId: 1
+    name: SlackNotifier
+    receivers:
+      - uid: a71b06e3-58b6-41fe-af65-fbbb29653951
+        type: slack
+        settings:
+          # Slack hook URL (see https://api.slack.com/messaging/webhooks)
+          url: <YOUR_SLACK_HOOK_URL>
+        disableResolveMessage: false

stack_orchestrator/data/config/monitoring/grafana/provisioning/alerting/policies.yml

@@ -0,0 +1,15 @@
+# https://grafana.com/docs/grafana/latest/alerting/alerting-rules/create-notification-policy/
+
+apiVersion: 1
+
+policies:
+  - orgId: 1
+    receiver: grafana-default-email
+    group_by:
+      - grafana_folder
+      - alertname
+    routes:
+      - receiver: SlackNotifier
+        object_matchers:
+          # Add matchers below
+          # - ['grafana_folder', '=', 'MyAlerts']

stack_orchestrator/data/config/monitoring/grafana/provisioning/dashboards/dashboards.yml

@@ -0,0 +1,10 @@
+apiVersion: 1
+
+providers:
+  - name: dashboards
+    type: file
+    updateIntervalSeconds: 10
+    allowUiUpdates: true
+    options:
+      path: /etc/grafana/dashboards
+      foldersFromFilesStructure: true

stack_orchestrator/data/config/monitoring/grafana/provisioning/datasources/prometheus.yml

@@ -0,0 +1,16 @@
+apiVersion: 1
+
+datasources:
+  - id: 1
+    orgId: 1
+    name: Prometheus
+    type: prometheus
+    typeName: Prometheus
+    typeLogoUrl: public/app/plugins/datasource/prometheus/img/prometheus_logo.svg
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    jsonData:
+      httpMethod: POST
+    version: 1
+    editable: true

stack_orchestrator/data/config/monitoring/postgres-exporter.yml

@@ -0,0 +1,8 @@
+auth_modules:
+  foo:
+    type: userpass
+    userpass:
+      username: username
+      password: password
+    options:
+      sslmode: disable

stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml

@@ -0,0 +1,67 @@
+global:
+  scrape_interval:     10s
+  evaluation_interval: 15s
+
+rule_files:
+  # - "first.rules"
+  # - "second.rules"
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: node
+    static_configs:
+      # Add node-exporter targets to be monitored below
+      # - targets: ['example-host:9100']
+      #   labels:
+      #     instance: 'my-host'
+
+  - job_name: 'blackbox'
+    scrape_interval: 10s
+    metrics_path: /probe
+    params:
+      module: [http_2xx]
+    static_configs:
+      # Add URLs to be monitored below
+      - targets:
+        # - https://github.com
+    relabel_configs:
+      - source_labels: [__address__]
+        regex: (.*)(:80)?
+        target_label: __param_target
+      - source_labels: [__param_target]
+        regex: (.*)
+        target_label: instance
+        replacement: ${1}
+      - source_labels: []
+        regex: .*
+        target_label: __address__
+        replacement: blackbox:9115
+
+  - job_name: chain_heads
+    scrape_interval: 10s
+    metrics_path: /metrics
+    scheme: http
+    static_configs:
+      - targets: ['chain-head-exporter:5000']
+
+  - job_name: 'postgres'
+    scrape_interval: 30s
+    scrape_timeout: 30s
+    static_configs:
+      # Add DB targets below
+      # - targets: [example-server:5432]
+        # labels:
+          # instance: 'example-label'
+    metrics_path: /probe
+    params:
+      auth_module: [foo]
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: postgres-exporter:9187

stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml

@@ -0,0 +1,933 @@
+# https://grafana.com/docs/grafana/latest/alerting/alerting-rules/create-grafana-managed-rule/
+
+apiVersion: 1
+
+groups:
+  - orgId: 1
+    name: watcher
+    folder: WatcherAlerts
+    interval: 30s
+    rules:
+      # Azimuth
+      - uid: azimuth_diff_external
+        title: azimuth_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: censures_diff_external
+        title: censures_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: claims_diff_external
+        title: claims_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: conditional_star_release_diff_external
+        title: conditional_star_release_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: delegated_sending_diff_external
+        title: delegated_sending_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: ecliptic_diff_external
+        title: ecliptic_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: linear_star_release_diff_external
+        title: linear_star_release_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="linear_star_release", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: polls_diff_external
+        title: polls_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+
+      # Sushi
+      - uid: sushiswap_diff_external
+        title: sushiswap_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: merkl_sushiswap_diff_external
+        title: merkl_sushiswap_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false

stack_orchestrator/data/config/osmosis/.env.production

@@ -0,0 +1,3 @@
+NEXT_PUBLIC_COINGECKO_URL=https://api.coingecko.com
+
+NEXT_PUBLIC_ENABLE_FEATURES=true

stack_orchestrator/data/config/osmosis/build-app-for-urbit.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+# Check and exit if a deployment already exists (on restarts)
+if [ -d /app-builds/osmosis/build ]; then
+  echo "Build already exists, remove volume to rebuild"
+  exit 0
+fi
+
+yarn build:static
+./build-urbit.sh
+
+# Move build to app-builds
+mkdir -p /app-builds/osmosis
+cp -r ./out /app-builds/osmosis/build
+
+cp -r mar /app-builds/osmosis/
+cp desk.docket-0 /app-builds/osmosis/

stack_orchestrator/data/config/osmosis/build-app.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+# Check and exit if a deployment already exists (on restarts)
+if [ -d /app-builds/osmosis/build ]; then
+  echo "Build already exists, remove volume to rebuild"
+  exit 0
+fi
+
+yarn build:static
+
+# Move build to app-builds
+mkdir -p /app-builds/osmosis
+cp -r ./out /app-builds/osmosis/build

stack_orchestrator/data/config/osmosis/nginx/web-app.conf

@@ -0,0 +1,16 @@
+server {
+    listen       80;
+    listen  [::]:80;
+    server_name  _;
+
+    error_page   500 502 503 504  /50x.html;
+
+    location / {
+        root   /usr/share/nginx/osmosis/build;
+        index index.html index.htm index.nginx-debian.html;
+
+        # First attempt to serve request as file, then as html,
+		# then as directory, then fall back to displaying a 404.
+		try_files $uri $uri.html $uri/ /index.html =404;
+    }
+}

stack_orchestrator/data/config/osmosis/urbit-files/desk.docket-0

@@ -0,0 +1,10 @@
+:~  title+'Osmosis'
+    info+'Osmosis DEX Frontend, built and maintained by Laconic'
+    color+0xcd.75df
+    image+'https://altcoinsbox.com/wp-content/uploads/2023/03/osmosis-logo.png'
+    base+'osmosis'
+    glob-http+['REPLACE_WITH_GLOB_URL' REPLACE_WITH_GLOB_HASH]
+    version+[0 0 2]
+    website+'https://osmosis.zone/'
+    license+'MIT'
+==

stack_orchestrator/data/config/osmosis/urbit-files/mar/ico.hoon

@@ -0,0 +1,12 @@
+|_  dat=@
+++  grow
+  |%
+  ++  mime  [/image/x-icon (as-octs:mimes:html dat)]
+  --
+++  grab
+  |%
+  ++  mime  |=([p=mite q=octs] q.q)
+  ++  noun  @
+  --
+++  grad  %mime
+--

stack_orchestrator/data/config/osmosis/urbit-files/mar/jpg.hoon

@@ -0,0 +1,12 @@
+|_  dat=@
+++  grow
+  |%
+  ++  mime  [/image/jpeg (as-octs:mimes:html dat)]
+  --
+++  grab
+  |%
+  ++  mime  |=([p=mite q=octs] q.q)
+  ++  noun  @
+  --
+++  grad  %mime
+--

stack_orchestrator/data/config/osmosis/urbit-files/mar/map.hoon

@@ -0,0 +1,18 @@
+::
+::::  /hoon/map/mar
+  ::  Mark for js source maps
+/?    310
+::
+=,  eyre
+|_  mud=@
+++  grow
+  |%
+  ++  mime  [/application/octet-stream (as-octs:mimes:html (@t mud))]
+  --
+++  grab
+  |%                                                    ::  convert from
+  ++  mime  |=([p=mite q=octs] (@t q.q))
+  ++  noun  cord                                        ::  clam from %noun
+  --
+++  grad  %mime
+--

stack_orchestrator/data/config/osmosis/urbit-files/mar/webp.hoon

@@ -0,0 +1,12 @@
+|_  dat=@
+++  grow
+  |%
+  ++  mime  [/image/webp (as-octs:mimes:html dat)]
+  --
+++  grab
+  |%
+  ++  mime  |=([p=mite q=octs] q.q)
+  ++  noun  @
+  --
+++  grad  %mime
+--

stack_orchestrator/data/config/proxy-server/run.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+if [ "$ENABLE_PROXY" = "true" ]; then
+  echo "Proxy server enabled"
+  yarn proxy
+else
+  echo "Proxy server disabled, exiting"
+  exit 0
+fi

stack_orchestrator/data/config/uniswap-interface/build-app.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+# Check and exit if a deployment already exists (on restarts)
+if [ -d /app-builds/uniswap/build ]; then
+  echo "Build already exists, remove volume to rebuild"
+  exit 0
+fi
+
+yarn build
+
+# Copy over build and other files to app-builds for urbit deployment
+mkdir -p /app-builds/uniswap
+cp -r ./build /app-builds/uniswap/
+
+cp -r mar /app-builds/uniswap/
+cp desk.docket-0 /app-builds/uniswap/

stack_orchestrator/data/config/uniswap-interface/urbit-files/desk.docket-0

@@ -0,0 +1,10 @@
+:~  title+'Uniswap'
+    info+'Self-hosted uniswap frontend.'
+    color+0xcd.75df
+    image+'https://logowik.com/content/uploads/images/uniswap-uni7403.jpg'
+    base+'uniswap'
+    glob-http+['REPLACE_WITH_GLOB_URL' REPLACE_WITH_GLOB_HASH]
+    version+[0 0 1]
+    website+'https://uniswap.org/'
+    license+'MIT'
+==

stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/map.hoon

@@ -0,0 +1,18 @@
+::
+::::  /hoon/map/mar
+  ::  Mark for js source maps
+/?    310
+::
+=,  eyre
+|_  mud=@
+++  grow
+  |%
+  ++  mime  [/application/octet-stream (as-octs:mimes:html (@t mud))]
+  --
+++  grab
+  |%                                                    ::  convert from
+  ++  mime  |=([p=mite q=octs] (@t q.q))
+  ++  noun  cord                                        ::  clam from %noun
+  --
+++  grad  %mime
+--

stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/ttf.hoon

@@ -0,0 +1,12 @@
+|_  dat=octs
+++  grow
+  |%
+  ++  mime  [/font/ttf dat]
+  --
+++  grab
+  |%
+  ++  mime  |=([=mite =octs] octs)
+  ++  noun  octs
+  --
+++  grad  %mime
+--

stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/woff.hoon

@@ -0,0 +1,12 @@
+|_  dat=octs
+++  grow
+  |%
+  ++  mime  [/font/woff dat]
+  --
+++  grab
+  |%
+  ++  mime  |=([=mite =octs] octs)
+  ++  noun  octs
+  --
+++  grad  %mime
+--

stack_orchestrator/data/config/urbit/deploy-app-to-remote-urbit.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# $1: Remote user host
+# $2: App name (eg. uniswap)
+# $3: Assets dir path (local) for app (eg. /home/user/myapp/urbit-files)
+# $4: Remote Urbit ship's pier dir path (eg. /home/user/zod)
+# $5: Glob file URL (eg. https://xyz.com/glob-0vabcd.glob)
+# $6: Glob file hash (eg. 0vabcd)
+
+if [ "$#" -ne 6 ]; then
+  echo "Incorrect number of arguments"
+  echo "Usage: $0 <username@remote_host> <app_name> </path/to/app/assets/folder> </path/to/remote/pier/folder> <glob_url> <glob_hash>"
+  exit 1
+fi
+
+remote_user_host="$1"
+app_name=$2
+app_assets_folder=$3
+remote_pier_folder="$4"
+glob_url="$5"
+glob_hash="$6"
+
+installation_script="./install-urbit-app.sh"
+
+# Copy over the assets to remote machine in a tmp dir
+remote_app_assets_folder=/tmp/urbit-app-assets/$app_name
+ssh "$remote_user_host" "mkdir -p $remote_app_assets_folder"
+scp -r $app_assets_folder/* $remote_user_host:$remote_app_assets_folder
+
+# Run the installation script
+ssh "$remote_user_host" "bash -s $app_name $remote_app_assets_folder '${glob_url}' $glob_hash $remote_pier_folder" < "$installation_script"
+
+# Remove the tmp assets dir
+ssh "$remote_user_host" "rm -rf $remote_app_assets_folder"

stack_orchestrator/data/config/urbit/deploy-app.sh

@@ -0,0 +1,110 @@
+#!/bin/bash
+
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+if [ -z "$CERC_URBIT_APP" ]; then
+  echo "CERC_URBIT_APP not set, exiting"
+  exit 0
+fi
+
+echo "Creating Urbit application for ${CERC_URBIT_APP}"
+
+app_desk_dir=/urbit/zod/${CERC_URBIT_APP}
+if [ -d ${app_desk_dir} ]; then
+  echo "Desk dir already exists for ${CERC_URBIT_APP}, skipping deployment..."
+  exit 0
+fi
+
+app_build=/app-builds/${CERC_URBIT_APP}/build
+app_mark_files=/app-builds/${CERC_URBIT_APP}/mar
+app_docket_file=/app-builds/${CERC_URBIT_APP}/desk.docket-0
+
+echo "Reading app build from ${app_build}"
+echo "Reading additional mark files from ${app_mark_files}"
+echo "Reading docket file ${app_docket_file}"
+
+# Loop until the app's build appears
+while [ ! -d ${app_build} ]; do
+  echo "${CERC_URBIT_APP} app build not found, retrying in 5s..."
+  sleep 5
+done
+echo "Build found..."
+
+echo "Using IPFS endpoint ${CERC_IPFS_GLOB_HOST_ENDPOINT} for hosting the ${CERC_URBIT_APP} glob"
+echo "Using IPFS server endpoint ${CERC_IPFS_SERVER_ENDPOINT} for reading ${CERC_URBIT_APP} glob"
+ipfs_host_endpoint=${CERC_IPFS_GLOB_HOST_ENDPOINT}
+ipfs_server_endpoint=${CERC_IPFS_SERVER_ENDPOINT}
+
+# Fire curl requests to perform operations on the ship
+dojo () {
+  curl -s --data '{"source":{"dojo":"'"$1"'"},"sink":{"stdout":null}}' http://localhost:12321
+}
+
+hood () {
+  curl -s --data '{"source":{"dojo":"+hood/'"$1"'"},"sink":{"app":"hood"}}' http://localhost:12321
+}
+
+# Create / mount the app's desk
+hood "merge %${CERC_URBIT_APP} our %landscape"
+hood "mount %${CERC_URBIT_APP}"
+
+# Copy over build to desk data dir
+cp -r ${app_build} ${app_desk_dir}
+
+# Copy over the additional mark files
+cp ${app_mark_files}/* ${app_desk_dir}/mar/
+
+rm "${app_desk_dir}/desk.bill"
+rm "${app_desk_dir}/desk.ship"
+
+# Commit changes and create a glob
+hood "commit %${CERC_URBIT_APP}"
+dojo "-landscape!make-glob %${CERC_URBIT_APP} /build"
+
+glob_file=$(ls -1 -c zod/.urb/put | head -1)
+echo "Created glob file: ${glob_file}"
+
+# Upload the glob file to IPFS
+echo "Uploading glob file to ${ipfs_host_endpoint}"
+upload_response=$(curl -X POST -F file=@./zod/.urb/put/${glob_file} ${ipfs_host_endpoint}/api/v0/add)
+glob_cid=$(echo "$upload_response" | grep -o '"Hash":"[^"]*' | sed 's/"Hash":"//')
+
+glob_url="${ipfs_server_endpoint}/ipfs/${glob_cid}?filename=${glob_file}"
+glob_hash=$(echo "$glob_file" | sed "s/glob-\([a-z0-9\.]*\).glob/\1/")
+
+echo "Glob file uploaded to IFPS:"
+echo "{ cid: ${glob_cid}, filename: ${glob_file} }"
+echo "{ url: ${glob_url}, hash: ${glob_hash} }"
+
+# Exit if the installation not required
+if [ "$CERC_ENABLE_APP_INSTALL" = "false" ]; then
+  echo "CERC_ENABLE_APP_INSTALL set to false, skipping app installation"
+  exit 0
+fi
+
+# Curl and wait for the glob to be hosted
+echo "Checking if glob file hosted at ${glob_url}"
+while true; do
+  response=$(curl -sL -w "%{http_code}" -o /dev/null "$glob_url")
+
+  if [ $response -eq 200 ]; then
+    echo "File found at $glob_url"
+    break  # Exit the loop if the file is found
+  else
+    echo "File not found, retrying in a 5s..."
+    sleep 5
+  fi
+done
+
+# Replace the docket file for app
+# Substitue the glob URL and hash
+cp ${app_docket_file} ${app_desk_dir}/
+sed -i "s|REPLACE_WITH_GLOB_URL|${glob_url}|g; s|REPLACE_WITH_GLOB_HASH|${glob_hash}|g" ${app_desk_dir}/desk.docket-0
+
+# Commit changes and install the app
+hood "commit %${CERC_URBIT_APP}"
+hood "install our %${CERC_URBIT_APP}"
+
+echo "${CERC_URBIT_APP} app installed"

stack_orchestrator/data/config/urbit/install-urbit-app.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# $1: App name (eg. uniswap)
+# $2: Assets dir path (local) for app (eg. /home/user/myapp/urbit-files)
+# $3: Glob file URL (eg. https://xyz.com/glob-0vabcd.glob)
+# $4: Glob file hash (eg. 0vabcd)
+# $5: Urbit ship's pier dir (default: ./zod)
+
+if [ "$#" -lt 4 ]; then
+  echo "Insufficient arguments"
+  echo "Usage: $0 <app_name> </path/to/app/assets/folder> <glob_url> <glob_hash> [/path/to/remote/pier/folder]"
+  exit 1
+fi
+
+app_name=$1
+app_mark_files=$2/mar
+app_docket_file=$2/desk.docket-0
+echo "Creating Urbit application for ${app_name}"
+echo "Reading additional mark files from ${app_mark_files}"
+echo "Reading docket file ${app_docket_file}"
+
+glob_url=$3
+glob_hash=$4
+echo "Using glob file from ${glob_url} with hash ${glob_hash}"
+
+# Default pier dir: ./zod
+# Default desk dir: ./zod/<app_name>
+pier_dir="${5:-./zod}"
+app_desk_dir="${pier_dir}/${app_name}"
+echo "Using ${app_desk_dir} as the ${app_name} desk dir path"
+
+# Fire curl requests to perform operations on the ship
+dojo () {
+  curl -s --data '{"source":{"dojo":"'"$1"'"},"sink":{"stdout":null}}' http://localhost:12321
+}
+
+hood () {
+  curl -s --data '{"source":{"dojo":"+hood/'"$1"'"},"sink":{"app":"hood"}}' http://localhost:12321
+}
+
+# Create / mount the app's desk
+hood "merge %${app_name} our %landscape"
+hood "mount %${app_name}"
+
+# Copy over the additional mark files
+cp ${app_mark_files}/* ${app_desk_dir}/mar/
+
+rm "${app_desk_dir}/desk.bill"
+rm "${app_desk_dir}/desk.ship"
+
+# Replace the docket file for app
+# Substitue the glob URL and hash
+cp ${app_docket_file} ${app_desk_dir}/
+sed -i "s|REPLACE_WITH_GLOB_URL|${glob_url}|g; s|REPLACE_WITH_GLOB_HASH|${glob_hash}|g" ${app_desk_dir}/desk.docket-0
+
+# Commit changes and install the app
+hood "commit %${app_name}"
+hood "install our %${app_name}"
+
+echo "${app_name} app installed"

stack_orchestrator/data/config/urbit/run-urbit-ship.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+pier_dir="/urbit/zod"
+
+# TODO: Bootstrap fake ship on the first run
+
+# Run urbit ship in daemon mode
+# Check if the directory exists
+if [ -d "$pier_dir" ]; then
+  echo "Pier directory already exists, rebooting..."
+  /urbit/zod/.run -d
+else
+  echo "Creating a new fake ship..."
+  urbit -d -F zod
+fi

stack_orchestrator/data/config/watcher-azimuth/start-job-runner.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}"
+echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}"
+echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}"
+
+# Replace env variables in template TOML file
+# Read in the config template TOML file and modify it
+WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
+  sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \
+          s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \
+          s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ")
+
+# Write the modified content to a new file
+echo "$WATCHER_CONFIG" > environments/watcher-config.toml
+
+# Merge SO watcher config with existing config file
+node merge-toml.js
+
+yarn watch:contract --address $CONTRACT_ADDRESS --kind $CONTRACT_NAME --checkpoint true --starting-block $STARTING_BLOCK
+
+echo 'yarn job-runner'
+yarn job-runner

stack_orchestrator/data/config/watcher-azimuth/start-server.sh

@@ -4,18 +4,17 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
   set -x
 fi
 
-CERC_IPLD_ETH_RPC="${CERC_IPLD_ETH_RPC:-${DEFAULT_CERC_IPLD_ETH_RPC}}"
-CERC_IPLD_ETH_GQL="${CERC_IPLD_ETH_GQL:-${DEFAULT_CERC_IPLD_ETH_GQL}}"
-
 echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}"
 echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}"
+echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}"
 
 # Replace env variables in template TOML file
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
   sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \
-          s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}| ")
+          s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \
+          s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ")
 
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/watcher-config.toml

stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml

@@ -2,6 +2,9 @@
   host = "0.0.0.0"
   maxSimultaneousRequests = -1
 
+[metrics]
+  host = "0.0.0.0"
+
 [database]
   host = "watcher-db"
   port = 5432
@@ -12,3 +15,7 @@
   [upstream.ethServer]
     gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL"
     rpcProviderEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_RPC"
+
+[jobQueue]
+  historicalLogsBlockRange = REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE
+  blockDelayInMilliSecs = 12000

stack_orchestrator/data/config/watcher-azimuth/watcher-params.env

@@ -1,5 +0,0 @@
-# Defaults
-
-# ipld-eth-server endpoints
-DEFAULT_CERC_IPLD_ETH_RPC=
-DEFAULT_CERC_IPLD_ETH_GQL=

stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-job-runner.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+set -u
+
+echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+
+# Read in the config template TOML file and modify it
+WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+
+# Write the modified content to a new file
+echo "$WATCHER_CONFIG" > environments/local.toml
+
+echo "Running job-runner..."
+DEBUG=vulcanize:* exec node --enable-source-maps dist/job-runner.js

stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-server.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+set -u
+
+echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+
+# Read in the config template TOML file and modify it
+WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+
+# Write the modified content to a new file
+echo "$WATCHER_CONFIG" > environments/local.toml
+
+echo "Running server..."
+DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js

stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml

@@ -0,0 +1,98 @@
+[server]
+  host = "0.0.0.0"
+  port = 3008
+  kind = "active"
+  gqlPath = '/'
+
+  # Checkpointing state.
+  checkpointing = true
+
+  # Checkpoint interval in number of blocks.
+  checkpointInterval = 2000
+
+  # Enable state creation
+  # CAUTION: Disable only if state creation is not desired or can be filled subsequently
+  enableState = false
+
+  subgraphPath = "./subgraph-build"
+
+  # Interval to restart wasm instance periodically
+  wasmRestartBlocksInterval = 20
+
+  # Interval in number of blocks at which to clear entities cache.
+  clearEntitiesCacheInterval = 1000
+
+  # Max block range for which to return events in eventsInRange GQL query.
+  # Use -1 for skipping check on block range.
+  maxEventsBlockRange = 1000
+
+  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
+  rpcSupportsBlockHashParam = false
+
+  # GQL cache settings
+  [server.gqlCache]
+    enabled = true
+
+    # Max in-memory cache size (in bytes) (default 8 MB)
+    # maxCacheSize
+
+    # GQL cache-control max-age settings (in seconds)
+    maxAge = 15
+    timeTravelMaxAge = 86400 # 1 day
+
+[metrics]
+  host = "0.0.0.0"
+  port = 9000
+  [metrics.gql]
+    port = 9001
+
+[database]
+  type = "postgres"
+  host = "merkl-sushiswap-v3-watcher-db"
+  port = 5432
+  database = "merkl-sushiswap-v3-watcher"
+  username = "vdbm"
+  password = "password"
+  synchronize = true
+  logging = false
+
+[upstream]
+  [upstream.ethServer]
+    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+
+    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
+    rpcClient = true
+
+    # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint
+    isFEVM = true
+
+    # Boolean flag to filter event logs by contracts
+    filterLogsByAddresses = true
+    # Boolean flag to filter event logs by topics
+    filterLogsByTopics = false
+
+  [upstream.cache]
+    name = "requests"
+    enabled = false
+    deleteOnStart = false
+
+[jobQueue]
+  dbConnectionString = "postgres://vdbm:password@merkl-sushiswap-v3-watcher-db/merkl-sushiswap-v3-watcher-job-queue"
+  maxCompletionLagInSecs = 300
+  jobDelayInMilliSecs = 100
+  eventsInBatch = 50
+  subgraphEventsOrder = true
+  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
+  blockDelayInMilliSecs = 30000
+
+  # Boolean to switch between modes of processing events when starting the server.
+  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
+  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
+  useBlockRanges = true
+
+  # Block range in which logs are fetched during historical blocks processing
+  historicalLogsBlockRange = 2000
+
+  # Max block range of historical processing after which it waits for completion of events processing
+  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
+  historicalMaxFetchAhead = 10000

stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+set -u
+
+echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+
+# Read in the config template TOML file and modify it
+WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+
+# Write the modified content to a new file
+echo "$WATCHER_CONFIG" > environments/local.toml
+
+echo "Running job-runner..."
+DEBUG=vulcanize:* exec node --enable-source-maps dist/job-runner.js

stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+set -u
+
+echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+
+# Read in the config template TOML file and modify it
+WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+
+# Write the modified content to a new file
+echo "$WATCHER_CONFIG" > environments/local.toml
+
+echo "Running server..."
+DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js

stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml

@@ -0,0 +1,98 @@
+[server]
+  host = "0.0.0.0"
+  port = 3008
+  kind = "active"
+  gqlPath = "/"
+
+  # Checkpointing state.
+  checkpointing = true
+
+  # Checkpoint interval in number of blocks.
+  checkpointInterval = 2000
+
+  # Enable state creation
+  # CAUTION: Disable only if state creation is not desired or can be filled subsequently
+  enableState = false
+
+  subgraphPath = "./subgraph-build"
+
+  # Interval to restart wasm instance periodically
+  wasmRestartBlocksInterval = 20
+
+  # Interval in number of blocks at which to clear entities cache.
+  clearEntitiesCacheInterval = 1000
+
+  # Max block range for which to return events in eventsInRange GQL query.
+  # Use -1 for skipping check on block range.
+  maxEventsBlockRange = 1000
+
+  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
+  rpcSupportsBlockHashParam = false
+
+  # GQL cache settings
+  [server.gqlCache]
+    enabled = true
+
+    # Max in-memory cache size (in bytes) (default 8 MB)
+    # maxCacheSize
+
+    # GQL cache-control max-age settings (in seconds)
+    maxAge = 15
+    timeTravelMaxAge = 86400 # 1 day
+
+[metrics]
+  host = "0.0.0.0"
+  port = 9000
+  [metrics.gql]
+    port = 9001
+
+[database]
+  type = "postgres"
+  host = "sushiswap-v3-watcher-db"
+  port = 5432
+  database = "sushiswap-v3-watcher"
+  username = "vdbm"
+  password = "password"
+  synchronize = true
+  logging = false
+
+[upstream]
+  [upstream.ethServer]
+    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+
+    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
+    rpcClient = true
+
+    # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint
+    isFEVM = true
+
+    # Boolean flag to filter event logs by contracts
+    filterLogsByAddresses = true
+    # Boolean flag to filter event logs by topics
+    filterLogsByTopics = true
+
+  [upstream.cache]
+    name = "requests"
+    enabled = false
+    deleteOnStart = false
+
+[jobQueue]
+  dbConnectionString = "postgres://vdbm:password@sushiswap-v3-watcher-db/sushiswap-v3-watcher-job-queue"
+  maxCompletionLagInSecs = 300
+  jobDelayInMilliSecs = 100
+  eventsInBatch = 50
+  subgraphEventsOrder = true
+  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
+  blockDelayInMilliSecs = 30000
+
+  # Boolean to switch between modes of processing events when starting the server.
+  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
+  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
+  useBlockRanges = true
+
+  # Block range in which logs are fetched during historical blocks processing
+  historicalLogsBlockRange = 2000
+
+  # Max block range of historical processing after which it waits for completion of events processing
+  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
+  historicalMaxFetchAhead = 10000

stack_orchestrator/data/container-build/cerc-act-runner-task-executor/build.sh

@@ -2,4 +2,6 @@
 # Build a local version of the task executor for act-runner
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-docker build -t cerc/act-runner-task-executor:local -f ${CERC_REPO_BASE_DIR}/hosting/act-runner/Dockerfile.task-executor ${build_command_args} ${SCRIPT_DIR}
+
+cd ${CERC_REPO_BASE_DIR}/hosting/act-runner
+docker build -t cerc/act-runner-task-executor:local -f Dockerfile.task-executor ${build_command_args} .

stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/Dockerfile

@@ -10,9 +10,10 @@ COPY genesis /opt/genesis
 COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/
 COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/
 COPY --from=ethgen /apps /apps
-RUN cd /apps/el-gen && pip3 install -r requirements.txt
+RUN cd /apps/el-gen && pip3 install --break-system-packages -r requirements.txt
 # web3==5.24.0 used by el-gen is broken on python 3.11
-RUN pip3 install --upgrade "web3==6.5.0"
+RUN pip3 install --break-system-packages --upgrade "web3==6.5.0"
+RUN pip3 install --break-system-packages --upgrade "typing-extensions"
 
 # Build genesis config
 RUN apk add --no-cache make bash envsubst jq

stack_orchestrator/data/container-build/cerc-laconic-console-host/config.yml

@@ -2,5 +2,5 @@
 
 services:
   wns:
-    server: 'LACONIC_HOSTED_ENDPOINT:9473/api'
-    webui: 'LACONIC_HOSTED_ENDPOINT:9473/console'
+    server: 'LACONIC_HOSTED_ENDPOINT/api'
+    webui: 'LACONIC_HOSTED_ENDPOINT/console'

stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-10.yml

@@ -0,0 +1,14 @@
+record:
+  type: ApplicationDeploymentRecord
+  version: 1.2.3
+  name: name
+  description: description
+  application: application
+  url: url
+  dns: dns
+  request: request
+  meta:
+    foo: bar
+  tags:
+    - a
+    - b 

stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-11.yml

@@ -0,0 +1,12 @@
+record:
+  type: GeneralRecord
+  version: 1.2.3
+  name: name
+  description: description
+  category: category
+  value: value
+  meta:
+    foo: bar
+  tags:
+    - a
+    - b 

stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-12.yml

@@ -0,0 +1,17 @@
+record:
+  type: ApplicationArtifact
+  version: 1.2.3
+  name: name
+  description: description
+  application: appidgoeshere
+  content_type: content_type
+  os: os
+  cpu: cpu
+  uri:
+    - uri://a
+    - uri://b
+  meta:
+    foo: bar
+  tags:
+    - a
+    - b

stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-7.yml

@@ -0,0 +1,11 @@
+record:
+  type: DnsRecord
+  version: 0.0.1
+  name: "foo"
+  resource_type: "A"
+  value: "bar"
+  meta:
+    foo: bar
+  tags:
+    - a
+    - b

stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-8.yml

@@ -0,0 +1,18 @@
+record:
+  type: ApplicationRecord
+  version: 0.0.1
+  name: my-demo-app
+  description: "Description of my app"
+  homepage: http://my.demo.app
+  license: license
+  author: author
+  repository:
+    - "https://my.demo.repo"
+  repository_ref: "v0.1.0"
+  app_version: "0.1.0"
+  app_type: "webapp"
+  meta:
+    foo: bar
+  tags:
+    - a
+    - b

stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-9.yml

@@ -0,0 +1,17 @@
+record:
+  type: ApplicationDeploymentRequest
+  version: 1.2.3
+  application: application
+  dns: dns
+  config:
+    env:
+      ENV_VAR_A: A
+      ENV_VAR_B: B
+  crn:
+    - crn://foo.bar
+    - crn://bar.baz
+  meta:
+    foo: bar
+  tags:
+    - a
+    - b

stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile

@@ -1,6 +1,6 @@
 # Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile
 # [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
-ARG VARIANT=20-bullseye
+ARG VARIANT=20-bullseye-slim
 FROM node:${VARIANT}
 
 ARG USERNAME=node
@@ -24,32 +24,21 @@ RUN \
     && su ${USERNAME} -c "npm config -g set prefix ${NPM_GLOBAL}" \
     # Install eslint
     && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \
+    # Install semver
+    && su ${USERNAME} -c "umask 0002 && npm install -g semver" \
     && npm cache clean --force > /dev/null 2>&1
 
 # [Optional] Uncomment this section to install additional OS packages.
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
     && apt-get -y install --no-install-recommends jq gettext-base
 
-# [Optional] Uncomment if you want to install an additional version of node using nvm
-# ARG EXTRA_NODE_VERSION=10
-# RUN su node -c "source /usr/local/share/nvm/nvm.sh && nvm install ${EXTRA_NODE_VERSION}"
-
-# We do this to get a yq binary from the published container, for the correct architecture we're building here
-# COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq
-
-COPY /scripts /scripts
-
 # [Optional] Uncomment if you want to install more global node modules
 # RUN su node -c "npm install -g <your-package-list-here>"
 
-# RUN mkdir -p /config
-# COPY ./config.yml /config
-
-# Install simple web server for now (use nginx perhaps later)
-# RUN yarn global add http-server
-
 # Expose port for http
 EXPOSE 3000
 
+COPY /scripts /scripts
+
 # Default command sleeps forever so docker doesn't kill it
-CMD ["/scripts/start-serving-app.sh"]
+ENTRYPOINT ["/scripts/start-serving-app.sh"]

stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile.webapp

@@ -1,4 +1,8 @@
 FROM cerc/nextjs-base:local
+
+ARG CERC_NEXT_VERSION=keep
+ARG CERC_BUILD_TOOL
+
 WORKDIR /app
 COPY . .
 RUN rm -rf node_modules build .next*

stack_orchestrator/data/container-build/cerc-nextjs-base/build.sh

@@ -11,3 +11,19 @@ CERC_CONTAINER_BUILD_DOCKERFILE=${CERC_CONTAINER_BUILD_DOCKERFILE:-$SCRIPT_DIR/D
 CERC_CONTAINER_BUILD_TAG=${CERC_CONTAINER_BUILD_TAG:-cerc/nextjs-base:local}
 
 docker build -t $CERC_CONTAINER_BUILD_TAG ${build_command_args} -f $CERC_CONTAINER_BUILD_DOCKERFILE $CERC_CONTAINER_BUILD_WORK_DIR
+
+if [ $? -eq 0 ] && [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/nextjs-base:local" ]; then
+  cat <<EOF
+
+#################################################################
+
+Built host container for $CERC_CONTAINER_BUILD_WORK_DIR with tag:
+
+    $CERC_CONTAINER_BUILD_TAG
+
+To test locally run:
+
+    laconic-so run-webapp --image $CERC_CONTAINER_BUILD_TAG --env-file /path/to/environment.env
+
+EOF
+fi

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh

@@ -8,6 +8,15 @@ WORK_DIR="${1:-./}"
 SRC_DIR="${2:-.next}"
 TRG_DIR="${3:-.next-r}"
 
+CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
+if [ -z "$CERC_BUILD_TOOL" ]; then
+  if [ -f "yarn.lock" ]; then
+    CERC_BUILD_TOOL=npm
+  else
+    CERC_BUILD_TOOL=yarn
+  fi
+fi
+
 cd "${WORK_DIR}" || exit 1
 
 rm -rf "$TRG_DIR"
@@ -29,6 +38,9 @@ for f in $(find "$TRG_DIR" -regex ".*.[tj]sx?$" -type f | grep -v 'node_modules'
     orig_name=$(echo -n "${e}" | sed 's/"//g')
     cur_name=$(echo -n "${orig_name}" | sed 's/CERC_RUNTIME_ENV_//g')
     cur_val=$(echo -n "\$${cur_name}" | envsubst)
+    if [ "$CERC_RETAIN_ENV_QUOTES" != "true" ]; then
+      cur_val=$(sed "s/^[\"']//" <<< "$cur_val" | sed "s/[\"']//")
+    fi
     esc_val=$(sed 's/[&/\]/\\&/g' <<< "$cur_val")
     echo "$f: $cur_name=$cur_val"
     sed -i "s/$orig_name/$esc_val/g" $f

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh

@@ -4,12 +4,28 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
     set -x
 fi
 
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+CERC_MIN_NEXTVER=13.4.2
 
+CERC_NEXT_VERSION="${CERC_NEXT_VERSION:-keep}"
+CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
+if [ -z "$CERC_BUILD_TOOL" ]; then
+  if [ -f "yarn.lock" ]; then
+    CERC_BUILD_TOOL=yarn
+  else
+    CERC_BUILD_TOOL=npm
+  fi
+fi
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 WORK_DIR="${1:-/app}"
 
 cd "${WORK_DIR}" || exit 1
 
+# If this file doesn't exist at all, we'll get errors below.
+if [ ! -f "next.config.js" ]; then
+  touch next.config.js
+fi
+
 if [ ! -f "next.config.dist" ]; then
   cp next.config.js next.config.dist
 fi
@@ -20,6 +36,7 @@ if [ $? -ne 0 ]; then
 fi
 
 js-beautify next.config.dist > next.config.js
+echo "" >> next.config.js
 
 WEBPACK_REQ_LINE=$(grep -n "require([\'\"]webpack[\'\"])" next.config.js | cut -d':' -f1)
 if [ -z "$WEBPACK_REQ_LINE" ]; then
@@ -58,7 +75,7 @@ if [ -n "$WEBPACK_CONF_LINE" ]; then
   cat > next.config.js.3 <<EOF
       $WEBPACK_CONF_VAR.plugins.push(new webpack.DefinePlugin(envMap));
 EOF
-  NEXT_SECTION_LINE=$((WEBPACK_CONF_LINE - 1))
+  NEXT_SECTION_LINE=$((WEBPACK_CONF_LINE))
 elif [ -n "$ENV_LINE" ]; then
   head -$(( ${ENV_LINE} - 1 )) next.config.js > next.config.js.2
   cat > next.config.js.3 <<EOF
@@ -67,7 +84,7 @@ elif [ -n "$ENV_LINE" ]; then
       return config;
     },
 EOF
-  NEXT_SECTION_ADJUSTMENT=2
+  NEXT_SECTION_ADJUSTMENT=1
   NEXT_SECTION_LINE=$ENV_LINE
 else
   echo "WARNING: Cannot find location to insert environment variable map in next.config.js" 1>&2
@@ -88,7 +105,40 @@ fi
 
 cat package.dist | jq '.scripts.cerc_compile = "next experimental-compile"' | jq '.scripts.cerc_generate = "next experimental-generate"' > package.json
 
-npm install || exit 1
-npm run cerc_compile || exit 1
+CUR_NEXT_VERSION="`jq -r '.dependencies.next' package.json`"
+
+if [ "$CERC_NEXT_VERSION" != "keep" ] && [ "$CUR_NEXT_VERSION" != "$CERC_NEXT_VERSION" ]; then
+  echo "Changing 'next' version specifier from '$CUR_NEXT_VERSION' to '$CERC_NEXT_VERSION' (set with '--extra-build-args \"--build-arg CERC_NEXT_VERSION=$CERC_NEXT_VERSION\"')"
+  cat package.json | jq ".dependencies.next = \"$CERC_NEXT_VERSION\"" > package.json.$$
+  mv package.json.$$ package.json
+fi
+
+$CERC_BUILD_TOOL install || exit 1
+
+CUR_NEXT_VERSION=`jq -r '.version' node_modules/next/package.json`
+
+semver -p -r ">=$CERC_MIN_NEXTVER" $CUR_NEXT_VERSION
+if [ $? -ne 0 ]; then
+  cat <<EOF
+
+###############################################################################
+
+WARNING: 'next' $CUR_NEXT_VERSION < minimum version $CERC_MIN_NEXTVER.
+
+Attempting to build with '^$CERC_MIN_NEXTVER'.  If this fails, you should upgrade
+the dependency in your webapp, or specify an explicit 'next' version
+to use for the build with:
+
+     --extra-build-args "--build-arg CERC_NEXT_VERSION=<version>"
+
+###############################################################################
+
+EOF
+  cat package.json | jq ".dependencies.next = \"^$CERC_MIN_NEXTVER\"" > package.json.$$
+  mv package.json.$$ package.json
+  $CERC_BUILD_TOOL install || exit 1
+fi
+
+$CERC_BUILD_TOOL run cerc_compile || exit 1
 
 exit 0

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh

@@ -3,7 +3,25 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
     set -x
 fi
 
+
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+CERC_MAX_GENERATE_TIME=${CERC_MAX_GENERATE_TIME:-60}
+tpid=""
+
+ctrl_c() {
+    kill $tpid $(ps -ef | grep node | grep next | awk '{print $2}') 2>/dev/null
+}
+
+trap ctrl_c INT
+
+CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
+if [ -z "$CERC_BUILD_TOOL" ]; then
+  if [ -f "yarn.lock" ] && [ ! -f "package-lock.json" ]; then
+    CERC_BUILD_TOOL=yarn
+  else
+    CERC_BUILD_TOOL=npm
+  fi
+fi
 
 CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/app}"
 cd "$CERC_WEBAPP_FILES_DIR"
@@ -16,19 +34,28 @@ if [ "$CERC_NEXTJS_SKIP_GENERATE" != "true" ]; then
   jq -e '.scripts.cerc_generate' package.json >/dev/null
   if [ $? -eq 0 ]; then
     npm run cerc_generate > gen.out 2>&1 &
-    tail -n0 -f gen.out | sed '/rendered as static HTML/ q'
+    tail -f gen.out &
+    tpid=$!
+
     count=0
-    while [ $count -lt 10 ]; do
+    generate_done="false"
+    while [ $count -lt $CERC_MAX_GENERATE_TIME ] && [ "$generate_done" == "false" ]; do
       sleep 1
-      ps -ef | grep 'node' | grep 'next' | grep 'generate' >/dev/null
-      if [ $? -ne 0 ]; then 
-        break
-      else
-        count=$((count + 1))
+      count=$((count + 1))
+      grep 'rendered as static' gen.out > /dev/null
+      if [ $? -eq 0 ]; then
+        generate_done="true"
       fi
     done
-    kill $(ps -ef |grep node | grep next | grep generate | awk '{print $2}') 2>/dev/null
+
+    if [ $generate_done != "true" ]; then
+      echo "ERROR: 'npm run cerc_generate' not successful within CERC_MAX_GENERATE_TIME" 1>&2
+      exit 1
+    fi
+
+    kill $tpid $(ps -ef | grep node | grep next | grep generate | awk '{print $2}') 2>/dev/null
+    tpid=""
   fi
 fi
 
-npm start . -p ${CERC_LISTEN_PORT:-3000}
+$CERC_BUILD_TOOL start . -p ${CERC_LISTEN_PORT:-3000}

stack_orchestrator/data/container-build/cerc-osmosis-front-end-urbit/build.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build the osmosis front end image
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+docker build -t cerc/osmosis-front-end-urbit:local -f ${CERC_REPO_BASE_DIR}/osmosis-frontend/docker/Dockerfile.static ${build_command_args} ${CERC_REPO_BASE_DIR}/osmosis-frontend

stack_orchestrator/data/container-build/cerc-osmosis-front-end/build.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build the osmosis front end image
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+docker build -t cerc/osmosis-front-end:local -f ${CERC_REPO_BASE_DIR}/osmosis-frontend/docker/Dockerfile.static ${build_command_args} ${CERC_REPO_BASE_DIR}/osmosis-frontend

stack_orchestrator/data/container-build/cerc-uniswap-interface/Dockerfile

@@ -0,0 +1,10 @@
+FROM node:18.17.1-alpine3.18
+
+RUN apk --update --no-cache add git make alpine-sdk bash
+
+WORKDIR /app
+
+COPY . .
+
+RUN echo "Building uniswap-interface" && \
+    yarn

stack_orchestrator/data/container-build/cerc-uniswap-interface/build.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# Build the uniswap-interface image
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/uniswap-interface:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/uniswap-interface

stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/Dockerfile

@@ -0,0 +1,10 @@
+FROM node:18.17.1-alpine3.18
+
+RUN apk --update --no-cache add git python3 alpine-sdk bash curl jq
+
+WORKDIR /app
+
+COPY . .
+
+RUN echo "Installing dependencies and building merkl-sushiswap-v3-watcher-ts" && \
+  yarn && yarn build

stack_orchestrator/data/container-build/cerc-watcher-merkl-sushiswap-v3/build.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# Build cerc/watcher-merkl-sushiswap-v3
+
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/my-new-stack:local -f ${CERC_REPO_BASE_DIR}/my-new-stack/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/my-new-stack
+
+docker build -t cerc/watcher-merkl-sushiswap-v3:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/merkl-sushiswap-v3-watcher-ts

stack_orchestrator/data/container-build/cerc-watcher-sushiswap-v3/Dockerfile

@@ -0,0 +1,10 @@
+FROM node:18.17.1-alpine3.18
+
+RUN apk --update --no-cache add git python3 alpine-sdk bash curl jq
+
+WORKDIR /app
+
+COPY . .
+
+RUN echo "Installing dependencies and building sushiswap-v3-watcher-ts" && \
+  yarn && yarn build

stack_orchestrator/data/container-build/cerc-watcher-sushiswap-v3/build.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# Build cerc/watcher-sushiswap-v3
+
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/watcher-sushiswap-v3:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/sushiswap-v3-watcher-ts

stack_orchestrator/data/container-image-list.txt

@@ -57,3 +57,6 @@ cerc/nitro-contracts
 cerc/mobymask-snap
 cerc/ponder
 cerc/nitro-rpc-client
+cerc/watcher-merkl-sushiswap-v3
+cerc/watcher-sushiswap-v3
+cerc/uniswap-interface

stack_orchestrator/data/pod-list.txt

@@ -43,3 +43,5 @@ nitro-contracts
 mobymask-snap
 ponder
 ipld-eth-server-payments
+merkl-sushiswap-v3
+sushiswap-v3

stack_orchestrator/data/repository-list.txt

@@ -47,3 +47,6 @@ github.com/cerc-io/go-nitro
 github.com/cerc-io/ts-nitro
 github.com/cerc-io/mobymask-snap
 github.com/cerc-io/ponder
+github.com/cerc-io/merkl-sushiswap-v3-watcher-ts
+github.com/cerc-io/sushiswap-v3-watcher-ts
+github.com/cerc-io/uniswap-interface

stack_orchestrator/data/stacks/act-runner/stack.yml

@@ -3,7 +3,7 @@ name: act-runner
 description: "Local act-runner"
 repos:
   - git.vdb.to/cerc-io/hosting
-  - gitea.com/gitea/act_runner
+  - gitea.com/telackey/act_runner@telackey/entrypoint
 containers:
   - cerc/act-runner
   - cerc/act-runner-task-executor

stack_orchestrator/data/stacks/azimuth/README.md

@@ -9,19 +9,11 @@ Prerequisite: `ipld-eth-server` RPC and GQL endpoints
 Clone required repositories:
 
 ```bash
-laconic-so --stack azimuth setup-repositories
+laconic-so --stack azimuth setup-repositories --pull
 ```
 
 NOTE: If the repository already exists and checked out to a different version, `setup-repositories` command will throw an error.
-For getting around this, the `azimuth-watcher-ts` repository can be removed and then run the command.
-
-Checkout to the required versions and branches in repos
-
-```bash
-# azimuth-watcher-ts
-cd ~/cerc/azimuth-watcher-ts
-git checkout v0.1.0
-```
+For getting around this, the `azimuth-watcher-ts` repository can be removed and then run the command again.
 
 Build the container images:
 
@@ -31,42 +23,101 @@ laconic-so --stack azimuth build-containers
 
 This should create the required docker images in the local image registry.
 
-### Configuration
+## Create a deployment
 
-* Create and update an env file to be used in the next step:
+First, create a spec file for the deployment, which will map the stack's ports and volumes to the host:
+```bash
+laconic-so --stack azimuth deploy init --output azimuth-spec.yml
+```
+
+### Ports
+
+Edit `network` in spec file to map container ports to same ports in host
+
+```yaml
+...
+network:
+  ports:
+    watcher-db:
+     - 0.0.0.0:15432:5432
+    azimuth-watcher-job-runner:
+     - 0.0.0.0:9000:9000
+    azimuth-watcher-server:
+     - 0.0.0.0:3001:3001
+    censures-watcher-job-runner:
+     - 0.0.0.0:9002:9002
+    censures-watcher-server:
+     - 0.0.0.0:3002:3002
+    claims-watcher-job-runner:
+     - 0.0.0.0:9004:9004
+    claims-watcher-server:
+     - 0.0.0.0:3003:3003
+    conditional-star-release-watcher-job-runner:
+     - 0.0.0.0:9006:9006
+    conditional-star-release-watcher-server:
+     - 0.0.0.0:3004:3004
+    delegated-sending-watcher-job-runner:
+     - 0.0.0.0:9008:9008
+    delegated-sending-watcher-server:
+     - 0.0.0.0:3005:3005
+    ecliptic-watcher-job-runner:
+     - 0.0.0.0:9010:9010
+    ecliptic-watcher-server:
+     - 0.0.0.0:3006:3006
+    linear-star-release-watcher-job-runner:
+     - 0.0.0.0:9012:9012
+    linear-star-release-watcher-server:
+     - 0.0.0.0:3007:3007
+    polls-watcher-job-runner:
+     - 0.0.0.0:9014:9014
+    polls-watcher-server:
+     - 0.0.0.0:3008:3008
+    gateway-server:
+     - 0.0.0.0:4000:4000
+...
+```
+
+### Data volumes
+Container data volumes are bind-mounted to specified paths in the host filesystem.
+The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory. The default mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`.
+
+---
+
+Once you've made any needed changes to the spec file, create a deployment from it:
+```bash
+laconic-so --stack azimuth deploy create --spec-file azimuth-spec.yml --deployment-dir azimuth-deployment
+```
+
+## Set env variables
+
+Inside the deployment directory, open the file `config.env` and add variable to update RPC endpoint :
 
   ```bash
-  # External ipld-eth-server endpoints
+  # External RPC endpoints
   CERC_IPLD_ETH_RPC=
-  CERC_IPLD_ETH_GQL=
   ```
 
-* NOTE: If `ipld-eth-server` is running on the host machine, use `host.docker.internal` as the hostname to access host ports
+* NOTE: If RPC endpoint is on the host machine, use `host.docker.internal` as the hostname to access the host port, or use the `ip a` command to find the IP address of the `docker0` interface (this will usually be something like `172.17.0.1` or `172.18.0.1`)
 
-### Deploy the stack
+## Start the stack
 
-* Deploy the containers:
-
-  ```bash
-  laconic-so --stack azimuth deploy-system --env-file <PATH_TO_ENV_FILE> up
-  ```
+Start the deployment:
+```bash
+laconic-so deployment --dir azimuth-deployment start
+```
 
 * List and check the health status of all the containers using `docker ps` and wait for them to be `healthy`
 
 ## Clean up
 
-Stop all the services running in background:
+To stop all azimuth services running in the background, while preserving chain data:
 
 ```bash
-laconic-so --stack azimuth deploy-system down
+laconic-so deployment --dir azimuth-deployment stop
 ```
 
-Clear volumes created by this stack:
+To stop all azimuth services and also delete data:
 
 ```bash
-# List all relevant volumes
-docker volume ls -q --filter "name=.*watcher_db_data"
-
-# Remove all the listed volumes
-docker volume rm $(docker volume ls -q --filter "name=.*watcher_db_data")
+laconic-so deployment --dir azimuth-deployment stop --delete-volumes
 ```

stack_orchestrator/data/stacks/azimuth/stack.yml

@@ -1,7 +1,7 @@
 version: "1.0"
 name: azimuth
 repos:
-  - github.com/cerc-io/azimuth-watcher-ts@v0.1.1
+  - github.com/cerc-io/azimuth-watcher-ts@v0.1.3
 containers:
   - cerc/watcher-azimuth
 pods:

stack_orchestrator/data/stacks/graph-node/README.md

@@ -59,7 +59,7 @@ ports:
 Create deployment:
 
 ```bash
-laconic-so deploy create --spec-file graph-node-spec.yml --deployment-dir graph-node-deployment
+laconic-so --stack graph-node deploy create --spec-file graph-node-spec.yml --deployment-dir graph-node-deployment
 ```
 
 ## Start the stack

stack_orchestrator/data/stacks/merkl-sushiswap-v3/README.md

@@ -0,0 +1,81 @@
+# Merkl SushiSwap v3 Watcher
+
+## Setup
+
+Clone required repositories:
+
+```bash
+laconic-so --stack merkl-sushiswap-v3 setup-repositories --git-ssh --pull
+```
+
+Build the container images:
+
+```bash
+laconic-so --stack merkl-sushiswap-v3 build-containers
+```
+
+## Deploy
+
+### Configuration
+
+Create and update an env file to be used in the next step:
+
+  ```bash
+  # External Filecoin (ETH RPC) endpoint to point the watcher
+  CERC_ETH_RPC_ENDPOINT=
+  ```
+
+### Deploy the stack
+
+```bash
+laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 --env-file <PATH_TO_ENV_FILE> up
+```
+
+* To list down and monitor the running containers:
+
+  ```bash
+  laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 ps
+
+  # With status
+  docker ps -a
+
+  # Check logs for a container
+  docker logs -f <CONTAINER_ID>
+  ```
+
+* Open the GQL playground at http://localhost:3007/graphql
+
+  ```graphql
+  {
+    _meta {
+      block {
+        number
+        timestamp
+      }
+      hasIndexingErrors
+    }
+    
+    factories {
+      id
+      poolCount
+    }
+  }
+  ```
+
+## Clean up
+
+Stop all the services running in background:
+
+```bash
+laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 down
+```
+
+Clear volumes created by this stack:
+
+```bash
+# List all relevant volumes
+docker volume ls -q --filter "name=merkl_sushiswap_v3"
+
+# Remove all the listed volumes
+docker volume rm $(docker volume ls -q --filter "name=merkl_sushiswap_v3")
+```

stack_orchestrator/data/stacks/merkl-sushiswap-v3/stack.yml

@@ -0,0 +1,9 @@
+version: "1.0"
+name: merkl-sushiswap-v3
+description: "SushiSwap v3 watcher stack"
+repos:
+  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.6
+containers:
+  - cerc/watcher-merkl-sushiswap-v3
+pods:
+  - watcher-merkl-sushiswap-v3

stack_orchestrator/data/stacks/monitoring/README.md

@@ -0,0 +1,180 @@
+# monitoring
+
+* Instructions to setup and run a Prometheus server and Grafana dashboard
+* Comes with the following built-in exporters / dashboards:
+  * Chain Head Exporter - for tracking chain heads given external ETH RPC endpoints
+  * Watchers dashboard
+  * [Prometheus Blackbox](https://grafana.com/grafana/dashboards/7587-prometheus-blackbox-exporter/) - for tracking HTTP endpoints
+  * [NodeJS Application Dashboard](https://grafana.com/grafana/dashboards/11159-nodejs-application-dashboard/) - for default NodeJS metrics
+  * [PostgreSQL Database](https://grafana.com/grafana/dashboards/9628-postgresql-database/) - for monitoring Postgres dbs
+  * [Node Exporter Full](https://grafana.com/grafana/dashboards/1860-node-exporter-full/) - for monitoring system metrics
+* See [monitoring-watchers.md](./monitoring-watchers.md) for an example usage of the stack with pre-configured dashboards for watchers
+
+## Setup
+
+Clone required repositories:
+
+```bash
+laconic-so --stack monitoring setup-repositories --git-ssh --pull
+```
+
+Build the container images:
+
+```bash
+laconic-so --stack monitoring build-containers
+```
+
+## Create a deployment
+
+First, create a spec file for the deployment, which will map the stack's ports and volumes to the host:
+
+```bash
+laconic-so --stack monitoring deploy init --output monitoring-spec.yml
+```
+
+### Ports
+
+Edit `network` in spec file to map container ports to same ports in host:
+
+```
+...
+network:
+  ports:
+    prometheus:
+      - '9090:9090'
+    grafana:
+      - '3000:3000'
+...
+```
+
+### Data volumes
+
+Container data volumes are bind-mounted to specified paths in the host filesystem.
+The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory. The default mappings can be customized by editing the "spec" file generated by `laconic-so deploy init`.
+
+---
+
+Once you've made any needed changes to the spec file, create a deployment from it:
+
+```bash
+laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --deployment-dir monitoring-deployment
+```
+
+## Configure
+
+### Prometheus Config
+
+* Add desired scrape configs to prometheus config file (`monitoring-deployment/config/monitoring/prometheus/prometheus.yml`) in the deployment folder; for example:
+
+  ```yml
+  ...
+  - job_name: <JOB_NAME>
+    metrics_path: /metrics/path
+    scheme: http
+    static_configs:
+      - targets: ['<METRICS_ENDPOINT_HOST>:<METRICS_ENDPOINT_PORT>']
+  ```
+
+* Node exporter: update the `node` job to add any node-exporter targets to be monitored:
+
+  ```yml
+  ...
+  - job_name: 'node'
+    ...
+    static_configs:
+      # Add node-exporter targets to be monitored below
+      - targets: [example-host:9100]
+        labels:
+          instance: 'my-host'
+  ```
+
+* Blackbox (in-stack exporter): update the `blackbox` job to add any endpoints to be monitored on the Blackbox dashboard:
+
+  ```yml
+  ...
+  - job_name: 'blackbox'
+    ...
+    static_configs:
+      # Add URLs to be monitored below
+      - targets:
+        - <HTTP_ENDPOINT_1>
+        - <HTTP_ENDPOINT_2>
+  ```
+
+* Postgres (in-stack exporter):
+  * Update the `postgres` job to add Postgres db targets to be monitored:
+
+    ```yml
+    ...
+    - job_name: 'postgres'
+      ...
+      static_configs:
+        # Add DB targets below
+        - targets: [example-server:5432]
+          labels:
+            instance: 'example-db'
+    ```
+  * Add database credentials to be used in `auth_modules` in the postgres-exporter config file (`monitoring-deployment/config/monitoring/postgres-exporter.yml`)
+
+Note: Use `host.docker.internal` as host to access ports on the host machine
+
+### Grafana Config
+
+Place the dashboard json files in grafana dashboards config directory (`monitoring-deployment/config/monitoring/grafana/dashboards`) in the deployment folder
+
+### Env
+
+Set the following env variables in the deployment env config file (`monitoring-deployment/config.env`):
+
+  ```bash
+  # For chain-head exporter
+
+  # External ETH RPC endpoint (ethereum)
+  # (Optional, default: https://mainnet.infura.io/v3)
+  CERC_ETH_RPC_ENDPOINT=
+
+  # Infura key to be used
+  # (Optional, used with ETH_RPC_ENDPOINT if provided)
+  CERC_INFURA_KEY=
+
+  # External ETH RPC endpoint (filecoin)
+  # (Optional, default: https://api.node.glif.io/rpc/v1)
+  CERC_FIL_RPC_ENDPOINT=
+
+  # Grafana server host URL (used in various links in alerts, etc.)
+  # (Optional, default: http://localhost:3000)
+  GF_SERVER_ROOT_URL=
+  ```
+
+## Start the stack
+
+Start the deployment:
+
+```bash
+laconic-so deployment --dir monitoring-deployment start
+```
+
+* List and check the health status of all the containers using `docker ps` and wait for them to be `healthy`
+
+* Grafana should now be visible at http://localhost:3000 with configured dashboards
+
+## Clean up
+
+To stop monitoring services running in the background, while preserving data:
+
+```bash
+# Only stop the docker containers
+laconic-so deployment --dir monitoring-deployment stop
+
+# Run 'start' to restart the deployment
+```
+
+To stop monitoring services and also delete data:
+
+```bash
+# Stop the docker containers
+laconic-so deployment --dir monitoring-deployment stop --delete-volumes
+
+# Remove deployment directory (deployment will have to be recreated for a re-run)
+rm -rf monitoring-deployment
+```