Remove unnecessary todo

This adds a stack for the backend from snowball/snowballtools-base.

Reviewed-on: cerc-io/stack-orchestrator#751
Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com>
Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>

.gitea/workflows/fixturenet-eth-plugeth-test.yml

@@ -6,6 +6,8 @@ on:
     paths:
       - '!**'
       - '.gitea/workflows/triggers/fixturenet-eth-plugeth-test'
+  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
+    - cron: '2 14 * * *'
 
 # Needed until we can incorporate docker startup into the executor container
 env:

.gitea/workflows/fixturenet-laconicd-test.yml

@@ -6,11 +6,8 @@ on:
     paths:
       - '!**'
       - '.gitea/workflows/triggers/fixturenet-laconicd-test'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
+  schedule:
+    - cron: '1 13 * * *'
 
 jobs:
   test:
@@ -47,9 +44,5 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
       - name: "Run fixturenet-laconicd tests"
         run: ./tests/fixturenet-laconicd/run-test.sh

.gitea/workflows/lint.yml

@@ -0,0 +1,21 @@
+name: Lint Checks
+
+on:
+  pull_request:
+    branches: '*'
+  push:
+    branches: '*'
+
+jobs:
+  test:
+    name: "Run linter"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      - name: "Install Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name : "Run flake8"
+        uses: py-actions/flake8@v2

.gitea/workflows/test-container-registry.yml

@@ -0,0 +1,54 @@
+name: Container Registry Test
+
+on:
+  push:
+    branches: '*'
+    paths:
+      - '!**'
+      - '.gitea/workflows/triggers/test-container-registry'
+      - '.gitea/workflows/test-container-registry.yml'
+      - 'tests/container-registry/run-test.sh'
+  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
+    - cron: '6 19 * * *'
+
+jobs:
+  test:
+    name: "Run contaier registry hosting test on kind/k8s"
+    runs-on: ubuntu-22.04
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      # At present the stock setup-python action fails on Linux/aarch64
+      # Conditional steps below workaroud this by using deadsnakes for that case only
+      - name: "Install Python for ARM on Linux"
+        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
+        uses: deadsnakes/action@v3.0.1
+        with:
+          python-version: '3.8'
+      - name: "Install Python cases other than ARM on Linux"
+        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: "Check cgroups version"
+        run: mount | grep cgroup
+      - name: "Install kind"
+        run: ./tests/scripts/install-kind.sh
+      - name: "Install Kubectl"
+        run: ./tests/scripts/install-kubectl.sh
+      - name: "Install ed" # Only needed until we remove the need to edit the spec file
+        run: apt update && apt install -y ed
+      - name: "Run container registry deployment test"
+        run: |
+          source /opt/bash-utils/cgroup-helper.sh
+          join_cgroup
+          ./tests/container-registry/run-test.sh
+

.gitea/workflows/test-database.yml

@@ -0,0 +1,52 @@
+name: Database Test
+
+on:
+  push:
+    branches: '*'
+    paths:
+      - '!**'
+      - '.gitea/workflows/triggers/test-database'
+      - '.gitea/workflows/test-database.yml'
+      - 'tests/database/run-test.sh'
+  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
+    - cron: '5 18 * * *'
+
+jobs:
+  test:
+    name: "Run database hosting test on kind/k8s"
+    runs-on: ubuntu-22.04
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      # At present the stock setup-python action fails on Linux/aarch64
+      # Conditional steps below workaroud this by using deadsnakes for that case only
+      - name: "Install Python for ARM on Linux"
+        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
+        uses: deadsnakes/action@v3.0.1
+        with:
+          python-version: '3.8'
+      - name: "Install Python cases other than ARM on Linux"
+        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: "Check cgroups version"
+        run: mount | grep cgroup
+      - name: "Install kind"
+        run: ./tests/scripts/install-kind.sh
+      - name: "Install Kubectl"
+        run: ./tests/scripts/install-kubectl.sh
+      - name: "Run database deployment test"
+        run: |
+          source /opt/bash-utils/cgroup-helper.sh
+          join_cgroup
+          ./tests/database/run-test.sh
+

.gitea/workflows/test-k8s-deploy.yml

@@ -4,20 +4,19 @@ on:
   pull_request:
     branches: '*'
   push:
-    branches:
-      - main
-      - ci-test
-    paths-ignore:
-      - '.gitea/workflows/triggers/*'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
+    branches: '*'
+    paths:
+      - '!**'
+      - '.gitea/workflows/triggers/test-k8s-deploy'
+      - '.gitea/workflows/test-k8s-deploy.yml'
+      - 'tests/k8s-deploy/run-deploy-test.sh'
+  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
+    - cron: '3 15 * * *'
 
 jobs:
   test:
-    name: "Run deploy test suite"
-    runs-on: ubuntu-latest
+    name: "Run deploy test suite on kind/k8s"
+    runs-on: ubuntu-22.04
     steps:
       - name: "Clone project repository"
         uses: actions/checkout@v3
@@ -41,15 +40,15 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
+      - name: "Check cgroups version"
+        run: mount | grep cgroup
+      - name: "Install kind"
+        run: ./tests/scripts/install-kind.sh
+      - name: "Install Kubectl"
+        run: ./tests/scripts/install-kubectl.sh
+      - name: "Run k8s deployment test"
         run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
-      - name: "Install Go"
-        uses: actions/setup-go@v4
-        with:
-          go-version: '1.21'
-      - name: "Install Kind"
-        run: go install sigs.k8s.io/kind@v0.20.0
-      - name: "Debug Kind"
-        run: kind create cluster --retain && docker logs kind-control-plane
+          source /opt/bash-utils/cgroup-helper.sh
+          join_cgroup
+          ./tests/k8s-deploy/run-deploy-test.sh
+

.gitea/workflows/test-webapp.yml

@@ -41,6 +41,8 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
+      - name: "Install wget" # 20240109 - Only needed until the executors are updated.
+        run: apt update && apt install -y wget
       - name: Start dockerd # Also needed until we can incorporate into the executor
         run: |
           dockerd -H $DOCKER_HOST --userland-proxy=false &

.gitea/workflows/triggers/fixturenet-eth-plugeth-test

@@ -1,2 +1,3 @@
 Change this file to trigger running the fixturenet-eth-plugeth-test CI job
 trigger
+trigger

.gitea/workflows/triggers/fixturenet-laconicd-test

@@ -1,2 +1,3 @@
 Change this file to trigger running the fixturenet-laconicd-test CI job
-
+Trigger
+Trigger

.gitea/workflows/triggers/test-container-registry

@@ -0,0 +1 @@
+Change this file to trigger running the test-container-registry CI job

.gitea/workflows/triggers/test-database

@@ -0,0 +1,2 @@
+Change this file to trigger running the test-database CI job
+Trigger test run

.gitea/workflows/triggers/test-k8s-deploy

@@ -0,0 +1,2 @@
+Change this file to trigger running the test-k8s-deploy CI job
+Trigger test on PR branch

README.md

@@ -29,10 +29,10 @@ chmod +x ~/.docker/cli-plugins/docker-compose
 Next decide on a directory where you would like to put the stack-orchestrator program. Typically this would be 
 a "user" binary directory such as `~/bin` or perhaps `/usr/local/laconic` or possibly just the current working directory.
 
-Now, having selected that directory, download the latest release from [this page](https://github.com/cerc-io/stack-orchestrator/tags) into it (we're using `~/bin` below for concreteness but edit to suit if you selected a different directory). Also be sure that the destination directory exists and is writable:
+Now, having selected that directory, download the latest release from [this page](https://git.vdb.to/cerc-io/stack-orchestrator/tags) into it (we're using `~/bin` below for concreteness but edit to suit if you selected a different directory). Also be sure that the destination directory exists and is writable:
 
 ```bash
-curl -L -o ~/bin/laconic-so https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
+curl -L -o ~/bin/laconic-so https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so
 ```
 
 Give it execute permissions:
@@ -52,7 +52,7 @@ Version: 1.1.0-7a607c2-202304260513
 Save the distribution url to `~/.laconic-so/config.yml`:
 ```bash
 mkdir ~/.laconic-so
-echo "distribution-url: https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so" >  ~/.laconic-so/config.yml
+echo "distribution-url: https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so" >  ~/.laconic-so/config.yml
 ```
 
 ### Update

docs/CONTRIBUTING.md

@@ -26,7 +26,7 @@ In addition to the pre-requisites listed in the [README](/README.md), the follow
 
 1. Clone this repository:
    ```
-   $ git clone https://github.com/cerc-io/stack-orchestrator.git
+   $ git clone https://git.vdb.to/cerc-io/stack-orchestrator.git
    ```
 
 2. Enter the project directory:

docs/adding-a-new-stack.md

@@ -1,10 +1,10 @@
 # Adding a new stack
 
-See [this PR](https://github.com/cerc-io/stack-orchestrator/pull/434) for an example of how to currently add a minimal stack to stack orchestrator. The [reth stack](https://github.com/cerc-io/stack-orchestrator/pull/435) is another good example.
+See [this PR](https://git.vdb.to/cerc-io/stack-orchestrator/pull/434) for an example of how to currently add a minimal stack to stack orchestrator. The [reth stack](https://git.vdb.to/cerc-io/stack-orchestrator/pull/435) is another good example.
 
 For external developers, we recommend forking this repo and adding your stack directly to your fork. This initially requires running in "developer mode" as described [here](/docs/CONTRIBUTING.md). Check out the [Namada stack](https://github.com/vknowable/stack-orchestrator/blob/main/app/data/stacks/public-namada/digitalocean_quickstart.md) from Knowable to see how that is done.
 
-Core to the feature completeness of stack orchestrator is to [decouple the tool functionality from payload](https://github.com/cerc-io/stack-orchestrator/issues/315) which will no longer require forking to add a stack.
+Core to the feature completeness of stack orchestrator is to [decouple the tool functionality from payload](https://git.vdb.to/cerc-io/stack-orchestrator/issues/315) which will no longer require forking to add a stack.
 
 ## Example
 

docs/spec.md

@@ -1,6 +1,6 @@
 # Specification
 
-Note: this page is out of date (but still useful) - it will no longer be useful once stacks are [decoupled from the tool functionality](https://github.com/cerc-io/stack-orchestrator/issues/315).
+Note: this page is out of date (but still useful) - it will no longer be useful once stacks are [decoupled from the tool functionality](https://git.vdb.to/cerc-io/stack-orchestrator/issues/315).
 
 ## Implementation
 

requirements.txt

@@ -10,3 +10,4 @@ pydantic==1.10.9
 tomli==2.0.1
 validators==0.22.0
 kubernetes>=28.1.0
+humanfriendly>=10.0

scripts/cloud-init-dev-mode-install.yaml

@@ -41,4 +41,4 @@ runcmd:
   - apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
   - systemctl enable docker
   - systemctl start docker
-  - git clone https://github.com/cerc-io/stack-orchestrator.git /home/ubuntu/stack-orchestrator
+  - git clone https://git.vdb.to/cerc-io/stack-orchestrator.git /home/ubuntu/stack-orchestrator

scripts/cloud-init-user-mode-install.yaml

@@ -31,5 +31,5 @@ runcmd:
   - apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
   - systemctl enable docker
   - systemctl start docker
-  - curl -L -o /usr/local/bin/laconic-so https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
+  - curl -L -o /usr/local/bin/laconic-so https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so
   - chmod +x /usr/local/bin/laconic-so

scripts/quick-deploy-test.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Beginnings of a script to quickly spin up and test a deployment
+if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi
+if [[ -n "$1" ]]; then
+	stack_name=$1
+else
+	stack_name="test"
+fi
+spec_file_name="${stack_name}-spec.yml"
+deployment_dir_name="${stack_name}-deployment"
+rm -f ${spec_file_name}
+rm -rf ${deployment_dir_name}
+laconic-so --stack ${stack_name} deploy --deploy-to compose init --output ${spec_file_name}
+laconic-so --stack ${stack_name} deploy --deploy-to compose create --deployment-dir ${deployment_dir_name} --spec-file ${spec_file_name}
+#laconic-so deployment --dir ${deployment_dir_name} start
+#laconic-so deployment --dir ${deployment_dir_name} ps
+#laconic-so deployment --dir ${deployment_dir_name} stop

scripts/quick-install-linux.sh

@@ -137,7 +137,7 @@ fi
 echo "**************************************************************************************"
 echo "Installing laconic-so"
 # install latest `laconic-so`
-distribution_url=https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
+distribution_url=https://git.vdb.to/cerc-io/stack-orchestrator/releases/download/latest/laconic-so
 install_filename=${install_dir}/laconic-so
 mkdir -p  ${install_dir}
 curl -L -o ${install_filename} ${distribution_url}

setup.py

@@ -13,7 +13,7 @@ setup(
     description='Orchestrates deployment of the Laconic stack',
     long_description=long_description,
     long_description_content_type="text/markdown",
-    url='https://github.com/cerc-io/stack-orchestrator',
+    url='https://git.vdb.to/cerc-io/stack-orchestrator',
     py_modules=['stack_orchestrator'],
     packages=find_packages(),
     install_requires=[requirements],

stack_orchestrator/build/build_containers.py

@@ -27,12 +27,13 @@ import subprocess
 import click
 import importlib.resources
 from pathlib import Path
-from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external
+from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external, warn_exit
 from stack_orchestrator.base import get_npm_registry_url
 
 # TODO: find a place for this
 #    epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
 
+
 def make_container_build_env(dev_root_path: str,
                              container_build_dir: str,
                              debug: bool,
@@ -104,6 +105,9 @@ def process_container(stack: str,
         build_command = os.path.join(container_build_dir,
                                      "default-build.sh") + f" {default_container_tag} {repo_dir_or_build_dir}"
     if not dry_run:
+        # No PATH at all causes failures with podman.
+        if "PATH" not in container_build_env:
+            container_build_env["PATH"] = os.environ["PATH"]
         if verbose:
             print(f"Executing: {build_command} with environment: {container_build_env}")
         build_result = subprocess.run(build_command, shell=True, env=container_build_env)
@@ -119,6 +123,7 @@ def process_container(stack: str,
     else:
         print("Skipped")
 
+
 @click.command()
 @click.option('--include', help="only build these containers")
 @click.option('--exclude', help="don\'t build these containers")
@@ -159,6 +164,8 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
     containers_in_scope = []
     if stack:
         stack_config = get_parsed_stack_config(stack)
+        if "containers" not in stack_config or stack_config["containers"] is None:
+            warn_exit(f"stack {stack} does not define any containers")
         containers_in_scope = stack_config['containers']
     else:
         containers_in_scope = all_containers

stack_orchestrator/build/build_webapp.py

@@ -25,10 +25,11 @@ from decouple import config
 import click
 from pathlib import Path
 from stack_orchestrator.build import build_containers
+from stack_orchestrator.deploy.webapp.util import determine_base_container
 
 
 @click.command()
-@click.option('--base-container', default="cerc/nextjs-base")
+@click.option('--base-container')
 @click.option('--source-repo', help="directory containing the webapp to build", required=True)
 @click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild")
 @click.option("--extra-build-args", help="Supply extra arguments to build")
@@ -57,6 +58,9 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
     if not quiet:
         print(f'Dev Root is: {dev_root_path}')
 
+    if not base_container:
+        base_container = determine_base_container(source_repo)
+
     # First build the base container.
     container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug,
                                                                     force_rebuild, extra_build_args)
@@ -64,13 +68,12 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
     build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet,
                                        verbose, dry_run, continue_on_error)
 
-
     # Now build the target webapp.  We use the same build script, but with a different Dockerfile and work dir.
     container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true"
     container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"] = os.path.abspath(source_repo)
     container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = os.path.join(container_build_dir,
-                                                                    base_container.replace("/", "-"),
-                                                                    "Dockerfile.webapp")
+                                                                          base_container.replace("/", "-"),
+                                                                          "Dockerfile.webapp")
     if not tag:
         webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1]
         container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local"

stack_orchestrator/constants.py

@@ -27,6 +27,12 @@ kube_config_key = "kube-config"
 deploy_to_key = "deploy-to"
 network_key = "network"
 http_proxy_key = "http-proxy"
-image_resigtry_key = "image-registry"
+image_registry_key = "image-registry"
+configmaps_key = "configmaps"
+resources_key = "resources"
+volumes_key = "volumes"
+security_key = "security"
+annotations_key = "annotations"
+labels_key = "labels"
 kind_config_filename = "kind-config.yml"
 kube_config_filename = "kubeconfig.yml"

stack_orchestrator/data/compose/docker-compose-container-registry.yml

@@ -0,0 +1,13 @@
+services:
+  registry:
+    image: registry:2.8
+    restart: always
+    environment:
+      REGISTRY_LOG_LEVEL: ${REGISTRY_LOG_LEVEL}
+    volumes:
+      - registry-data:/var/lib/registry
+    ports:
+      - "5000"
+
+volumes:
+  registry-data:

stack_orchestrator/data/compose/docker-compose-fixturenet-urbit.yml

@@ -0,0 +1,36 @@
+version: '3.7'
+
+services:
+  # Runs an Urbit fake ship and attempts an app installation using given data
+  # Uploads the app glob to given IPFS endpoint
+  # From urbit_app_builds volume:
+  # - takes app build from ${CERC_URBIT_APP}/build (waits for it to appear)
+  # - takes additional mark files from ${CERC_URBIT_APP}/mar
+  # - takes the docket file from ${CERC_URBIT_APP}/desk.docket-0
+  urbit-fake-ship:
+    restart: unless-stopped
+    image: tloncorp/vere
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_URBIT_APP: ${CERC_URBIT_APP}
+      CERC_ENABLE_APP_INSTALL: ${CERC_ENABLE_APP_INSTALL:-true}
+      CERC_IPFS_GLOB_HOST_ENDPOINT: ${CERC_IPFS_GLOB_HOST_ENDPOINT:-http://ipfs:5001}
+      CERC_IPFS_SERVER_ENDPOINT: ${CERC_IPFS_SERVER_ENDPOINT:-http://ipfs:8080}
+    entrypoint: ["bash", "-c", "./run-urbit-ship.sh && ./deploy-app.sh && tail -f /dev/null"]
+    volumes:
+      - urbit_data:/urbit
+      - urbit_app_builds:/app-builds
+      - ../config/urbit/run-urbit-ship.sh:/urbit/run-urbit-ship.sh
+      - ../config/urbit/deploy-app.sh:/urbit/deploy-app.sh
+    ports:
+      - "80"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "80"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+
+volumes:
+  urbit_data:
+  urbit_app_builds:

stack_orchestrator/data/compose/docker-compose-grafana.yml

@@ -0,0 +1,23 @@
+version: "3.7"
+
+services:
+  grafana:
+    image: grafana/grafana:10.2.2
+    restart: always
+    environment:
+      GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL}
+    volumes:
+      - ../config/monitoring/grafana/provisioning:/etc/grafana/provisioning
+      - ../config/monitoring/grafana/dashboards:/etc/grafana/dashboards
+      - grafana_storage:/var/lib/grafana
+    ports:
+      - "3000"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "3000"]
+      interval: 30s
+      timeout: 5s
+      retries: 10
+      start_period: 3s
+
+volumes:
+  grafana_storage:

stack_orchestrator/data/compose/docker-compose-kubo.yml

@@ -1,13 +1,24 @@
 version: "3.2"
+
 # See: https://docs.ipfs.tech/install/run-ipfs-inside-docker/#set-up
 services:
   ipfs:
-    image: ipfs/kubo:master-2023-02-20-714a968
+    image: ipfs/kubo:v0.24.0
     restart: always
     volumes:
-      - ./ipfs/import:/import
-      - ./ipfs/data:/data/ipfs
+      - ipfs-import:/import
+      - ipfs-data:/data/ipfs
     ports:
-      - "0.0.0.0:8080:8080"
-      - "0.0.0.0:4001:4001"
+      - "4001"
+      - "8080"
       - "0.0.0.0:5001:5001"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5001"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+
+volumes:
+  ipfs-import:
+  ipfs-data:

stack_orchestrator/data/compose/docker-compose-mars-v2.yml

@@ -0,0 +1,12 @@
+version: "3.2"
+
+services:
+  mars:
+    image: cerc/mars-v2:local
+    restart: always
+    ports:
+      - "3000:3000"
+    environment:
+      - URL_OSMOSIS_REST=https://lcd-osmosis.blockapsis.com
+      - URL_OSMOSIS_RPC=https://rpc-osmosis.blockapsis.com
+      - WALLET_CONNECT_ID=0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x

stack_orchestrator/data/compose/docker-compose-mars.yml

@@ -0,0 +1,20 @@
+version: "3.2"
+
+services:
+  mars:
+    image: cerc/mars:local
+    restart: always
+    ports:
+      - "3000:3000"
+    environment:
+      - URL_OSMOSIS_GQL=https://osmosis-node.marsprotocol.io/GGSFGSFGFG34/osmosis-hive-front/graphql
+      - URL_OSMOSIS_REST=https://lcd-osmosis.blockapsis.com
+      - URL_OSMOSIS_RPC=https://rpc-osmosis.blockapsis.com
+      - URL_NEUTRON_GQL=https://neutron.rpc.p2p.world/qgrnU6PsQZA8F9S5Fb8Fn3tV3kXmMBl2M9bcc9jWLjQy8p/hive/graphql
+      - URL_NEUTRON_REST=https://rest-kralum.neutron-1.neutron.org
+      - URL_NEUTRON_RPC=https://rpc-kralum.neutron-1.neutron.org
+      - URL_NEUTRON_TEST_GQL=https://testnet-neutron-gql.marsprotocol.io/graphql
+      - URL_NEUTRON_TEST_REST=https://rest-palvus.pion-1.ntrn.tech
+      - URL_NEUTRON_TEST_RPC=https://rpc-palvus.pion-1.ntrn.tech
+      - WALLET_CONNECT_ID=0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x0x
+

stack_orchestrator/data/compose/docker-compose-mobymask-app-v3-demo.yml

@@ -0,0 +1,32 @@
+version: '3.2'
+
+services:
+  # Builds and serves the MobyMask v3 react-app
+  mobymask-v3-app:
+    restart: unless-stopped
+    image: cerc/mobymask-ui:local
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_APP_WATCHER_URL: ${CERC_APP_WATCHER_URL}
+      CERC_SNAP_URL: ${CERC_SNAP_URL}
+    working_dir: /app
+    command: ["bash", "/scripts/mobymask-app-start.sh"]
+    volumes:
+      - ../config/watcher-mobymask-v3-demo/mobymask-app-start.sh:/scripts/mobymask-app-start.sh
+      - ../config/watcher-mobymask-v3-demo/local/keys:/server/keys
+      - mobymask_deployment:/server
+      - nitro_deployment:/nitro
+    ports:
+      - "127.0.0.1:3004:80"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "80"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+volumes:
+  mobymask_deployment:
+  nitro_deployment:

stack_orchestrator/data/compose/docker-compose-node-exporter.yml

@@ -0,0 +1,15 @@
+version: '3.8'
+
+services:
+  node-exporter:
+    image: prom/node-exporter:latest
+    restart: unless-stopped
+    command:
+      - '--web.listen-address=:9100'
+      - '--path.rootfs=/host'
+      - '--collector.systemd'
+      - '--collector.processes'
+    network_mode: host
+    pid: host
+    volumes:
+      - '/:/host:ro,rslave'

stack_orchestrator/data/compose/docker-compose-osmosis-front-end-urbit.yml

@@ -0,0 +1,22 @@
+version: "3.2"
+
+services:
+  osmosis-front-end:
+    image: cerc/osmosis-front-end-urbit:local
+    restart: on-failure
+    environment:
+      - NEXT_PUBLIC_WEB_API_BASE_URL=${CERC_WEB_API_BASE_URL}
+      - ASSET_LIST_COMMIT_HASH=a326bcefc51372b4912be5a2a2fa84a5d142a438
+      - NEXT_PUBLIC_BASEPATH=/apps/osmosis
+      - NEXT_PUBLIC_URBIT_DEPLOYMENT=true
+    working_dir: /app/packages/web
+    command: ["./build-app-for-urbit.sh"]
+    volumes:
+      - ../config/osmosis/build-app-for-urbit.sh:/app/packages/web/build-app-for-urbit.sh
+      - ../config/osmosis/.env.production:/app/packages/web/.env.production
+      - urbit_app_builds:/app-builds
+      - ../config/osmosis/urbit-files/mar:/app/packages/web/mar
+      - ../config/osmosis/urbit-files/desk.docket-0:/app/packages/web/desk.docket-0
+
+volumes:
+  urbit_app_builds:

stack_orchestrator/data/compose/docker-compose-osmosis-front-end.yml

@@ -0,0 +1,36 @@
+version: "3.2"
+
+services:
+  osmosis-front-end:
+    image: cerc/osmosis-front-end:local
+    restart: on-failure
+    environment:
+      - NEXT_PUBLIC_WEB_API_BASE_URL=${CERC_WEB_API_BASE_URL}
+      - ASSET_LIST_COMMIT_HASH=a326bcefc51372b4912be5a2a2fa84a5d142a438
+    working_dir: /app/packages/web
+    command: ["./build-app.sh"]
+    volumes:
+      - ../config/osmosis/build-app.sh:/app/packages/web/build-app.sh
+      - ../config/osmosis/.env.production:/app/packages/web/.env.production
+      - app_builds:/app-builds
+
+  nginx:
+    image: nginx:1.23-alpine
+    restart: always
+    depends_on:
+      osmosis-front-end:
+        condition: service_completed_successfully
+    volumes:
+      - ../config/osmosis/nginx:/etc/nginx/conf.d
+      - app_builds:/usr/share/nginx
+    ports:
+      - "80"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "80"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+
+volumes:
+  app_builds:

stack_orchestrator/data/compose/docker-compose-ping-pub.yml

@@ -0,0 +1,8 @@
+version: "3.2"
+
+services:
+  ping-pub:
+    image: cerc/ping-pub:local
+    restart: always
+    ports:
+      - "5173:5173"

stack_orchestrator/data/compose/docker-compose-prom-server.yml

@@ -0,0 +1,57 @@
+version: "3.7"
+
+services:
+  prometheus:
+    image: prom/prometheus:v2.49.1
+    restart: always
+    volumes:
+      - ../config/monitoring/prometheus:/etc/prometheus
+      - prometheus_data:/prometheus
+    ports:
+      - "9090"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "9090"]
+      interval: 30s
+      timeout: 5s
+      retries: 10
+      start_period: 3s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  blackbox:
+    image: prom/blackbox-exporter:latest
+    restart: always
+    volumes:
+      - ../config/monitoring/blackbox.yml:/etc/blackbox_exporter/config.yml
+    ports:
+      - '9115'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  chain-head-exporter:
+    image: cerc/watcher-ts:local
+    restart: always
+    working_dir: /app/packages/cli
+    environment:
+      ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      FIL_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT}
+      ETH_RPC_API_KEY: ${CERC_INFURA_KEY}
+      PORT: ${CERC_METRICS_PORT}
+    command: ["sh", "-c", "yarn export-metrics:chain-heads"]
+    ports:
+      - '5000'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  postgres-exporter:
+    image: quay.io/prometheuscommunity/postgres-exporter
+    restart: always
+    volumes:
+      - ../config/monitoring/postgres-exporter.yml:/postgres_exporter.yml
+    ports:
+      - '9187'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+volumes:
+  prometheus_data:

stack_orchestrator/data/compose/docker-compose-proxy-server.yml

@@ -6,7 +6,7 @@ services:
     restart: on-failure
     working_dir: /app/packages/cli
     environment:
-      ENABLE_PROXY: ${ENABLE_PROXY:-true}
+      ENABLE_PROXY: ${CERC_ENABLE_PROXY:-true}
       PROXY_UPSTREAM: ${CERC_PROXY_UPSTREAM}
       PROXY_ORIGIN_HEADER: ${CERC_PROXY_ORIGIN_HEADER}
     command: ["sh", "-c", "./run.sh"]

stack_orchestrator/data/compose/docker-compose-snowballtools-base-backend.yml

@@ -0,0 +1,13 @@
+services:
+  snowballtools-base-backend:
+    image: cerc/snowballtools-base-backend:local
+    restart: always
+    volumes:
+      - data:/data
+      - config:/config:ro
+    ports:
+      - 8000
+
+volumes:
+  data:
+  config:

stack_orchestrator/data/compose/docker-compose-test-database.yml

@@ -0,0 +1,20 @@
+services:
+
+  database:
+    image: cerc/test-database-container:local
+    restart: always
+    volumes:
+      - db-data:/var/lib/postgresql/data
+    environment:
+      POSTGRES_USER: "test-user"
+      POSTGRES_DB: "test-db"
+      POSTGRES_PASSWORD: "password"
+      POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C"
+    ports:
+      - "5432"
+    
+  test-client:
+    image: cerc/test-database-client:local
+
+volumes:
+  db-data:

stack_orchestrator/data/compose/docker-compose-test.yml

@@ -5,10 +5,15 @@ services:
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
       CERC_TEST_PARAM_1: ${CERC_TEST_PARAM_1:-FAILED}
+      CERC_TEST_PARAM_2: "CERC_TEST_PARAM_2_VALUE"
     volumes:
-      - test-data:/data
+      - test-data-bind:/data
+      - test-data-auto:/data2
+      - test-config:/config:ro
     ports:
       - "80"
 
 volumes:
-  test-data:
+  test-data-bind:
+  test-data-auto:
+  test-config:

stack_orchestrator/data/compose/docker-compose-uniswap-interface.yml

@@ -9,9 +9,10 @@ services:
       - REACT_APP_AWS_API_ENDPOINT=${CERC_UNISWAP_GQL}
     command: ["./build-app.sh"]
     volumes:
-      - app_builds:/app-builds
       - ../config/uniswap-interface/build-app.sh:/app/build-app.sh
+      - urbit_app_builds:/app-builds
+      - ../config/uniswap-interface/urbit-files/mar:/app/mar
+      - ../config/uniswap-interface/urbit-files/desk.docket-0:/app/desk.docket-0
 
 volumes:
-  app_builds:
-  app_globs:
+  urbit_app_builds:

stack_orchestrator/data/compose/docker-compose-uniswap-urbit.yml

@@ -1,46 +0,0 @@
-version: '3.7'
-
-services:
-  urbit-fake-ship:
-    restart: unless-stopped
-    image: tloncorp/vere
-    environment:
-      CERC_IPFS_GLOB_HOST_ENDPOINT: ${CERC_IPFS_GLOB_HOST_ENDPOINT:-http://ipfs-glob-host:5001}
-      CERC_IPFS_SERVER_ENDPOINT: ${CERC_IPFS_SERVER_ENDPOINT:-http://ipfs-glob-host:8080}
-    entrypoint: ["bash", "-c", "./run-urbit-ship.sh && ./deploy-uniswap-app.sh && tail -f /dev/null"]
-    volumes:
-      - urbit_data:/urbit
-      - app_builds:/app-builds
-      - app_globs:/app-globs
-      - ../config/urbit/run-urbit-ship.sh:/urbit/run-urbit-ship.sh
-      - ../config/uniswap-interface/deploy-uniswap-app.sh:/urbit/deploy-uniswap-app.sh
-    ports:
-      - "80"
-    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "80"]
-      interval: 20s
-      timeout: 5s
-      retries: 15
-      start_period: 10s
-
-  ipfs-glob-host:
-    image: ipfs/kubo:master-2023-02-20-714a968
-    volumes:
-      - ipfs-import:/import
-      - ipfs-data:/data/ipfs
-    ports:
-      - "8080"
-      - "5001"
-    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "5001"]
-      interval: 20s
-      timeout: 5s
-      retries: 15
-      start_period: 10s
-
-volumes:
-  urbit_data:
-  app_builds:
-  app_globs:
-  ipfs-import:
-  ipfs-data:

stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml

@@ -35,7 +35,7 @@ services:
       - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
       - ../config/watcher-merkl-sushiswap-v3/start-job-runner.sh:/app/start-job-runner.sh
     ports:
-      - "9000"
+      - "9002:9000"
     healthcheck:
       test: ["CMD", "nc", "-v", "localhost", "9000"]
       interval: 20s
@@ -62,7 +62,7 @@ services:
       - ../config/watcher-merkl-sushiswap-v3/start-server.sh:/app/start-server.sh
     ports:
       - "127.0.0.1:3007:3008"
-      - "9001"
+      - "9003:9001"
     healthcheck:
       test: ["CMD", "nc", "-v", "localhost", "3008"]
       interval: 20s

stack_orchestrator/data/compose/docker-compose-watcher-mobymask-v3-demo.yml

@@ -0,0 +1,162 @@
+version: '3.2'
+
+services:
+  # Starts the PostgreSQL database for watcher
+  mobymask-watcher-db:
+    restart: unless-stopped
+    image: postgres:14-alpine
+    environment:
+      - POSTGRES_USER=vdbm
+      - POSTGRES_MULTIPLE_DATABASES=mobymask-watcher-1,mobymask-watcher-1-job-queue,mobymask-watcher-2,mobymask-watcher-2-job-queue,mobymask-watcher-3,mobymask-watcher-3-job-queue
+      - POSTGRES_EXTENSION=mobymask-watcher-1-job-queue:pgcrypto,mobymask-watcher-2-job-queue:pgcrypto,mobymask-watcher-3-job-queue:pgcrypto
+      - POSTGRES_PASSWORD=password
+    volumes:
+      - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
+      - mobymask_watcher_db_data:/var/lib/postgresql/data
+    ports:
+      - "5432"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+
+  # Deploys the MobyMask contract and generates an invite link
+  # Deployment is skipped if CERC_DEPLOYED_CONTRACT env is set
+  mobymask:
+    image: cerc/mobymask:local
+    working_dir: /app/packages/server
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      # ENV: "PROD" # Check if required
+      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_PRIVATE_KEY_DEPLOYER: ${CERC_PRIVATE_KEY_DEPLOYER}
+      CERC_MOBYMASK_APP_BASE_URI: ${CERC_MOBYMASK_APP_BASE_URI}
+      CERC_DEPLOYED_CONTRACT: ${CERC_DEPLOYED_CONTRACT}
+    command: ["bash", "-c", "./deploy-and-generate-invite.sh"]
+    volumes:
+      - ../config/watcher-mobymask-v3-demo/deploy-and-generate-invite.sh:/app/packages/server/deploy-and-generate-invite.sh
+      - mobymask_deployment:/app/packages/server
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  # Starts a MobyMask v3 watcher server #1
+  mobymask-watcher-1-server:
+    image: cerc/watcher-mobymask-v3:local
+    restart: unless-stopped
+    depends_on:
+      mobymask-watcher-db:
+        condition: service_healthy
+      mobymask:
+        condition: service_completed_successfully
+      nitro-contracts:
+        condition: service_completed_successfully
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_WATCHER_INDEX: 1
+      CERC_ETH_RPC_QUERY_ENDPOINT: ${CERC_ETH_RPC_QUERY_ENDPOINT}
+      CERC_ETH_RPC_MUTATION_ENDPOINT: ${CERC_ETH_RPC_MUTATION_ENDPOINT}
+      CERC_NITRO_CHAIN_URL: ${CERC_NITRO_CHAIN_URL}
+    command: ["bash", "./start-server.sh"]
+    volumes:
+      - ../config/watcher-mobymask-v3-demo/watcher-config-template.toml:/app/environments/watcher-config-template.toml
+      - ../config/watcher-mobymask-v3-demo/watcher-config-rates.toml:/app/environments/rates.toml
+      - ../config/watcher-mobymask-v3-demo/local/keys:/app/keys
+      - ../config/watcher-mobymask-v3-demo/start-server.sh:/app/start-server.sh
+      - watcher_nitro_data:/app/out/nitro-db
+      - nitro_deployment:/nitro
+      - mobymask_deployment:/server
+    # Expose GQL, metrics and relay node ports
+    ports:
+      - "0.0.0.0:3001:3001"
+      - "9001"
+      - "0.0.0.0:9090:9090"
+    healthcheck:
+      test: ["CMD", "busybox", "nc", "localhost", "9090"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  # Starts a MobyMask v3 watcher server #2
+  mobymask-watcher-2-server:
+    image: cerc/watcher-mobymask-v3:local
+    restart: unless-stopped
+    depends_on:
+      mobymask-watcher-1-server:
+        condition: service_healthy
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_WATCHER_INDEX: 2
+      CERC_ETH_RPC_QUERY_ENDPOINT: ${CERC_ETH_RPC_QUERY_ENDPOINT}
+      CERC_ETH_RPC_MUTATION_ENDPOINT: ${CERC_ETH_RPC_MUTATION_ENDPOINT}
+      CERC_NITRO_CHAIN_URL: ${CERC_NITRO_CHAIN_URL}
+    command: ["bash", "./start-server.sh"]
+    volumes:
+      - ../config/watcher-mobymask-v3-demo/watcher-config-template.toml:/app/environments/watcher-config-template.toml
+      - ../config/watcher-mobymask-v3-demo/watcher-config-rates.toml:/app/environments/rates.toml
+      - ../config/watcher-mobymask-v3-demo/local/keys:/app/keys
+      - ../config/watcher-mobymask-v3-demo/start-server.sh:/app/start-server.sh
+      - watcher_nitro_data:/app/out/nitro-db
+      - nitro_deployment:/nitro
+      - mobymask_deployment:/server
+    # Expose GQL, metrics and relay node ports
+    ports:
+      - "3001"
+      - "9001"
+      - "0.0.0.0:9091:9090"
+    healthcheck:
+      test: ["CMD", "busybox", "nc", "localhost", "9090"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  # Starts a MobyMask v3 watcher server #3
+  mobymask-watcher-3-server:
+    image: cerc/watcher-mobymask-v3:local
+    restart: unless-stopped
+    depends_on:
+      mobymask-watcher-1-server:
+        condition: service_healthy
+      mobymask-watcher-2-server:
+        condition: service_healthy
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_WATCHER_INDEX: 3
+      CERC_ETH_RPC_QUERY_ENDPOINT: ${CERC_ETH_RPC_QUERY_ENDPOINT}
+      CERC_ETH_RPC_MUTATION_ENDPOINT: ${CERC_ETH_RPC_MUTATION_ENDPOINT}
+      CERC_NITRO_CHAIN_URL: ${CERC_NITRO_CHAIN_URL}
+    command: ["bash", "./start-server.sh"]
+    volumes:
+      - ../config/watcher-mobymask-v3-demo/watcher-config-template.toml:/app/environments/watcher-config-template.toml
+      - ../config/watcher-mobymask-v3-demo/watcher-config-rates.toml:/app/environments/rates.toml
+      - ../config/watcher-mobymask-v3-demo/local/keys:/app/keys
+      - ../config/watcher-mobymask-v3-demo/start-server.sh:/app/start-server.sh
+      - watcher_nitro_data:/app/out/nitro-db
+      - nitro_deployment:/nitro
+      - mobymask_deployment:/server
+    # Expose GQL, metrics and relay node ports
+    ports:
+      - "3001"
+      - "9001"
+      - "0.0.0.0:9092:9090"
+    healthcheck:
+      test: ["CMD", "busybox", "nc", "localhost", "9090"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+volumes:
+  mobymask_watcher_db_data:
+  mobymask_deployment:
+  nitro_deployment:
+  watcher_nitro_data:

stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml

@@ -35,7 +35,7 @@ services:
       - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
       - ../config/watcher-sushiswap-v3/start-job-runner.sh:/app/start-job-runner.sh
     ports:
-      - "9000"
+      - "9000:9000"
     healthcheck:
       test: ["CMD", "nc", "-v", "localhost", "9000"]
       interval: 20s
@@ -62,7 +62,7 @@ services:
       - ../config/watcher-sushiswap-v3/start-server.sh:/app/start-server.sh
     ports:
       - "127.0.0.1:3008:3008"
-      - "9001"
+      - "9001:9001"
     healthcheck:
       test: ["CMD", "nc", "-v", "localhost", "3008"]
       interval: 20s

stack_orchestrator/data/compose/docker-compose-webapp-template.yml

@@ -5,4 +5,4 @@ services:
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
     ports:
-      - "3000"
+      - "80"

stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/deploy-contracts.sh

@@ -65,7 +65,7 @@ if [ -n "$CERC_L1_ADDRESS" ] && [ -n "$CERC_L1_PRIV_KEY" ]; then
   # Sequencer
   SEQ=$(echo "$wallet3" | awk '/Address:/{print $2}')
   SEQ_KEY=$(echo "$wallet3" | awk '/Private key:/{print $3}')
-  
+
   echo "Funding accounts."
   wait_for_block 1 300
   cast send --from $ADMIN --rpc-url $CERC_L1_RPC --value 5ether $PROPOSER --private-key $ADMIN_KEY
@@ -169,4 +169,4 @@ cp -a /app/packages/contracts-bedrock/deployments/$DEPLOYMENT_CONTEXT /l1-deploy
 cp /app/packages/contracts-bedrock/deploy-config/$DEPLOYMENT_CONTEXT.json /l2-config
 openssl rand -hex 32 > /l2-config/l2-jwt.txt
 
-echo "Deployment successful. Exiting"
+echo "Deployment successful. Exiting."

stack_orchestrator/data/config/monitoring/blackbox.yml

@@ -0,0 +1,7 @@
+modules:
+  http_2xx:
+    prober: http
+    timeout: 5s
+    http:
+      valid_status_codes: [] #default to 2xx
+      method: GET

stack_orchestrator/data/config/monitoring/grafana/dashboards/nodejs-app-dashboard.json

@@ -0,0 +1,943 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "node.js prometheus client basic metrics",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "gnetId": 11159,
+  "graphTooltip": 0,
+  "id": 15,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 10,
+        "x": 0,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 6,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "irate(process_cpu_user_seconds_total{instance=~\"$instance\"}[2m]) * 100",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "User CPU - {{instance}}",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "irate(process_cpu_system_seconds_total{instance=~\"$instance\"}[2m]) * 100",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Sys CPU - {{instance}}",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Process CPU Usage",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "percent",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 9,
+        "x": 10,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 8,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_eventloop_lag_seconds{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "{{instance}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Event Loop Lag",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "s",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "text",
+            "mode": "fixed"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 5,
+        "x": 19,
+        "y": 0
+      },
+      "id": 2,
+      "interval": "",
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "name",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.2.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(nodejs_version_info{instance=~\"$instance\"}) by (version)",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{version}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Node.js Version",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "#F2495C",
+            "mode": "fixed"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 5,
+        "x": 19,
+        "y": 3
+      },
+      "id": 4,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.2.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(changes(process_start_time_seconds{instance=~\"$instance\"}[1m]))",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "{{instance}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Process Restart Times",
+      "type": "stat"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 16,
+        "x": 0,
+        "y": 7
+      },
+      "hiddenSeries": false,
+      "id": 7,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "rightSide": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "process_resident_memory_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Process Memory - {{instance}}",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_heap_size_total_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Heap Total - {{instance}}",
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_heap_size_used_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Heap Used - {{instance}}",
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_external_memory_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "External Memory - {{instance}}",
+          "refId": "D"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Process Memory Usage",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 8,
+        "x": 16,
+        "y": 7
+      },
+      "hiddenSeries": false,
+      "id": 9,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_active_handles_total{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Active Handler - {{instance}}",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_active_requests_total{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Active Request - {{instance}}",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Active Handlers/Requests Total",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 14
+      },
+      "hiddenSeries": false,
+      "id": 10,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_heap_space_size_total_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Heap Total - {{instance}} - {{space}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Heap Total Detail",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 8,
+        "y": 14
+      },
+      "hiddenSeries": false,
+      "id": 11,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_heap_space_size_used_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Heap Used - {{instance}} - {{space}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Heap Used Detail",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 16,
+        "y": 14
+      },
+      "hiddenSeries": false,
+      "id": 12,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": true,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "paceLength": 10,
+      "percentage": false,
+      "pluginVersion": "10.2.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "nodejs_heap_space_size_available_bytes{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "Heap Used - {{instance}} - {{space}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Heap Available Detail",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    }
+  ],
+  "refresh": "10s",
+  "schemaVersion": 38,
+  "tags": [
+    "nodejs"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": true,
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
+        "definition": "label_values(nodejs_version_info, instance)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "instance",
+        "multi": true,
+        "name": "instance",
+        "options": [],
+        "query": "label_values(nodejs_version_info, instance)",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-15m",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "",
+  "title": "NodeJS Application Dashboard",
+  "uid": "PTSqcpJWk",
+  "version": 3,
+  "weekStart": ""
+}

stack_orchestrator/data/config/monitoring/grafana/provisioning/alerting/contactpoints.yml

@@ -0,0 +1,14 @@
+# https://www.clever-cloud.com/blog/features/2021/12/03/slack-alerts-for-grafana/
+
+apiVersion: 1
+
+contactPoints:
+  - orgId: 1
+    name: SlackNotifier
+    receivers:
+      - uid: a71b06e3-58b6-41fe-af65-fbbb29653951
+        type: slack
+        settings:
+          # Slack hook URL (see https://api.slack.com/messaging/webhooks)
+          url: <YOUR_SLACK_HOOK_URL>
+        disableResolveMessage: false

stack_orchestrator/data/config/monitoring/grafana/provisioning/alerting/policies.yml

@@ -0,0 +1,15 @@
+# https://grafana.com/docs/grafana/latest/alerting/alerting-rules/create-notification-policy/
+
+apiVersion: 1
+
+policies:
+  - orgId: 1
+    receiver: grafana-default-email
+    group_by:
+      - grafana_folder
+      - alertname
+    routes:
+      - receiver: SlackNotifier
+        object_matchers:
+          # Add matchers below
+          # - ['grafana_folder', '=', 'MyAlerts']

stack_orchestrator/data/config/monitoring/grafana/provisioning/dashboards/dashboards.yml

@@ -0,0 +1,10 @@
+apiVersion: 1
+
+providers:
+  - name: dashboards
+    type: file
+    updateIntervalSeconds: 10
+    allowUiUpdates: true
+    options:
+      path: /etc/grafana/dashboards
+      foldersFromFilesStructure: true

stack_orchestrator/data/config/monitoring/grafana/provisioning/datasources/prometheus.yml

@@ -0,0 +1,16 @@
+apiVersion: 1
+
+datasources:
+  - id: 1
+    orgId: 1
+    name: Prometheus
+    type: prometheus
+    typeName: Prometheus
+    typeLogoUrl: public/app/plugins/datasource/prometheus/img/prometheus_logo.svg
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    jsonData:
+      httpMethod: POST
+    version: 1
+    editable: true

stack_orchestrator/data/config/monitoring/postgres-exporter.yml

@@ -0,0 +1,8 @@
+auth_modules:
+  foo:
+    type: userpass
+    userpass:
+      username: username
+      password: password
+    options:
+      sslmode: disable

stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml

@@ -0,0 +1,67 @@
+global:
+  scrape_interval:     10s
+  evaluation_interval: 15s
+
+rule_files:
+  # - "first.rules"
+  # - "second.rules"
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: node
+    static_configs:
+      # Add node-exporter targets to be monitored below
+      # - targets: ['example-host:9100']
+      #   labels:
+      #     instance: 'my-host'
+
+  - job_name: 'blackbox'
+    scrape_interval: 10s
+    metrics_path: /probe
+    params:
+      module: [http_2xx]
+    static_configs:
+      # Add URLs to be monitored below
+      - targets:
+        # - https://github.com
+    relabel_configs:
+      - source_labels: [__address__]
+        regex: (.*)(:80)?
+        target_label: __param_target
+      - source_labels: [__param_target]
+        regex: (.*)
+        target_label: instance
+        replacement: ${1}
+      - source_labels: []
+        regex: .*
+        target_label: __address__
+        replacement: blackbox:9115
+
+  - job_name: chain_heads
+    scrape_interval: 10s
+    metrics_path: /metrics
+    scheme: http
+    static_configs:
+      - targets: ['chain-head-exporter:5000']
+
+  - job_name: 'postgres'
+    scrape_interval: 30s
+    scrape_timeout: 30s
+    static_configs:
+      # Add DB targets below
+      # - targets: [example-server:5432]
+        # labels:
+          # instance: 'example-label'
+    metrics_path: /probe
+    params:
+      auth_module: [foo]
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: postgres-exporter:9187

stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml

@@ -0,0 +1,933 @@
+# https://grafana.com/docs/grafana/latest/alerting/alerting-rules/create-grafana-managed-rule/
+
+apiVersion: 1
+
+groups:
+  - orgId: 1
+    name: watcher
+    folder: WatcherAlerts
+    interval: 30s
+    rules:
+      # Azimuth
+      - uid: azimuth_diff_external
+        title: azimuth_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: censures_diff_external
+        title: censures_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: claims_diff_external
+        title: claims_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: conditional_star_release_diff_external
+        title: conditional_star_release_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: delegated_sending_diff_external
+        title: delegated_sending_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: ecliptic_diff_external
+        title: ecliptic_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: linear_star_release_diff_external
+        title: linear_star_release_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="linear_star_release", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: polls_diff_external
+        title: polls_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="ethereum"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+
+      # Sushi
+      - uid: sushiswap_diff_external
+        title: sushiswap_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      - uid: merkl_sushiswap_diff_external
+        title: merkl_sushiswap_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: latest_indexed
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_indexed
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false

stack_orchestrator/data/config/osmosis/.env.production

@@ -0,0 +1,3 @@
+NEXT_PUBLIC_COINGECKO_URL=https://api.coingecko.com
+
+NEXT_PUBLIC_ENABLE_FEATURES=true

stack_orchestrator/data/config/osmosis/build-app-for-urbit.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+# Check and exit if a deployment already exists (on restarts)
+if [ -d /app-builds/osmosis/build ]; then
+  echo "Build already exists, remove volume to rebuild"
+  exit 0
+fi
+
+yarn build:static
+./build-urbit.sh
+
+# Move build to app-builds
+mkdir -p /app-builds/osmosis
+cp -r ./out /app-builds/osmosis/build
+
+cp -r mar /app-builds/osmosis/
+cp desk.docket-0 /app-builds/osmosis/

stack_orchestrator/data/config/osmosis/build-app.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+# Check and exit if a deployment already exists (on restarts)
+if [ -d /app-builds/osmosis/build ]; then
+  echo "Build already exists, remove volume to rebuild"
+  exit 0
+fi
+
+yarn build:static
+
+# Move build to app-builds
+mkdir -p /app-builds/osmosis
+cp -r ./out /app-builds/osmosis/build

stack_orchestrator/data/config/osmosis/nginx/web-app.conf

@@ -0,0 +1,16 @@
+server {
+    listen       80;
+    listen  [::]:80;
+    server_name  _;
+
+    error_page   500 502 503 504  /50x.html;
+
+    location / {
+        root   /usr/share/nginx/osmosis/build;
+        index index.html index.htm index.nginx-debian.html;
+
+        # First attempt to serve request as file, then as html,
+		# then as directory, then fall back to displaying a 404.
+		try_files $uri $uri.html $uri/ /index.html =404;
+    }
+}

stack_orchestrator/data/config/osmosis/urbit-files/desk.docket-0

@@ -0,0 +1,10 @@
+:~  title+'Osmosis'
+    info+'Osmosis DEX Frontend, built and maintained by Laconic'
+    color+0xcd.75df
+    image+'https://altcoinsbox.com/wp-content/uploads/2023/03/osmosis-logo.png'
+    base+'osmosis'
+    glob-http+['REPLACE_WITH_GLOB_URL' REPLACE_WITH_GLOB_HASH]
+    version+[0 0 2]
+    website+'https://osmosis.zone/'
+    license+'MIT'
+==

stack_orchestrator/data/config/osmosis/urbit-files/mar/ico.hoon

@@ -0,0 +1,12 @@
+|_  dat=@
+++  grow
+  |%
+  ++  mime  [/image/x-icon (as-octs:mimes:html dat)]
+  --
+++  grab
+  |%
+  ++  mime  |=([p=mite q=octs] q.q)
+  ++  noun  @
+  --
+++  grad  %mime
+--

stack_orchestrator/data/config/osmosis/urbit-files/mar/jpg.hoon

@@ -0,0 +1,12 @@
+|_  dat=@
+++  grow
+  |%
+  ++  mime  [/image/jpeg (as-octs:mimes:html dat)]
+  --
+++  grab
+  |%
+  ++  mime  |=([p=mite q=octs] q.q)
+  ++  noun  @
+  --
+++  grad  %mime
+--

stack_orchestrator/data/config/osmosis/urbit-files/mar/map.hoon

@@ -0,0 +1,18 @@
+::
+::::  /hoon/map/mar
+  ::  Mark for js source maps
+/?    310
+::
+=,  eyre
+|_  mud=@
+++  grow
+  |%
+  ++  mime  [/application/octet-stream (as-octs:mimes:html (@t mud))]
+  --
+++  grab
+  |%                                                    ::  convert from
+  ++  mime  |=([p=mite q=octs] (@t q.q))
+  ++  noun  cord                                        ::  clam from %noun
+  --
+++  grad  %mime
+--

stack_orchestrator/data/config/osmosis/urbit-files/mar/webp.hoon

@@ -0,0 +1,12 @@
+|_  dat=@
+++  grow
+  |%
+  ++  mime  [/image/webp (as-octs:mimes:html dat)]
+  --
+++  grab
+  |%
+  ++  mime  |=([p=mite q=octs] q.q)
+  ++  noun  @
+  --
+++  grad  %mime
+--

stack_orchestrator/data/config/uniswap-interface/build-app.sh

@@ -13,6 +13,9 @@ fi
 
 yarn build
 
-# Move build to app-builds so urbit can deploy it
-mkdir /app-builds/uniswap
+# Copy over build and other files to app-builds for urbit deployment
+mkdir -p /app-builds/uniswap
 cp -r ./build /app-builds/uniswap/
+
+cp -r mar /app-builds/uniswap/
+cp desk.docket-0 /app-builds/uniswap/

stack_orchestrator/data/config/uniswap-interface/deploy-uniswap-app.sh

@@ -1,149 +0,0 @@
-#!/bin/bash
-
-if [ -n "$CERC_SCRIPT_DEBUG" ]; then
-  set -x
-fi
-
-echo "Using IPFS endpoint ${CERC_IPFS_GLOB_HOST_ENDPOINT} for hosting globs"
-echo "Using IPFS server endpoint ${CERC_IPFS_SERVER_ENDPOINT} for reading glob files"
-ipfs_host_endpoint=${CERC_IPFS_GLOB_HOST_ENDPOINT}
-ipfs_server_endpoint=${CERC_IPFS_SERVER_ENDPOINT}
-
-uniswap_app_build='/app-builds/uniswap/build'
-uniswap_desk_dir='/urbit/zod/uniswap'
-
-if [ -d ${uniswap_desk_dir} ]; then
-  echo "Uniswap desk dir already exists, skipping deployment..."
-  exit 0
-fi
-
-# Fire curl requests to perform operations on the ship
-dojo () {
-  curl -s --data '{"source":{"dojo":"'"$1"'"},"sink":{"stdout":null}}' http://localhost:12321
-}
-
-hood () {
-  curl -s --data '{"source":{"dojo":"+hood/'"$1"'"},"sink":{"app":"hood"}}' http://localhost:12321
-}
-
-# Create/mount a uniswap desk
-hood "merge %uniswap our %landscape"
-hood "mount %uniswap"
-
-# Loop until the uniswap build appears
-while [ ! -d ${uniswap_app_build} ]; do
-  echo "Uniswap app build not found, retrying in 5s..."
-  sleep 5
-done
-echo "Build found..."
-
-# Copy over build to desk data dir
-cp -r ${uniswap_app_build} ${uniswap_desk_dir}
-
-# Create a mark file for .map file type
-cat << EOF > "${uniswap_desk_dir}/mar/map.hoon"
-::
-::::  /hoon/map/mar
-  ::  Mark for js source maps
-/?    310
-::
-=,  eyre
-|_  mud=@
-++  grow
-  |%
-  ++  mime  [/application/octet-stream (as-octs:mimes:html (@t mud))]
-  --
-++  grab
-  |%                                                    ::  convert from
-  ++  mime  |=([p=mite q=octs] (@t q.q))
-  ++  noun  cord                                        ::  clam from %noun
-  --
-++  grad  %mime
---
-EOF
-
-# Create a mark file for .woff file type
-cat << EOF > "${uniswap_desk_dir}/mar/woff.hoon"
-|_  dat=octs
-++  grow
-  |%
-  ++  mime  [/font/woff dat]
-  --
-++  grab
-  |%
-  ++  mime  |=([=mite =octs] octs)
-  ++  noun  octs
-  --
-++  grad  %mime
---
-EOF
-
-# Create a mark file for .ttf file type
-cat << EOF > "${uniswap_desk_dir}/mar/ttf.hoon"
-|_  dat=octs
-++  grow
-  |%
-  ++  mime  [/font/ttf dat]
-  --
-++  grab
-  |%
-  ++  mime  |=([=mite =octs] octs)
-  ++  noun  octs
-  --
-++  grad  %mime
---
-EOF
-
-rm "${uniswap_desk_dir}/desk.bill"
-rm "${uniswap_desk_dir}/desk.ship"
-
-# Commit changes and create a glob
-hood "commit %uniswap"
-dojo "-landscape!make-glob %uniswap /build"
-
-glob_file=$(ls -1 -c zod/.urb/put | head -1)
-echo "Created glob file: ${glob_file}"
-
-upload_response=$(curl -X POST -F file=@./zod/.urb/put/${glob_file} ${ipfs_host_endpoint}/api/v0/add)
-glob_cid=$(echo "$upload_response" | grep -o '"Hash":"[^"]*' | sed 's/"Hash":"//')
-
-echo "Glob file uploaded to IFPS:"
-echo "{ cid: ${glob_cid}, filename: ${glob_file} }"
-
-# Curl and wait for the glob to be hosted
-glob_url="${ipfs_server_endpoint}/ipfs/${glob_cid}?filename=${glob_file}"
-
-echo "Checking if glob file hosted at ${glob_url}"
-while true; do
-  response=$(curl -sL -w "%{http_code}" -o /dev/null "$glob_url")
-
-  if [ $response -eq 200 ]; then
-    echo "File found at $glob_url"
-    break  # Exit the loop if the file is found
-  else
-    echo "File not found. Retrying in a few seconds..."
-    sleep 5
-  fi
-done
-
-glob_hash=$(echo "$glob_file" | sed "s/glob-\([a-z0-9\.]*\).glob/\1/")
-
-# Update the docket file
-cat << EOF > "${uniswap_desk_dir}/desk.docket-0"
-:~  title+'Uniswap'
-    info+'Self-hosted uniswap frontend.'
-    color+0xcd.75df
-    image+'https://logowik.com/content/uploads/images/uniswap-uni7403.jpg'
-    base+'uniswap'
-    glob-http+['${glob_url}' ${glob_hash}]
-    version+[0 0 1]
-    website+'https://uniswap.org/'
-    license+'MIT'
-==
-EOF
-
-# Commit changes and install the app
-hood "commit %uniswap"
-hood "install our %uniswap"
-
-echo "Uniswap app installed"

stack_orchestrator/data/config/uniswap-interface/install-uniswap-app.sh

@@ -1,110 +0,0 @@
-#!/bin/bash
-
-# $1: Glob file URL (eg. https://xyz.com/glob-0vabcd.glob)
-# $2: Glob file hash (eg. 0vabcd)
-# $3: Urbit ship's pier dir (default: ./zod)
-
-if [ "$#" -lt 2 ]; then
-  echo "Insufficient arguments"
-  exit 0
-fi
-
-glob_url=$1
-glob_hash=$2
-echo "Using glob file from ${glob_url} with hash ${glob_hash}"
-
-# Default pier dir: ./zod
-# Default desk dir: ./zod/uniswap
-pier_dir="${3:-./zod}"
-uniswap_desk_dir="${pier_dir}/uniswap"
-echo "Using ${uniswap_desk_dir} as the Uniswap desk dir path"
-
-# Fire curl requests to perform operations on the ship
-dojo () {
-  curl -s --data '{"source":{"dojo":"'"$1"'"},"sink":{"stdout":null}}' http://localhost:12321
-}
-
-hood () {
-  curl -s --data '{"source":{"dojo":"+hood/'"$1"'"},"sink":{"app":"hood"}}' http://localhost:12321
-}
-
-# Create/mount a uniswap desk
-hood "merge %uniswap our %landscape"
-hood "mount %uniswap"
-
-# Create a mark file for .map file type
-cat << EOF > "${uniswap_desk_dir}/mar/map.hoon"
-::
-::::  /hoon/map/mar
-  ::  Mark for js source maps
-/?    310
-::
-=,  eyre
-|_  mud=@
-++  grow
-  |%
-  ++  mime  [/application/octet-stream (as-octs:mimes:html (@t mud))]
-  --
-++  grab
-  |%                                                    ::  convert from
-  ++  mime  |=([p=mite q=octs] (@t q.q))
-  ++  noun  cord                                        ::  clam from %noun
-  --
-++  grad  %mime
---
-EOF
-
-# Create a mark file for .woff file type
-cat << EOF > "${uniswap_desk_dir}/mar/woff.hoon"
-|_  dat=octs
-++  grow
-  |%
-  ++  mime  [/font/woff dat]
-  --
-++  grab
-  |%
-  ++  mime  |=([=mite =octs] octs)
-  ++  noun  octs
-  --
-++  grad  %mime
---
-EOF
-
-# Create a mark file for .ttf file type
-cat << EOF > "${uniswap_desk_dir}/mar/ttf.hoon"
-|_  dat=octs
-++  grow
-  |%
-  ++  mime  [/font/ttf dat]
-  --
-++  grab
-  |%
-  ++  mime  |=([=mite =octs] octs)
-  ++  noun  octs
-  --
-++  grad  %mime
---
-EOF
-
-rm "${uniswap_desk_dir}/desk.bill"
-rm "${uniswap_desk_dir}/desk.ship"
-
-# Update the docket file
-cat << EOF > "${uniswap_desk_dir}/desk.docket-0"
-:~  title+'Uniswap'
-    info+'Self-hosted uniswap frontend.'
-    color+0xcd.75df
-    image+'https://logowik.com/content/uploads/images/uniswap-uni7403.jpg'
-    base+'uniswap'
-    glob-http+['${glob_url}' ${glob_hash}]
-    version+[0 0 1]
-    website+'https://uniswap.org/'
-    license+'MIT'
-==
-EOF
-
-# Commit changes and install the app
-hood "commit %uniswap"
-hood "install our %uniswap"
-
-echo "Uniswap app installed"

stack_orchestrator/data/config/uniswap-interface/remote-deploy-uniswap.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-
-# $1: Remote user host
-# $2: Remote Urbit ship's pier dir path (eg. /home/user/zod)
-# $3: Glob file URL (eg. https://xyz.com/glob-0vabcd.glob)
-# $4: Glob file hash (eg. 0vabcd)
-
-if [ "$#" -ne 4 ]; then
-  echo "Incorrect number of arguments"
-  echo "Usage: $0 <username@remote_host> </path/to/remote/pier/folder> <glob_url> <glob_hash>"
-  exit 1
-fi
-
-remote_user_host="$1"
-remote_pier_folder="$2"
-glob_url="$3"
-glob_hash="$4"
-
-installation_script="./install-uniswap-app.sh"
-
-ssh "$remote_user_host" "bash -s $glob_url $glob_hash $remote_pier_folder" < "$installation_script"

stack_orchestrator/data/config/uniswap-interface/urbit-files/desk.docket-0

@@ -0,0 +1,10 @@
+:~  title+'Uniswap'
+    info+'Self-hosted uniswap frontend.'
+    color+0xcd.75df
+    image+'https://logowik.com/content/uploads/images/uniswap-uni7403.jpg'
+    base+'uniswap'
+    glob-http+['REPLACE_WITH_GLOB_URL' REPLACE_WITH_GLOB_HASH]
+    version+[0 0 1]
+    website+'https://uniswap.org/'
+    license+'MIT'
+==

stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/map.hoon

@@ -0,0 +1,18 @@
+::
+::::  /hoon/map/mar
+  ::  Mark for js source maps
+/?    310
+::
+=,  eyre
+|_  mud=@
+++  grow
+  |%
+  ++  mime  [/application/octet-stream (as-octs:mimes:html (@t mud))]
+  --
+++  grab
+  |%                                                    ::  convert from
+  ++  mime  |=([p=mite q=octs] (@t q.q))
+  ++  noun  cord                                        ::  clam from %noun
+  --
+++  grad  %mime
+--

stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/ttf.hoon

@@ -0,0 +1,12 @@
+|_  dat=octs
+++  grow
+  |%
+  ++  mime  [/font/ttf dat]
+  --
+++  grab
+  |%
+  ++  mime  |=([=mite =octs] octs)
+  ++  noun  octs
+  --
+++  grad  %mime
+--

stack_orchestrator/data/config/uniswap-interface/urbit-files/mar/woff.hoon

@@ -0,0 +1,12 @@
+|_  dat=octs
+++  grow
+  |%
+  ++  mime  [/font/woff dat]
+  --
+++  grab
+  |%
+  ++  mime  |=([=mite =octs] octs)
+  ++  noun  octs
+  --
+++  grad  %mime
+--

stack_orchestrator/data/config/urbit/deploy-app-to-remote-urbit.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# $1: Remote user host
+# $2: App name (eg. uniswap)
+# $3: Assets dir path (local) for app (eg. /home/user/myapp/urbit-files)
+# $4: Remote Urbit ship's pier dir path (eg. /home/user/zod)
+# $5: Glob file URL (eg. https://xyz.com/glob-0vabcd.glob)
+# $6: Glob file hash (eg. 0vabcd)
+
+if [ "$#" -ne 6 ]; then
+  echo "Incorrect number of arguments"
+  echo "Usage: $0 <username@remote_host> <app_name> </path/to/app/assets/folder> </path/to/remote/pier/folder> <glob_url> <glob_hash>"
+  exit 1
+fi
+
+remote_user_host="$1"
+app_name=$2
+app_assets_folder=$3
+remote_pier_folder="$4"
+glob_url="$5"
+glob_hash="$6"
+
+installation_script="./install-urbit-app.sh"
+
+# Copy over the assets to remote machine in a tmp dir
+remote_app_assets_folder=/tmp/urbit-app-assets/$app_name
+ssh "$remote_user_host" "mkdir -p $remote_app_assets_folder"
+scp -r $app_assets_folder/* $remote_user_host:$remote_app_assets_folder
+
+# Run the installation script
+ssh "$remote_user_host" "bash -s $app_name $remote_app_assets_folder '${glob_url}' $glob_hash $remote_pier_folder" < "$installation_script"
+
+# Remove the tmp assets dir
+ssh "$remote_user_host" "rm -rf $remote_app_assets_folder"

stack_orchestrator/data/config/urbit/deploy-app.sh

@@ -0,0 +1,110 @@
+#!/bin/bash
+
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+if [ -z "$CERC_URBIT_APP" ]; then
+  echo "CERC_URBIT_APP not set, exiting"
+  exit 0
+fi
+
+echo "Creating Urbit application for ${CERC_URBIT_APP}"
+
+app_desk_dir=/urbit/zod/${CERC_URBIT_APP}
+if [ -d ${app_desk_dir} ]; then
+  echo "Desk dir already exists for ${CERC_URBIT_APP}, skipping deployment..."
+  exit 0
+fi
+
+app_build=/app-builds/${CERC_URBIT_APP}/build
+app_mark_files=/app-builds/${CERC_URBIT_APP}/mar
+app_docket_file=/app-builds/${CERC_URBIT_APP}/desk.docket-0
+
+echo "Reading app build from ${app_build}"
+echo "Reading additional mark files from ${app_mark_files}"
+echo "Reading docket file ${app_docket_file}"
+
+# Loop until the app's build appears
+while [ ! -d ${app_build} ]; do
+  echo "${CERC_URBIT_APP} app build not found, retrying in 5s..."
+  sleep 5
+done
+echo "Build found..."
+
+echo "Using IPFS endpoint ${CERC_IPFS_GLOB_HOST_ENDPOINT} for hosting the ${CERC_URBIT_APP} glob"
+echo "Using IPFS server endpoint ${CERC_IPFS_SERVER_ENDPOINT} for reading ${CERC_URBIT_APP} glob"
+ipfs_host_endpoint=${CERC_IPFS_GLOB_HOST_ENDPOINT}
+ipfs_server_endpoint=${CERC_IPFS_SERVER_ENDPOINT}
+
+# Fire curl requests to perform operations on the ship
+dojo () {
+  curl -s --data '{"source":{"dojo":"'"$1"'"},"sink":{"stdout":null}}' http://localhost:12321
+}
+
+hood () {
+  curl -s --data '{"source":{"dojo":"+hood/'"$1"'"},"sink":{"app":"hood"}}' http://localhost:12321
+}
+
+# Create / mount the app's desk
+hood "merge %${CERC_URBIT_APP} our %landscape"
+hood "mount %${CERC_URBIT_APP}"
+
+# Copy over build to desk data dir
+cp -r ${app_build} ${app_desk_dir}
+
+# Copy over the additional mark files
+cp ${app_mark_files}/* ${app_desk_dir}/mar/
+
+rm "${app_desk_dir}/desk.bill"
+rm "${app_desk_dir}/desk.ship"
+
+# Commit changes and create a glob
+hood "commit %${CERC_URBIT_APP}"
+dojo "-landscape!make-glob %${CERC_URBIT_APP} /build"
+
+glob_file=$(ls -1 -c zod/.urb/put | head -1)
+echo "Created glob file: ${glob_file}"
+
+# Upload the glob file to IPFS
+echo "Uploading glob file to ${ipfs_host_endpoint}"
+upload_response=$(curl -X POST -F file=@./zod/.urb/put/${glob_file} ${ipfs_host_endpoint}/api/v0/add)
+glob_cid=$(echo "$upload_response" | grep -o '"Hash":"[^"]*' | sed 's/"Hash":"//')
+
+glob_url="${ipfs_server_endpoint}/ipfs/${glob_cid}?filename=${glob_file}"
+glob_hash=$(echo "$glob_file" | sed "s/glob-\([a-z0-9\.]*\).glob/\1/")
+
+echo "Glob file uploaded to IFPS:"
+echo "{ cid: ${glob_cid}, filename: ${glob_file} }"
+echo "{ url: ${glob_url}, hash: ${glob_hash} }"
+
+# Exit if the installation not required
+if [ "$CERC_ENABLE_APP_INSTALL" = "false" ]; then
+  echo "CERC_ENABLE_APP_INSTALL set to false, skipping app installation"
+  exit 0
+fi
+
+# Curl and wait for the glob to be hosted
+echo "Checking if glob file hosted at ${glob_url}"
+while true; do
+  response=$(curl -sL -w "%{http_code}" -o /dev/null "$glob_url")
+
+  if [ $response -eq 200 ]; then
+    echo "File found at $glob_url"
+    break  # Exit the loop if the file is found
+  else
+    echo "File not found, retrying in a 5s..."
+    sleep 5
+  fi
+done
+
+# Replace the docket file for app
+# Substitue the glob URL and hash
+cp ${app_docket_file} ${app_desk_dir}/
+sed -i "s|REPLACE_WITH_GLOB_URL|${glob_url}|g; s|REPLACE_WITH_GLOB_HASH|${glob_hash}|g" ${app_desk_dir}/desk.docket-0
+
+# Commit changes and install the app
+hood "commit %${CERC_URBIT_APP}"
+hood "install our %${CERC_URBIT_APP}"
+
+echo "${CERC_URBIT_APP} app installed"

stack_orchestrator/data/config/urbit/install-urbit-app.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# $1: App name (eg. uniswap)
+# $2: Assets dir path (local) for app (eg. /home/user/myapp/urbit-files)
+# $3: Glob file URL (eg. https://xyz.com/glob-0vabcd.glob)
+# $4: Glob file hash (eg. 0vabcd)
+# $5: Urbit ship's pier dir (default: ./zod)
+
+if [ "$#" -lt 4 ]; then
+  echo "Insufficient arguments"
+  echo "Usage: $0 <app_name> </path/to/app/assets/folder> <glob_url> <glob_hash> [/path/to/remote/pier/folder]"
+  exit 1
+fi
+
+app_name=$1
+app_mark_files=$2/mar
+app_docket_file=$2/desk.docket-0
+echo "Creating Urbit application for ${app_name}"
+echo "Reading additional mark files from ${app_mark_files}"
+echo "Reading docket file ${app_docket_file}"
+
+glob_url=$3
+glob_hash=$4
+echo "Using glob file from ${glob_url} with hash ${glob_hash}"
+
+# Default pier dir: ./zod
+# Default desk dir: ./zod/<app_name>
+pier_dir="${5:-./zod}"
+app_desk_dir="${pier_dir}/${app_name}"
+echo "Using ${app_desk_dir} as the ${app_name} desk dir path"
+
+# Fire curl requests to perform operations on the ship
+dojo () {
+  curl -s --data '{"source":{"dojo":"'"$1"'"},"sink":{"stdout":null}}' http://localhost:12321
+}
+
+hood () {
+  curl -s --data '{"source":{"dojo":"+hood/'"$1"'"},"sink":{"app":"hood"}}' http://localhost:12321
+}
+
+# Create / mount the app's desk
+hood "merge %${app_name} our %landscape"
+hood "mount %${app_name}"
+
+# Copy over the additional mark files
+cp ${app_mark_files}/* ${app_desk_dir}/mar/
+
+rm "${app_desk_dir}/desk.bill"
+rm "${app_desk_dir}/desk.ship"
+
+# Replace the docket file for app
+# Substitue the glob URL and hash
+cp ${app_docket_file} ${app_desk_dir}/
+sed -i "s|REPLACE_WITH_GLOB_URL|${glob_url}|g; s|REPLACE_WITH_GLOB_HASH|${glob_hash}|g" ${app_desk_dir}/desk.docket-0
+
+# Commit changes and install the app
+hood "commit %${app_name}"
+hood "install our %${app_name}"
+
+echo "${app_name} app installed"

stack_orchestrator/data/config/urbit/run-urbit-ship.sh

@@ -7,11 +7,13 @@ fi
 
 pier_dir="/urbit/zod"
 
+# TODO: Bootstrap fake ship on the first run
+
 # Run urbit ship in daemon mode
 # Check if the directory exists
 if [ -d "$pier_dir" ]; then
   echo "Pier directory already exists, rebooting..."
-  urbit -d zod
+  /urbit/zod/.run -d
 else
   echo "Creating a new fake ship..."
   urbit -d -F zod

stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml

@@ -41,7 +41,7 @@
     timeTravelMaxAge = 86400 # 1 day
 
 [metrics]
-  host = "127.0.0.1"
+  host = "0.0.0.0"
   port = 9000
   [metrics.gql]
     port = 9001
@@ -84,8 +84,6 @@
   subgraphEventsOrder = true
   # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
   blockDelayInMilliSecs = 30000
-  prefetchBlocksInMem = false
-  prefetchBlockCount = 10
 
   # Boolean to switch between modes of processing events when starting the server.
   # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.

stack_orchestrator/data/config/watcher-mobymask-v3-demo/deploy-and-generate-invite.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+# From env:
+# CERC_ETH_RPC_ENDPOINT
+# CERC_MOBYMASK_APP_BASE_URI
+# CERC_DEPLOYED_CONTRACT
+# CERC_PRIVATE_KEY_DEPLOYER
+
+# Check if CERC_DEPLOYED_CONTRACT environment variable set to skip contract deployment
+if [ -n "$CERC_DEPLOYED_CONTRACT" ]; then
+  echo "CERC_DEPLOYED_CONTRACT is set to '$CERC_DEPLOYED_CONTRACT'"
+  echo "Skipping contract deployment"
+  exit 0
+fi
+
+echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+
+# Wait till ETH RPC endpoint is available with block number > 1
+retry_interval=5
+while true; do
+  block_number_hex=$(curl -s -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' ${CERC_ETH_RPC_ENDPOINT} | jq -r '.result')
+
+  # Check if the request call was successful
+  if [ $? -ne 0 ]; then
+    echo "RPC endpoint not yet available, retrying in $retry_interval seconds..."
+    sleep $retry_interval
+    continue
+  fi
+
+  # Convert hex to decimal
+  block_number_dec=$(printf %u ${block_number_hex})
+
+  # Check if block number is > 1 to avoid failures in the deployment
+  if [ "$block_number_dec" -ge 1 ]; then
+    echo "RPC endpoint is up"
+    break
+  else
+    echo "RPC endpoint not yet available, retrying in $retry_interval seconds..."
+    sleep $retry_interval
+    continue
+  fi
+done
+
+echo "Using CERC_PRIVATE_KEY_DEPLOYER from env"
+
+# Create the required JSON and write it to a file
+secrets_file="secrets.json"
+secrets_json=$(jq -n \
+  --arg privateKey "$CERC_PRIVATE_KEY_DEPLOYER" \
+  --arg rpcUrl "$CERC_ETH_RPC_ENDPOINT" \
+  --arg baseURI "$CERC_MOBYMASK_APP_BASE_URI" \
+  '.privateKey = $privateKey | .rpcUrl = $rpcUrl | .baseURI = $baseURI')
+echo "$secrets_json" > "${secrets_file}"
+
+export RPC_URL="${CERC_ETH_RPC_ENDPOINT}"
+
+# Check and exit if a deployment already exists (on restarts)
+if [ -f ./config.json ]; then
+  echo "config.json already exists, checking the contract deployment"
+
+  # Read JSON file
+  deployment_details=$(cat config.json)
+  deployed_contract=$(echo "$deployment_details" | jq -r '.address')
+
+  cd ../hardhat
+  if yarn verifyDeployment --network optimism --contract "${deployed_contract}"; then
+    echo "Deployment verfication successful"
+    cd ../server
+  else
+    echo "Deployment verfication failed, please clear MobyMask deployment volume before starting"
+    exit 1
+  fi
+fi
+
+npm run deployAndGenerateInvite

stack_orchestrator/data/config/watcher-mobymask-v3-demo/local/config.env

@@ -0,0 +1,16 @@
+# Watchers
+
+CERC_ETH_RPC_ENDPOINT="http://op-geth:8545"
+
+# L1 pre-funded account #4, funded on L2 as well
+CERC_PRIVATE_KEY_DEPLOYER="0xbe4aa664815ea3bc3d63118649a733f6c96b243744310806ecb6d96359ab62cf"
+CERC_MOBYMASK_APP_BASE_URI="http://127.0.0.1:3004/#"
+
+CERC_ETH_RPC_QUERY_ENDPOINT="http://op-geth:8545"
+CERC_ETH_RPC_MUTATION_ENDPOINT="http://op-geth:8545"
+CERC_NITRO_CHAIN_URL="http://op-geth:8546"
+
+# App
+
+CERC_APP_WATCHER_URL="http://localhost:3001"
+CERC_SNAP_URL=http://localhost:8080

stack_orchestrator/data/config/watcher-mobymask-v3-demo/local/fund-accounts-on-l2.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+echo "*************************************"
+
+CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}"
+CERC_L1_CHAIN_ID="${CERC_L1_CHAIN_ID:-${DEFAULT_CERC_L1_CHAIN_ID}}"
+CERC_L1_ACCOUNTS_CSV_URL="${CERC_L1_ACCOUNTS_CSV_URL:-${DEFAULT_CERC_L1_ACCOUNTS_CSV_URL}}"
+
+# Fixture accounts
+WATCHER_1_ACCOUNT=0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266
+WATCHER_2_ACCOUNT=0x70997970C51812dc3A010C7d01b50e0d17dc79C8
+WATCHER_3_ACCOUNT=0x15d34AAf54267DB7D7c367839AAf71A00a2C6A65
+NITRO_APP_ACCOUNT=0xbDA5747bFD65F08deb54cb465eB87D40e51B197E
+
+WATCHER_1_ACCOUNT_KEY=0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80
+WATCHER_2_ACCOUNT_KEY=0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d
+WATCHER_3_ACCOUNT_KEY=0x47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a
+NITRO_APP_ACCOUNT_KEY=0x689af8efa8c651a91ad287602527f3af2fe9f6501a7ac4b061667b5a93e037fd
+
+# Check if watcher 1 account already funded
+WATCHER_1_ACCOUNT_BALANCE=$(cast balance ${WATCHER_1_ACCOUNT} --rpc-url $CERC_L1_RPC)
+echo "WATCHER_1_ACCOUNT_BALANCE ${WATCHER_1_ACCOUNT_BALANCE}"
+if [ "$WATCHER_1_ACCOUNT_BALANCE" != "0" ]; then
+  echo "Watcher account already funded, exiting."
+  exit 0
+fi
+
+echo "Funding accounts on L2"
+
+# Fetch the L1 funded accounts
+curl -o accounts.csv $CERC_L1_ACCOUNTS_CSV_URL
+
+# Use the accounts other than the ones used for Optimism deployment
+ACCOUNT_1=$(awk -F ',' 'NR == 5 {print $2}' accounts.csv)
+ACCOUNT_1_KEY=$(awk -F ',' 'NR == 5 {print $3}' accounts.csv)
+
+ACCOUNT_2=$(awk -F ',' 'NR == 6 {print $2}' accounts.csv)
+ACCOUNT_2_KEY=$(awk -F ',' 'NR == 6 {print $3}' accounts.csv)
+
+# Get the bridge contract address
+DEPLOYMENT_CONTEXT="$CERC_L1_CHAIN_ID"
+BRIDGE=$(cat /l1-deployment/$DEPLOYMENT_CONTEXT/L1StandardBridgeProxy.json | jq -r .address)
+
+# Send balance to bridge contract on L1
+cast send --rpc-url $CERC_L1_RPC --from $ACCOUNT_1 --value 10000ether $BRIDGE --private-key $ACCOUNT_1_KEY
+cast send --rpc-url $CERC_L1_RPC --from $ACCOUNT_2 --value 10000ether $BRIDGE --private-key $ACCOUNT_2_KEY
+
+echo "Following accounts have been funded; use them for transactions on L2:"
+echo "${ACCOUNT_1}"
+echo "${ACCOUNT_2}"
+
+echo "*************************************"
+echo "Funding the watcher and app Nitro accounts on L2"
+
+cast send --rpc-url $CERC_L1_RPC --from $ACCOUNT_1 --value 200ether $WATCHER_1_ACCOUNT --private-key $ACCOUNT_1_KEY
+cast send --rpc-url $CERC_L1_RPC --from $ACCOUNT_1 --value 200ether $WATCHER_2_ACCOUNT --private-key $ACCOUNT_1_KEY
+cast send --rpc-url $CERC_L1_RPC --from $ACCOUNT_1 --value 200ether $WATCHER_3_ACCOUNT --private-key $ACCOUNT_1_KEY
+cast send --rpc-url $CERC_L1_RPC --from $ACCOUNT_1 --value 200ether $NITRO_APP_ACCOUNT --private-key $ACCOUNT_1_KEY
+
+cast send --rpc-url $CERC_L1_RPC --from $WATCHER_1_ACCOUNT --value 100ether $BRIDGE --private-key $WATCHER_1_ACCOUNT_KEY
+sleep 5
+cast send --rpc-url $CERC_L1_RPC --from $WATCHER_2_ACCOUNT --value 100ether $BRIDGE --private-key $WATCHER_2_ACCOUNT_KEY
+sleep 5
+cast send --rpc-url $CERC_L1_RPC --from $WATCHER_3_ACCOUNT --value 100ether $BRIDGE --private-key $WATCHER_3_ACCOUNT_KEY
+sleep 5
+cast send --rpc-url $CERC_L1_RPC --from $NITRO_APP_ACCOUNT --value 100ether $BRIDGE --private-key $NITRO_APP_ACCOUNT_KEY
+
+echo "Done, exiting."

stack_orchestrator/data/config/watcher-mobymask-v3-demo/local/keys/watcher-1.json

@@ -0,0 +1,21 @@
+{
+  "peerId": {
+    "id": "12D3KooWAMjBkFCT9DtCnSDcxftxJzSuTBvzVojabv64cnEvX4AZ",
+    "privKey": "CAESQAKCrnY0QKTky1I18fqn+VPydXGUv1NYiV+nVKqBFkw/CAjE9sKKIDGnYAo8mivnI6dngFenERY+0Q8AJrPTaXY=",
+    "pubKey": "CAESIAgIxPbCiiAxp2AKPJor5yOnZ4BXpxEWPtEPACaz02l2"
+  },
+  "relayPeerId": {
+    "id": "12D3KooWNQJDGtH3YAZk2qG1ZbcoQvw6HaD814pfd9HHQthgX63K",
+    "privKey": "CAESQGc996T2OuyV3aY+hnna+GIexKBEcE6jDNX/RaofHr58uv2EYxXm4kcL2tO2tKgnPgtkegHFdjqy6h/xOA+YMZo=",
+    "pubKey": "CAESILr9hGMV5uJHC9rTtrSoJz4LZHoBxXY6suof8TgPmDGa"
+  },
+  "peer": {
+    "privateKey": "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80",
+    "nitroAddress": "0xAAA6628Ec44A8a742987EF3A114dDFE2D4F7aDCE",
+    "nitroPrivateKey": "2d999770f7b5d49b694080f987b82bbc9fc9ac2b4dcc10b0f8aba7d700f69c6d"
+  },
+  "consensus": {
+    "publicKey": "02cd17b05ca998955be5ca7bf4fd4531243d438f1aae7ce8a0ed5159f53cee5b40",
+    "privateKey": "67d80505614bdf61fca11cbad31d93acb2c7df1c653dc25975d77d05f05f154f"
+  }
+}

stack_orchestrator/data/config/watcher-mobymask-v3-demo/local/keys/watcher-2.json

@@ -0,0 +1,21 @@
+{
+  "peerId": {
+    "id": "12D3KooWBNEbY3QS4y23ngupDw9PDc4bvNvRJGVRejjV9EZLjux5",
+    "privKey": "CAESQGSTw0ymvn8+wX9Dbvyr4/Gib1q2voe0CC0VyeClMQP6FwW14x0fpRbBIx0XhLdxWHkRndphVg3gVAHyC+7ZI8o=",
+    "pubKey": "CAESIBcFteMdH6UWwSMdF4S3cVh5EZ3aYVYN4FQB8gvu2SPK"
+  },
+  "relayPeerId": {
+    "id": "12D3KooWPpBt2coC5BKEsewogHSkdgdyY9guUHJBgJFZoh66R4UL",
+    "privKey": "CAESQBdoGZzpiHsE8GlIhGGP+tl6A13sIPxOhgSwsWZR4cwmz/f7627X5GQRLD41ikH6oZ7ZeSbiiPsBtelL1+t3M80=",
+    "pubKey": "CAESIM/3++tu1+RkESw+NYpB+qGe2Xkm4oj7AbXpS9frdzPN"
+  },
+  "peer": {
+    "privateKey": "59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d",
+    "nitroAddress": "0xBBB676f9cFF8D242e9eaC39D063848807d3D1D94",
+    "nitroPrivateKey": "0279651921cd800ac560c21ceea27aab0107b67daf436cdd25ce84cad30159b4"
+  },
+  "consensus": {
+    "publicKey": "029c8035b3e9401b8f178f7c37285b5cb22501e017340e2058b3b842f2a1f0ae45",
+    "privateKey": "0261008e8e3ec808168e99333599da38ca59a056a2ae4510a6ad3d8b5cb0918c"
+  }
+}

stack_orchestrator/data/config/watcher-mobymask-v3-demo/local/keys/watcher-3.json

@@ -0,0 +1,21 @@
+{
+  "peerId": {
+    "id": "12D3KooWSRH6ftgkAZsKZK7UX1Zr6Hx6YAsEepHqzopFszqfTxxi",
+    "privKey": "CAESQHBjlHxfVhZ2gXsBItrIEEgSGKcjMkFiGs3PPz9E3ace9qyWEkvR4oit5ve9SAROVoh20hoa42IC91NIafMaqws=",
+    "pubKey": "CAESIPaslhJL0eKIreb3vUgETlaIdtIaGuNiAvdTSGnzGqsL"
+  },
+  "relayPeerId": {
+    "id": "12D3KooWLudDC3NnS8k9xLiqTZcS9xbFNrGSttu5GPynQgNgenPW",
+    "privKey": "CAESQHuEXPB1hTQATlZaNRQnAfSvnfYUeLF9PjSNKp9L33K2pMl37tTBYghnfQ0yluse0zHOIXRJM+9+ARn9+jIr0gU=",
+    "pubKey": "CAESIKTJd+7UwWIIZ30NMpbrHtMxziF0STPvfgEZ/foyK9IF"
+  },
+  "peer": {
+    "privateKey": "47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a",
+    "nitroAddress": "0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d",
+    "nitroPrivateKey": "58368d20ff12f17669c06158c21d885897aa56f9be430edc789614bf9851d53f"
+  },
+  "consensus": {
+    "publicKey": "039160c244a7ad8be16a64bdb69e6dbacdcfe20b37076792a0d06032a8097468ca",
+    "privateKey": "8894685fe81001d75662b079905472699373967451d1255ee5fc669d0a09a9ca"
+  }
+}

stack_orchestrator/data/config/watcher-mobymask-v3-demo/mobymask-app-start.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+watcher_keys_dir="/server/keys"
+
+# Configure relay nodes to connect to
+# Note: Assuming host ports for watchers (1-3) relay nodes
+RELAY_NODES="[\"/ip4/127.0.0.1/tcp/9090/ws/p2p/$(jq -r '.relayPeerId.id' ${watcher_keys_dir}/watcher-1.json)\", \"/ip4/127.0.0.1/tcp/9091/ws/p2p/$(jq -r '.relayPeerId.id' ${watcher_keys_dir}/watcher-2.json)\", \"/ip4/127.0.0.1/tcp/9092/ws/p2p/$(jq -r '.relayPeerId.id' ${watcher_keys_dir}/watcher-3.json)\"]"
+
+# Configure watcher (1) Nitro account to make payments to
+PAYMENT_NITRO_ADDRESS=$(jq -r '.peer.nitroAddress' ${watcher_keys_dir}/watcher-1.json)
+
+echo "Using RELAY_NODES $RELAY_NODES"
+echo "Using PAYMENT_NITRO_ADDRESS $PAYMENT_NITRO_ADDRESS"
+
+# Use config from mounted volume
+echo "Taking config for deployed contracts from mounted volume"
+while [ ! -f /server/config.json ]; do
+  echo "Config not found, retrying in 5 seconds..."
+  sleep 5
+done
+
+# Get deployed contract address and chain id
+DEPLOYED_CONTRACT=$(jq -r '.address' /server/config.json | tr -d '"')
+CHAIN_ID=$(jq -r '.chainId' /server/config.json)
+
+echo "Using DEPLOYED_CONTRACT $DEPLOYED_CONTRACT"
+echo "Using CHAIN_ID $CHAIN_ID"
+
+nitro_addresses_file="/nitro/nitro-addresses.json"
+nitro_addresses_destination_file="/app/src/utils/nitro-addresses.json"
+if [ -f ${nitro_addresses_file} ]; then
+  echo "Using Nitro addresses from ${nitro_addresses_file}:"
+  cat "$nitro_addresses_file"
+  cat "$nitro_addresses_file" > "$nitro_addresses_destination_file"
+else
+  echo "Nitro addresses not available"
+  exit 1
+fi
+
+# Export config values in a json file
+app_config_file="/app/src/utils/config.json"
+app_config_json=$(jq -n \
+  --arg name "MobyMask" \
+  --argjson enableDebugInfo true \
+  --arg address "$DEPLOYED_CONTRACT" \
+  --argjson chainId "$CHAIN_ID" \
+  --argjson relayNodes "$RELAY_NODES" \
+  --argjson denyMultiaddrs "[]" \
+  --arg pubsub "" \
+  --argjson directPeers "[]" \
+  '.name = $name | .address = $address | .chainId = $chainId | .relayNodes = $relayNodes | .peer.enableDebugInfo = $enableDebugInfo | .peer.denyMultiaddrs = $denyMultiaddrs | .peer.pubsub = $pubsub | .peer.directPeers = $directPeers')
+echo "$app_config_json" > "${app_config_file}"
+
+echo "Using CERC_APP_WATCHER_URL $CERC_APP_WATCHER_URL"
+echo "Using CERC_SNAP_URL $CERC_SNAP_URL"
+
+REACT_APP_DEBUG_PEER=true \
+REACT_APP_WATCHER_URI="$CERC_APP_WATCHER_URL/graphql" \
+REACT_APP_PAY_TO_NITRO_ADDRESS="$PAYMENT_NITRO_ADDRESS" \
+REACT_APP_SNAP_ORIGIN="local:$CERC_SNAP_URL" \
+yarn build
+
+http-server -p 80 /app/build

stack_orchestrator/data/config/watcher-mobymask-v3-demo/start-server.sh

@@ -0,0 +1,119 @@
+#!/bin/bash
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+watcher_keys_dir="./keys"
+
+echo "Watcher index: ${CERC_WATCHER_INDEX}"
+echo "Using RPC query endpoint ${CERC_ETH_RPC_QUERY_ENDPOINT}"
+echo "Using RPC mutation endpoint ${CERC_ETH_RPC_MUTATION_ENDPOINT}"
+echo "Using Nitro chain URL ${CERC_NITRO_CHAIN_URL}"
+
+WATCHER_DB="mobymask-watcher-${CERC_WATCHER_INDEX}"
+
+# Use the docker container's host IP for relay multiaddr in peer config
+CERC_RELAY_MULTIADDR="/dns4/mobymask-watcher-${CERC_WATCHER_INDEX}-server/tcp/9090/ws/p2p/$(jq -r '.relayPeerId.id' ${watcher_keys_dir}/watcher-${CERC_WATCHER_INDEX}.json)"
+
+# Assign deployed contract address from server config (created by mobymask container after deploying contract)
+CONTRACT_ADDRESS=$(jq -r '.address' /server/config.json | tr -d '"')
+
+nitro_addresses_file="/nitro/nitro-addresses.json"
+nitro_addresses_destination_file="./src/nitro-addresses.json"
+if [ -f ${nitro_addresses_file} ]; then
+  echo "Using Nitro addresses from ${nitro_addresses_file}:"
+  cat "$nitro_addresses_file"
+  cat "$nitro_addresses_file" > "$nitro_addresses_destination_file"
+else
+  echo "Nitro addresses not available"
+  exit 1
+fi
+
+# Build after setting the Nitro addresses
+yarn build
+
+PEER_ID_FILE='./peer-id.json'
+RELAY_PEER_ID_FILE='./relay-id.json'
+
+# Create watcher party array
+watcher_parties=()
+
+# Iterate over each fixture JSON file
+for watcher_keys_file in "$watcher_keys_dir"/*.json; do
+  # Extract the filename without the path and extension
+  filename=$(basename "$watcher_keys_file" .json)
+
+  # Read the consensus keys
+  peer_id=$(jq -r '.peerId.id' "$watcher_keys_file")
+
+  # Read the consensus keys
+  consensus_public_key=$(jq -r '.consensus.publicKey' "$watcher_keys_file")
+  consensus_private_key=$(jq -r '.consensus.privateKey' "$watcher_keys_file")
+
+  # Append watcher party
+  watcher_party=$(jq -n \
+    --arg peerId "$peer_id" \
+    --arg publicKey "$consensus_public_key" \
+    '.peerId = $peerId | .publicKey = $publicKey')
+  watcher_parties+=("$watcher_party")
+
+  if [ "$filename" = "watcher-${CERC_WATCHER_INDEX}" ]; then
+    # Export watcher peer and relay ids
+    peer_id_data=$(jq '.peerId' "$watcher_keys_file")
+    relay_peer_id_data=$(jq '.relayPeerId' "$watcher_keys_file")
+    echo "$peer_id_data" > "${PEER_ID_FILE}"
+    echo "$relay_peer_id_data" > "${RELAY_PEER_ID_FILE}"
+
+    # Set peer and nitro account keys
+    CERC_PRIVATE_KEY_PEER=$(jq -r '.peer.privateKey' "$watcher_keys_file")
+    CERC_WATCHER_NITRO_PK=$(jq -r '.peer.nitroPrivateKey' "$watcher_keys_file")
+
+    # Set consensus keys for this peer
+    CONSENSUS_PUBLIC_KEY=${consensus_public_key}
+    CONSENSUS_PRIVATE_KEY=${consensus_private_key}
+  fi
+done
+
+echo "Consensus module enabled"
+CONSENSUS_ENABLED=true
+WATCHER_PARTY_PEERS_FILE='./watcher-party-peers.json'
+
+# Export watcher party file
+watcher_parties_json=$(printf '%s\n' "${watcher_parties[@]}" | jq -s .)
+echo "$watcher_parties_json" > "${WATCHER_PARTY_PEERS_FILE}"
+echo "Watcher party peers exported to ${WATCHER_PARTY_PEERS_FILE}"
+
+# Configure relay peers for watcher 2 and 3
+CERC_RELAY_PEERS=[]
+if [ "${CERC_WATCHER_INDEX}" = "2" ]; then
+  # 2 -> 1
+  CERC_RELAY_PEERS="['/dns4/mobymask-watcher-1-server/tcp/9090/ws/p2p/$(jq -r '.relayPeerId.id' ${watcher_keys_dir}/watcher-1.json)']"
+elif [ "${CERC_WATCHER_INDEX}" = "3" ]; then
+  # 3 -> 1, 2
+  CERC_RELAY_PEERS="['/dns4/mobymask-watcher-1-server/tcp/9090/ws/p2p/$(jq -r '.relayPeerId.id' ${watcher_keys_dir}/watcher-1.json)', '/dns4/mobymask-watcher-2-server/tcp/9090/ws/p2p/$(jq -r '.relayPeerId.id' ${watcher_keys_dir}/watcher-2.json)']"
+fi
+
+# Read in the config template TOML file and modify it
+WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
+  sed -E "s|REPLACE_WITH_CERC_RELAY_PEERS|${CERC_RELAY_PEERS}|g; \
+    s|REPLACE_WITH_CERC_RELAY_MULTIADDR|${CERC_RELAY_MULTIADDR}|g; \
+    s/REPLACE_WITH_CERC_PRIVATE_KEY_PEER/${CERC_PRIVATE_KEY_PEER}/g; \
+    s/REPLACE_WITH_CERC_WATCHER_NITRO_PK/${CERC_WATCHER_NITRO_PK}/g; \
+    s/REPLACE_WITH_CONTRACT_ADDRESS/${CONTRACT_ADDRESS}/g; \
+    s|REPLACE_WITH_CERC_NITRO_CHAIN_URL|${CERC_NITRO_CHAIN_URL}|g; \
+    s/REPLACE_WITH_CONSENSUS_ENABLED/${CONSENSUS_ENABLED}/g; \
+    s/REPLACE_WITH_CONSENSUS_PUBLIC_KEY/${CONSENSUS_PUBLIC_KEY}/g; \
+    s/REPLACE_WITH_CONSENSUS_PRIVATE_KEY/${CONSENSUS_PRIVATE_KEY}/g; \
+    s|REPLACE_WITH_WATCHER_PARTY_PEERS_FILE|${WATCHER_PARTY_PEERS_FILE}|g; \
+    s|REPLACE_WITH_CERC_ETH_RPC_QUERY_ENDPOINT|${CERC_ETH_RPC_QUERY_ENDPOINT}|g; \
+    s|REPLACE_WITH_CERC_ETH_RPC_MUTATION_ENDPOINT|${CERC_ETH_RPC_MUTATION_ENDPOINT}|g; \
+    s|REPLACE_WITH_WATCHER_DB|${WATCHER_DB}| ")
+
+# Write the modified content to a new file
+echo "$WATCHER_CONFIG" > environments/local.toml
+
+echo 'Running watcher server'
+yarn server

stack_orchestrator/data/config/watcher-mobymask-v3-demo/watcher-config-rates.toml

@@ -0,0 +1,14 @@
+freeQueriesLimit = 10
+
+freeQueriesList = []
+
+[queries]
+  multiNonce = '50'
+  _owner = '50'
+  isRevoked = '50'
+  isPhisher = '50'
+  isMember = '50'
+
+[mutations]
+  invoke = '100'
+  revoke = '100'

stack_orchestrator/data/config/watcher-mobymask-v3-demo/watcher-config-template.toml

@@ -0,0 +1,111 @@
+[server]
+  host = "0.0.0.0"
+  port = 3001
+  kind = "lazy"
+
+  # Checkpointing state.
+  checkpointing = false
+
+  # Checkpoint interval in number of blocks.
+  checkpointInterval = 2000
+
+  # Enable state creation
+  enableState = false
+
+  # Boolean to filter logs by contract.
+  filterLogs = true
+
+  # Max block range for which to return events in eventsInRange GQL query.
+  # Use -1 for skipping check on block range.
+  maxEventsBlockRange = -1
+
+  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
+  rpcSupportsBlockHashParam = true
+
+  [server.p2p]
+    enableRelay = true
+    enablePeer = true
+
+    [server.p2p.relay]
+      host = "0.0.0.0"
+      port = 9090
+      relayPeers = REPLACE_WITH_CERC_RELAY_PEERS
+      denyMultiaddrs = []
+      peerIdFile = './relay-id.json'
+      announce = ''
+      pubsub = ''
+      enableDebugInfo = true
+
+    [server.p2p.peer]
+      relayMultiaddr = 'REPLACE_WITH_CERC_RELAY_MULTIADDR'
+      pubSubTopic = 'mobymask'
+      denyMultiaddrs = []
+      peerIdFile = './peer-id.json'
+      pubsub = ''
+      enableDebugInfo = true
+      enableL2Txs = true
+      pingInterval = 4000
+      pingTimeout = 1500
+      maxRelayConnections = 10
+
+      [server.p2p.peer.l2TxsConfig]
+        privateKey = 'REPLACE_WITH_CERC_PRIVATE_KEY_PEER'
+        contractAddress = 'REPLACE_WITH_CONTRACT_ADDRESS'
+
+    [server.p2p.nitro]
+      chainUrl = 'REPLACE_WITH_CERC_NITRO_CHAIN_URL'
+      store = './out/nitro-db'
+      privateKey = 'REPLACE_WITH_CERC_WATCHER_NITRO_PK'
+      chainPrivateKey = 'REPLACE_WITH_CERC_PRIVATE_KEY_PEER'
+
+      [server.p2p.nitro.payments]
+        ratesFile = './environments/rates.toml'
+        requestTimeoutInSecs = 10
+
+        [server.p2p.nitro.payments.cache]
+          maxAccounts = 1000
+          accountTTLInSecs = 1800
+          maxVouchersPerAccount = 1000
+          voucherTTLInSecs = 300
+          maxPaymentChannels = 10000
+          paymentChannelTTLInSecs = 1800
+
+    [server.p2p.consensus]
+      enabled = REPLACE_WITH_CONSENSUS_ENABLED
+      publicKey = 'REPLACE_WITH_CONSENSUS_PUBLIC_KEY'
+      privateKey = 'REPLACE_WITH_CONSENSUS_PRIVATE_KEY'
+      watcherPartyPeersFile = 'REPLACE_WITH_WATCHER_PARTY_PEERS_FILE'
+
+[metrics]
+  host = "0.0.0.0"
+  port = 9000
+  [metrics.gql]
+    port = 9001
+
+[database]
+  type = "postgres"
+  host = "mobymask-watcher-db"
+  port = 5432
+  database = "REPLACE_WITH_WATCHER_DB"
+  username = "vdbm"
+  password = "password"
+  synchronize = true
+  logging = false
+
+[upstream]
+  [upstream.ethServer]
+    gqlApiEndpoint = "http://ipld-eth-server:8083/graphql"
+    rpcProviderEndpoint = 'REPLACE_WITH_CERC_ETH_RPC_QUERY_ENDPOINT'
+    rpcProviderMutationEndpoint = 'REPLACE_WITH_CERC_ETH_RPC_MUTATION_ENDPOINT'
+
+  [upstream.cache]
+    name = "requests"
+    enabled = false
+    deleteOnStart = false
+
+[jobQueue]
+  dbConnectionString = "postgres://vdbm:password@mobymask-watcher-db/REPLACE_WITH_WATCHER_DB-job-queue"
+  maxCompletionLagInSecs = 300
+  jobDelayInMilliSecs = 100
+  eventsInBatch = 50
+  blockDelayInMilliSecs = 60000

stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml

@@ -41,7 +41,7 @@
     timeTravelMaxAge = 86400 # 1 day
 
 [metrics]
-  host = "127.0.0.1"
+  host = "0.0.0.0"
   port = 9000
   [metrics.gql]
     port = 9001
@@ -84,8 +84,6 @@
   subgraphEventsOrder = true
   # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
   blockDelayInMilliSecs = 30000
-  prefetchBlocksInMem = false
-  prefetchBlockCount = 10
 
   # Boolean to switch between modes of processing events when starting the server.
   # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.

stack_orchestrator/data/container-build/cerc-act-runner-task-executor/build.sh

@@ -2,4 +2,6 @@
 # Build a local version of the task executor for act-runner
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-docker build -t cerc/act-runner-task-executor:local -f ${CERC_REPO_BASE_DIR}/hosting/act-runner/Dockerfile.task-executor ${build_command_args} ${SCRIPT_DIR}
+
+cd ${CERC_REPO_BASE_DIR}/hosting/act-runner
+docker build -t cerc/act-runner-task-executor:local -f Dockerfile.task-executor ${build_command_args} .

stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/Dockerfile

@@ -10,9 +10,10 @@ COPY genesis /opt/genesis
 COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/
 COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/
 COPY --from=ethgen /apps /apps
-RUN cd /apps/el-gen && pip3 install -r requirements.txt
+RUN cd /apps/el-gen && pip3 install --break-system-packages -r requirements.txt
 # web3==5.24.0 used by el-gen is broken on python 3.11
-RUN pip3 install --upgrade "web3==6.5.0"
+RUN pip3 install --break-system-packages --upgrade "web3==6.5.0"
+RUN pip3 install --break-system-packages --upgrade "typing-extensions"
 
 # Build genesis config
 RUN apk add --no-cache make bash envsubst jq

stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-10.yml

@@ -0,0 +1,14 @@
+record:
+  type: ApplicationDeploymentRecord
+  version: 1.2.3
+  name: name
+  description: description
+  application: application
+  url: url
+  dns: dns
+  request: request
+  meta:
+    foo: bar
+  tags:
+    - a
+    - b 

stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-11.yml

@@ -0,0 +1,12 @@
+record:
+  type: GeneralRecord
+  version: 1.2.3
+  name: name
+  description: description
+  category: category
+  value: value
+  meta:
+    foo: bar
+  tags:
+    - a
+    - b 

stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-12.yml

@@ -0,0 +1,17 @@
+record:
+  type: ApplicationArtifact
+  version: 1.2.3
+  name: name
+  description: description
+  application: appidgoeshere
+  content_type: content_type
+  os: os
+  cpu: cpu
+  uri:
+    - uri://a
+    - uri://b
+  meta:
+    foo: bar
+  tags:
+    - a
+    - b

stack_orchestrator/data/container-build/cerc-laconic-registry-cli/demo-records/demo-record-7.yml

@@ -0,0 +1,11 @@
+record:
+  type: DnsRecord
+  version: 0.0.1
+  name: "foo"
+  resource_type: "A"
+  value: "bar"
+  meta:
+    foo: bar
+  tags:
+    - a
+    - b