Configure prometheus alert manager with Slack notifications

Part of [Investigate subgraph watchers lagging behind head](https://www.notion.so/Investigate-subgraph-watchers-lagging-behind-head-01b72294ca8e4f658e4c0e86b36d19e2)

Co-authored-by: Nabarun <nabarun@deepstacksoft.com>
Reviewed-on: cerc-io/stack-orchestrator#827
Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com>
Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>

.gitea/workflows/fixturenet-eth-plugeth-arm-test.yml

@@ -0,0 +1,57 @@
+name: Fixturenet-Eth-Plugeth-Arm-Test
+
+on:
+  push:
+    branches: '*'
+    paths:
+      - '!**'
+      - '.gitea/workflows/triggers/fixturenet-eth-plugeth-arm-test'
+  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
+    - cron: '2 14 * * *'
+
+
+jobs:
+  test:
+    name: "Run an Ethereum plugeth fixturenet test"
+    runs-on: ubuntu-latest-arm
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      # At present the stock setup-python action fails on Linux/aarch64
+      # Conditional steps below workaroud this by using deadsnakes for that case only
+      - name: "Install Python for ARM on Linux"
+        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
+        uses: deadsnakes/action@v3.0.1
+        with:
+          python-version: '3.8'
+      - name: "Install Python cases other than ARM on Linux"
+        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: "Run fixturenet-eth tests"
+        run: ./tests/fixturenet-eth-plugeth/run-test.sh
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/fixturenet-eth-plugeth-test.yml

@@ -9,10 +9,6 @@ on:
   schedule: # Note: coordinate with other tests to not overload runners at the same time of day
     - cron: '2 14 * * *'
 
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
 
 jobs:
   test:
@@ -41,9 +37,21 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
       - name: "Run fixturenet-eth tests"
         run: ./tests/fixturenet-eth-plugeth/run-test.sh
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/fixturenet-eth-test.yml

@@ -7,10 +7,6 @@ on:
       - '!**'
       - '.gitea/workflows/triggers/fixturenet-eth-test'
 
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
 
 jobs:
   test:
@@ -39,10 +35,21 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
       - name: "Run fixturenet-eth tests"
         run: ./tests/fixturenet-eth/run-test.sh
-
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/fixturenet-laconicd-test.yml

@@ -11,7 +11,7 @@ on:
 
 jobs:
   test:
-    name: "Run an Laconicd fixturenet test"
+    name: "Run Laconicd fixturenet and Laconic CLI tests"
     runs-on: ubuntu-latest
     steps:
       - name: 'Update'
@@ -46,3 +46,21 @@ jobs:
         run: ./scripts/build_shiv_package.sh
       - name: "Run fixturenet-laconicd tests"
         run: ./tests/fixturenet-laconicd/run-test.sh
+      - name: "Run laconic CLI tests"
+        run: ./tests/fixturenet-laconicd/run-cli-test.sh
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/lint.yml

@@ -19,3 +19,19 @@ jobs:
           python-version: '3.8'
       - name : "Run flake8"
         uses: py-actions/flake8@v2
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/publish.yml

@@ -54,3 +54,19 @@ jobs:
           # Hack using endsWith to workaround Gitea sometimes sending "publish-test" vs "refs/heads/publish-test"
           draft: ${{ endsWith('publish-test', github.ref ) }}
           files: ./laconic-so
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/test-container-registry.yml

@@ -51,4 +51,19 @@ jobs:
           source /opt/bash-utils/cgroup-helper.sh
           join_cgroup
           ./tests/container-registry/run-test.sh
-
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/test-database.yml

@@ -49,4 +49,19 @@ jobs:
           source /opt/bash-utils/cgroup-helper.sh
           join_cgroup
           ./tests/database/run-test.sh
-
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/test-deploy.yml

@@ -10,9 +10,6 @@ on:
     paths-ignore:
       - '.gitea/workflows/triggers/*'
 
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
 
 jobs:
   test:
@@ -41,9 +38,21 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
       - name: "Run deploy tests"
         run: ./tests/deploy/run-deploy-test.sh
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/test-external-stack.yml

@@ -0,0 +1,58 @@
+name: External Stack Test
+
+on:
+  push:
+    branches: '*'
+    paths:
+      - '!**'
+      - '.gitea/workflows/triggers/test-external-stack'
+      - '.gitea/workflows/test-external-stack.yml'
+      - 'tests/external-stack/run-test.sh'
+  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
+    - cron: '8 19 * * *'
+
+jobs:
+  test:
+    name: "Run external stack test suite"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      # At present the stock setup-python action fails on Linux/aarch64
+      # Conditional steps below workaroud this by using deadsnakes for that case only
+      - name: "Install Python for ARM on Linux"
+        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
+        uses: deadsnakes/action@v3.0.1
+        with:
+          python-version: '3.8'
+      - name: "Install Python cases other than ARM on Linux"
+        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: "Run external stack tests"
+        run: ./tests/external-stack/run-test.sh
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/test-k8s-deploy.yml

@@ -51,4 +51,19 @@ jobs:
           source /opt/bash-utils/cgroup-helper.sh
           join_cgroup
           ./tests/k8s-deploy/run-deploy-test.sh
-
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/test-webapp.yml

@@ -10,10 +10,6 @@ on:
     paths-ignore:
       - '.gitea/workflows/triggers/*'
 
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
 jobs:
   test:
     name: "Run webapp test suite"
@@ -43,9 +39,21 @@ jobs:
         run: ./scripts/build_shiv_package.sh
       - name: "Install wget" # 20240109 - Only needed until the executors are updated.
         run: apt update && apt install -y wget
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
       - name: "Run webapp tests"
         run: ./tests/webapp-test/run-webapp-test.sh
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/test.yml

@@ -10,9 +10,6 @@ on:
     paths-ignore:
       - '.gitea/workflows/triggers/*'
 
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
 
 jobs:
   test:
@@ -41,11 +38,21 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
       - name: "Run smoke tests"
         run: ./tests/smoke-test/run-smoke-test.sh
-
+      - name: Notify Vulcanize Slack on CI failure
-
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/triggers/fixturenet-eth-plugeth-arm-test

@@ -0,0 +1,2 @@
+Change this file to trigger running the fixturenet-eth-plugeth-arm-test CI job
+

.gitea/workflows/triggers/fixturenet-laconicd-test

@@ -1,3 +1,6 @@
 Change this file to trigger running the fixturenet-laconicd-test CI job
 Trigger
 Trigger
+Trigger
+Trigger
+Trigger

.gitea/workflows/triggers/test-external-stack

@@ -0,0 +1,2 @@
+Change this file to trigger running the external-stack CI job
+trigger

docs/fetching-containers.md

@@ -0,0 +1,9 @@
+# Fetching pre-built container images
+When Stack Orchestrator deploys a stack containing a suite of one or more containers it expects images for those containers to be on the local machine with a tag of the form `<image-name>:local` Images for these containers can be built from source (and optionally base container images from public registries) with the `build-containers` subcommand. 
+
+However, the task of building a large number of containers from source may consume considerable time and machine resources. This is where the `fetch-containers` subcommand steps in. It is designed to work exactly like `build-containers` but instead the images, pre-built, are fetched from an image registry then re-tagged for deployment. It can be used in place of `build-containers` for any stack provided the necessary containers, built for the local machine architecture (e.g. arm64 or x86-64) have already been published in an image registry.
+## Usage
+To use `fetch-containers`, provide an image registry path, a username and token/password with read access to the registry, and optionally specify `--force-local-overwrite`. If this argument is not specified, if there is already a locally built or previously fetched image for a stack container on the machine, it will not be overwritten and a warning issued.
+```
+$ laconic-so --stack mobymask-v3-demo fetch-containers --image-registry git.vdb.to/cerc-io --registry-username <registry-user> --registry-token <registry-token> --force-local-overwrite
+```

docs/laconicd-with-console.md

@@ -15,132 +15,139 @@ To avoid hiccups on Mac M1/M2 and any local machine nuances that may affect the
 16 GB Memory / 8 Intel vCPUs / 160 GB Disk.
 
 1. Login to the droplet as root (either by SSH key or password set in the DO console)
+    ```
+    ssh root@IP
+    ```
 
-```
+1. Get the install script, give it executable permissions, and run it:
-ssh root@IP
-```
 
-2. Get the install script, give it executable permissions, and run it:
+    ```
+    curl -o install.sh https://raw.githubusercontent.com/cerc-io/stack-orchestrator/main/scripts/quick-install-linux.sh
+    ```
+    ```
+    chmod +x install.sh
+    ```
+    ```
+    bash install.sh
+    ```
 
-```
+1. Confirm docker was installed and activate the changes in `~/.profile`:
-curl -o install.sh https://raw.githubusercontent.com/cerc-io/stack-orchestrator/main/scripts/quick-install-linux.sh
-```
-```
-chmod +x install.sh
-```
-```
-bash install.sh
-```
 
-3. Confirm docker was installed and activate the changes in `~/.profile`:
+    ```
+    docker run hello-world
+   ```
+   ```
+   source ~/.profile
+   ```
 
-```
+1. Verify installation:
-docker run hello-world
-```
-```
-source ~/.profile
-```
 
-4. Verify installation:
+    ```
-
+    laconic-so version
-```
+    ```
-laconic-so version
-```
 
 ## Setup the laconic fixturenet stack
 
 1. Get the repositories
 
-```
+    ```
-laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd,git.vdb.to/cerc-io/laconic-sdk,git.vdb.to/cerc-io/laconic-registry-cli,git.vdb.to/cerc-io/laconic-console
+    laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd,git.vdb.to/cerc-io/laconic-sdk,git.vdb.to/cerc-io/laconic-registry-cli,git.vdb.to/cerc-io/laconic-console
-```
+    ```
 
-2. Set this environment variable to the Laconic self-hosted Gitea instance:
+1. Build the containers:
 
-```
+    ```
-export CERC_NPM_REGISTRY_URL=https://git.vdb.to/api/packages/cerc-io/npm/
+    laconic-so --stack fixturenet-laconic-loaded build-containers
-```
+    ```
 
-3. Build the containers:
+    It's possible to run into an `ESOCKETTIMEDOUT` error, e.g., `error An unexpected error occurred: "https://registry.yarnpkg.com/@material-ui/icons/-/icons-4.11.3.tgz: ESOCKETTIMEDOUT"`. This may happen even if you have a great internet connection. In that case, re-run the `build-containers` command.
 
-```
-laconic-so --stack fixturenet-laconic-loaded build-containers
-```
 
-It's possible to run into an `ESOCKETTIMEDOUT` error, e.g., `error An unexpected error occurred: "https://registry.yarnpkg.com/@material-ui/icons/-/icons-4.11.3.tgz: ESOCKETTIMEDOUT"`. This may happen even if you have a great internet connection. In that case, re-run the `build-containers` command.
+1. Set this environment variable to your droplet's IP address or fully qualified DNS host name if it has one:
 
-4. Set this environment variable to your droplet's IP address:
+    ```
+    export BACKEND_ENDPOINT=http://<your-IP-or-hostname>:9473
+    ```
+    e.g.
+    ```
+    export BACKEND_ENDPOINT=http://my-test-server.example.com:9473
+    ```
 
-```
+1. Create a deployment directory for the stack:
-export LACONIC_HOSTED_ENDPOINT=http://<your-IP>
+    ```
-```
+    laconic-so --stack fixturenet-laconic-loaded deploy init --output laconic-loaded.spec --map-ports-to-host any-same --config LACONIC_HOSTED_ENDPOINT=$BACKEND_ENDPOINT
+    ```
+    ```
+    laconic-so --stack fixturenet-laconic-loaded deploy create --deployment-dir laconic-loaded-deployment --spec-file laconic-loaded.spec
+    ```
+2. Start the stack:
 
-5. Deploy the stack:
+    ```
+    laconic-so deployment --dir laconic-loaded-deployment start
+    ```
 
-```
+3. Check the logs:
-laconic-so --stack fixturenet-laconic-loaded deploy up
-```
 
-6. Check the logs:
+    ```
+    laconic-so deployment --dir laconic-loaded-deployment logs
+    ```
 
-```
+    You'll see output from `laconicd` and the block height should be >1 to confirm it is running:
-laconic-so --stack fixturenet-laconic-loaded deploy logs
-```
 
-You'll see output from `laconicd` and the block height should be >1 to confirm it is running:
+    ```
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:29PM INF indexed block exents height=12 module=txindex server=node
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF Timed out dur=4976.960115 height=13 module=consensus round=0 server=node step=1
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF received proposal module=consensus proposal={"Type":32,"block_id":{"hash":"D26C088A711F912ADB97888C269F628DA33153795621967BE44DCB43C3D03CA4","parts":{"hash":"22411A20B7F14CDA33244420FBDDAF24450C0628C7A06034FF22DAC3699DDCC8","total":1}},"height":13,"pol_round":-1,"round":0,"signature":"DEuqnaQmvyYbUwckttJmgKdpRu6eVm9i+9rQ1pIrV2PidkMNdWRZBLdmNghkIrUzGbW8Xd7UVJxtLRmwRASgBg==","timestamp":"2023-04-18T21:30:01.49450663Z"} server=node
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF received complete proposal block hash=D26C088A711F912ADB97888C269F628DA33153795621967BE44DCB43C3D03CA4 height=13 module=consensus server=node
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF finalizing commit of block hash={} height=13 module=consensus num_txs=0 root=1A8CA1AF139CCC80EC007C6321D8A63A46A793386EE2EDF9A5CA0AB2C90728B7 server=node
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF minted coins from module account amount=2059730459416582643aphoton from=mint module=x/bank
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF executed block height=13 module=state num_invalid_txs=0 num_valid_txs=0 server=node
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF commit synced commit=436F6D6D697449447B5B363520313037203630203232372039352038352032303820313334203231392032303520313433203130372031343920313431203139203139322038362031323720362031383520323533203137362031333820313735203135392031383620323334203135382031323120313431203230342037335D3A447D
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF committed state app_hash=416B3CE35F55D086DBCD8F6B958D13C0567F06B9FDB08AAF9FBAEA9E798DCC49 height=13 module=state num_txs=0 server=node
+    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF indexed block exents height=13 module=txindex server=node
+    ```
 
-```
+4. Confirm operation of the registry CLI:
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:29PM INF indexed block exents height=12 module=txindex server=node
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF Timed out dur=4976.960115 height=13 module=consensus round=0 server=node step=1
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF received proposal module=consensus proposal={"Type":32,"block_id":{"hash":"D26C088A711F912ADB97888C269F628DA33153795621967BE44DCB43C3D03CA4","parts":{"hash":"22411A20B7F14CDA33244420FBDDAF24450C0628C7A06034FF22DAC3699DDCC8","total":1}},"height":13,"pol_round":-1,"round":0,"signature":"DEuqnaQmvyYbUwckttJmgKdpRu6eVm9i+9rQ1pIrV2PidkMNdWRZBLdmNghkIrUzGbW8Xd7UVJxtLRmwRASgBg==","timestamp":"2023-04-18T21:30:01.49450663Z"} server=node
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF received complete proposal block hash=D26C088A711F912ADB97888C269F628DA33153795621967BE44DCB43C3D03CA4 height=13 module=consensus server=node
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF finalizing commit of block hash={} height=13 module=consensus num_txs=0 root=1A8CA1AF139CCC80EC007C6321D8A63A46A793386EE2EDF9A5CA0AB2C90728B7 server=node
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF minted coins from module account amount=2059730459416582643aphoton from=mint module=x/bank
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF executed block height=13 module=state num_invalid_txs=0 num_valid_txs=0 server=node
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF commit synced commit=436F6D6D697449447B5B363520313037203630203232372039352038352032303820313334203231392032303520313433203130372031343920313431203139203139322038362031323720362031383520323533203137362031333820313735203135392031383620323334203135382031323120313431203230342037335D3A447D
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF committed state app_hash=416B3CE35F55D086DBCD8F6B958D13C0567F06B9FDB08AAF9FBAEA9E798DCC49 height=13 module=state num_txs=0 server=node
-laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF indexed block exents height=13 module=txindex server=node
-```
 
-7. Confirm operation of the registry CLI:
+   ```
+   laconic-so deployment --dir laconic-loaded-deployment exec cli "laconic cns status"
+   ```
 
-```
+   ```
-laconic-so --stack fixturenet-laconic-loaded deploy exec cli "laconic cns status"
+   {
-```
+     "version": "0.3.0",
-
+     "node": {
-```
+       "id": "4216af2ac9f68bda33a38803fc1b5c9559312c1d",
-{
+       "network": "laconic_9000-1",
-  "version": "0.3.0",
+       "moniker": "localtestnet"
-  "node": {
+     },
-    "id": "4216af2ac9f68bda33a38803fc1b5c9559312c1d",
+     "sync": {
-    "network": "laconic_9000-1",
+       "latest_block_hash": "1BDF4CB9AE2390DA65BCF997C83133C18014FCDDCAE03708488F0B56FCEEA429",
-    "moniker": "localtestnet"
+       "latest_block_height": "5",
-  },
+       "latest_block_time": "2023-08-09 16:00:30.386903172 +0000 UTC",
-  "sync": {
+       "catching_up": false
-    "latest_block_hash": "1BDF4CB9AE2390DA65BCF997C83133C18014FCDDCAE03708488F0B56FCEEA429",
+     },
-    "latest_block_height": "5",
+     "validator": {
-    "latest_block_time": "2023-08-09 16:00:30.386903172 +0000 UTC",
+       "address": "651FBC700B747C76E90ACFC18CC9508C3D0905B9",
-    "catching_up": false
+       "voting_power": "1000000000000000"
-  },
+     },
-  "validator": {
+     "validators": [
-    "address": "651FBC700B747C76E90ACFC18CC9508C3D0905B9",
+       {
-    "voting_power": "1000000000000000"
+         "address": "651FBC700B747C76E90ACFC18CC9508C3D0905B9",
-  },
+         "voting_power": "1000000000000000",
-  "validators": [
+         "proposer_priority": "0"
-    {
+       }
-      "address": "651FBC700B747C76E90ACFC18CC9508C3D0905B9",
+     ],
-      "voting_power": "1000000000000000",
+     "num_peers": "0",
-      "proposer_priority": "0"
+     "peers": [],
-    }
+     "disk_usage": "292.0K"
-  ],
+   }
-  "num_peers": "0",
+   ```
-  "peers": [],
-  "disk_usage": "292.0K"
-}
-```
 
 ## Configure Digital Ocean firewall
 
+(Note this step may not be necessary depending on the droplet image used)
+
 Let's open some ports.
 
 1. In the Digital Ocean web console, navigate to your droplet's main page. Select the "Networking" tab and scroll down to "Firewall".

stack_orchestrator/build/build_containers.py

@@ -25,10 +25,13 @@ import sys
 from decouple import config
 import subprocess
 import click
-import importlib.resources
 from pathlib import Path
-from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external, warn_exit
+from stack_orchestrator.opts import opts
+from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit
 from stack_orchestrator.base import get_npm_registry_url
+from stack_orchestrator.build.build_types import BuildContext
+from stack_orchestrator.build.publish import publish_image
+from stack_orchestrator.build.build_util import get_containers_in_scope
 
 # TODO: find a place for this
 #    epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
@@ -59,69 +62,58 @@ def make_container_build_env(dev_root_path: str,
     return container_build_env
 
 
-def process_container(stack: str,
+def process_container(build_context: BuildContext) -> bool:
-                      container,
+    if not opts.o.quiet:
-                      container_build_dir: str,
+        print(f"Building: {build_context.container}")
-                      container_build_env: dict,
-                      dev_root_path: str,
-                      quiet: bool,
-                      verbose: bool,
-                      dry_run: bool,
-                      continue_on_error: bool,
-                      ):
-    if not quiet:
-        print(f"Building: {container}")
 
-    default_container_tag = f"{container}:local"
+    default_container_tag = f"{build_context.container}:local"
-    container_build_env.update({"CERC_DEFAULT_CONTAINER_IMAGE_TAG": default_container_tag})
+    build_context.container_build_env.update({"CERC_DEFAULT_CONTAINER_IMAGE_TAG": default_container_tag})
 
     # Check if this is in an external stack
-    if stack_is_external(stack):
+    if stack_is_external(build_context.stack):
-        container_parent_dir = Path(stack).joinpath("container-build")
+        container_parent_dir = Path(build_context.stack).parent.parent.joinpath("container-build")
-        temp_build_dir = container_parent_dir.joinpath(container.replace("/", "-"))
+        temp_build_dir = container_parent_dir.joinpath(build_context.container.replace("/", "-"))
         temp_build_script_filename = temp_build_dir.joinpath("build.sh")
         # Now check if the container exists in the external stack.
         if not temp_build_script_filename.exists():
             # If not, revert to building an internal container
-            container_parent_dir = container_build_dir
+            container_parent_dir = build_context.container_build_dir
     else:
-        container_parent_dir = container_build_dir
+        container_parent_dir = build_context.container_build_dir
 
-    build_dir = container_parent_dir.joinpath(container.replace("/", "-"))
+    build_dir = container_parent_dir.joinpath(build_context.container.replace("/", "-"))
     build_script_filename = build_dir.joinpath("build.sh")
 
-    if verbose:
+    if opts.o.verbose:
         print(f"Build script filename: {build_script_filename}")
     if os.path.exists(build_script_filename):
         build_command = build_script_filename.as_posix()
     else:
-        if verbose:
+        if opts.o.verbose:
             print(f"No script file found: {build_script_filename}, using default build script")
-        repo_dir = container.split('/')[1]
+        repo_dir = build_context.container.split('/')[1]
         # TODO: make this less of a hack -- should be specified in some metadata somewhere
         # Check if we have a repo for this container. If not, set the context dir to the container-build subdir
-        repo_full_path = os.path.join(dev_root_path, repo_dir)
+        repo_full_path = os.path.join(build_context.dev_root_path, repo_dir)
         repo_dir_or_build_dir = repo_full_path if os.path.exists(repo_full_path) else build_dir
-        build_command = os.path.join(container_build_dir,
+        build_command = os.path.join(build_context.container_build_dir,
                                      "default-build.sh") + f" {default_container_tag} {repo_dir_or_build_dir}"
-    if not dry_run:
+    if not opts.o.dry_run:
         # No PATH at all causes failures with podman.
-        if "PATH" not in container_build_env:
+        if "PATH" not in build_context.container_build_env:
-            container_build_env["PATH"] = os.environ["PATH"]
+            build_context.container_build_env["PATH"] = os.environ["PATH"]
-        if verbose:
+        if opts.o.verbose:
-            print(f"Executing: {build_command} with environment: {container_build_env}")
+            print(f"Executing: {build_command} with environment: {build_context.container_build_env}")
-        build_result = subprocess.run(build_command, shell=True, env=container_build_env)
+        build_result = subprocess.run(build_command, shell=True, env=build_context.container_build_env)
-        if verbose:
+        if opts.o.verbose:
             print(f"Return code is: {build_result.returncode}")
         if build_result.returncode != 0:
-            print(f"Error running build for {container}")
+            return False
-            if not continue_on_error:
+        else:
-                print("FATAL Error: container build failed and --continue-on-error not set, exiting")
+            return True
-                sys.exit(1)
-            else:
-                print("****** Container Build Error, continuing because --continue-on-error is set")
     else:
         print("Skipped")
+        return True
 
 
 @click.command()
@@ -129,17 +121,14 @@ def process_container(stack: str,
 @click.option('--exclude', help="don\'t build these containers")
 @click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild")
 @click.option("--extra-build-args", help="Supply extra arguments to build")
+@click.option("--publish-images", is_flag=True, default=False, help="Publish the built images in the specified image registry")
+@click.option("--image-registry", help="Specify the image registry for --publish-images")
 @click.pass_context
-def command(ctx, include, exclude, force_rebuild, extra_build_args):
+def command(ctx, include, exclude, force_rebuild, extra_build_args, publish_images, image_registry):
     '''build the set of containers required for a complete stack'''
 
-    quiet = ctx.obj.quiet
-    verbose = ctx.obj.verbose
-    dry_run = ctx.obj.dry_run
-    debug = ctx.obj.debug
     local_stack = ctx.obj.local_stack
     stack = ctx.obj.stack
-    continue_on_error = ctx.obj.continue_on_error
 
     # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
     container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
@@ -150,41 +139,45 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
     else:
         dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
 
-    if not quiet:
+    if not opts.o.quiet:
         print(f'Dev Root is: {dev_root_path}')
 
     if not os.path.isdir(dev_root_path):
         print('Dev root directory doesn\'t exist, creating')
 
-    # See: https://stackoverflow.com/a/20885799/1701505
+    if publish_images:
-    from stack_orchestrator import data
+        if not image_registry:
-    with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
+            error_exit("--image-registry must be supplied with --publish-images")
-        all_containers = container_list_file.read().splitlines()
 
-    containers_in_scope = []
+    containers_in_scope = get_containers_in_scope(stack)
-    if stack:
-        stack_config = get_parsed_stack_config(stack)
-        if "containers" not in stack_config or stack_config["containers"] is None:
-            warn_exit(f"stack {stack} does not define any containers")
-        containers_in_scope = stack_config['containers']
-    else:
-        containers_in_scope = all_containers
-
-    if verbose:
-        print(f'Containers: {containers_in_scope}')
-        if stack:
-            print(f"Stack: {stack}")
 
     container_build_env = make_container_build_env(dev_root_path,
                                                    container_build_dir,
-                                                   debug,
+                                                   opts.o.debug,
                                                    force_rebuild,
                                                    extra_build_args)
 
     for container in containers_in_scope:
         if include_exclude_check(container, include, exclude):
-            process_container(stack, container, container_build_dir, container_build_env,
+
-                              dev_root_path, quiet, verbose, dry_run, continue_on_error)
+            build_context = BuildContext(
+                stack,
+                container,
+                container_build_dir,
+                container_build_env,
+                dev_root_path
+            )
+            result = process_container(build_context)
+            if result:
+                if publish_images:
+                    publish_image(f"{container}:local", image_registry)
+            else:
+                print(f"Error running build for {build_context.container}")
+                if not opts.o.continue_on_error:
+                    error_exit("container build failed and --continue-on-error not set, exiting")
+                    sys.exit(1)
+                else:
+                    print("****** Container Build Error, continuing because --continue-on-error is set")
         else:
-            if verbose:
+            if opts.o.verbose:
                 print(f"Excluding: {container}")

stack_orchestrator/build/build_types.py

@@ -0,0 +1,29 @@
+# Copyright © 2024 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Mapping
+
+
+@dataclass
+class BuildContext:
+    stack: str
+    container: str
+    container_build_dir: Path
+    container_build_env: Mapping[str,str]
+    dev_root_path: str
+

stack_orchestrator/build/build_util.py

@@ -0,0 +1,43 @@
+# Copyright © 2024 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+import importlib.resources
+
+from stack_orchestrator.opts import opts
+from stack_orchestrator.util import get_parsed_stack_config, warn_exit
+
+
+def get_containers_in_scope(stack: str):
+
+    # See: https://stackoverflow.com/a/20885799/1701505
+    from stack_orchestrator import data
+    with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
+        all_containers = container_list_file.read().splitlines()
+
+    containers_in_scope = []
+    if stack:
+        stack_config = get_parsed_stack_config(stack)
+        if "containers" not in stack_config or stack_config["containers"] is None:
+            warn_exit(f"stack {stack} does not define any containers")
+        containers_in_scope = stack_config['containers']
+    else:
+        containers_in_scope = all_containers
+
+    if opts.o.verbose:
+        print(f'Containers: {containers_in_scope}')
+        if stack:
+            print(f"Stack: {stack}")
+
+    return containers_in_scope

stack_orchestrator/build/build_webapp.py

@@ -21,11 +21,14 @@
 # TODO: display the available list of containers; allow re-build of either all or specific containers
 
 import os
+import sys
+
 from decouple import config
 import click
 from pathlib import Path
 from stack_orchestrator.build import build_containers
-from stack_orchestrator.deploy.webapp.util import determine_base_container
+from stack_orchestrator.deploy.webapp.util import determine_base_container, TimedLogger
+from stack_orchestrator.build.build_types import BuildContext
 
 
 @click.command()
@@ -37,26 +40,25 @@ from stack_orchestrator.deploy.webapp.util import determine_base_container
 @click.pass_context
 def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, tag):
     '''build the specified webapp container'''
+    logger = TimedLogger()
 
     quiet = ctx.obj.quiet
-    verbose = ctx.obj.verbose
-    dry_run = ctx.obj.dry_run
     debug = ctx.obj.debug
+    verbose = ctx.obj.verbose
     local_stack = ctx.obj.local_stack
     stack = ctx.obj.stack
-    continue_on_error = ctx.obj.continue_on_error
 
     # See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
     container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
 
     if local_stack:
         dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
-        print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
+        logger.log(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
     else:
         dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
 
-    if not quiet:
+    if verbose:
-        print(f'Dev Root is: {dev_root_path}')
+        logger.log(f'Dev Root is: {dev_root_path}')
 
     if not base_container:
         base_container = determine_base_container(source_repo)
@@ -65,8 +67,23 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
     container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug,
                                                                     force_rebuild, extra_build_args)
 
-    build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet,
+    if verbose:
-                                       verbose, dry_run, continue_on_error)
+        logger.log(f"Building base container: {base_container}")
+
+    build_context_1 = BuildContext(
+        stack,
+        base_container,
+        container_build_dir,
+        container_build_env,
+        dev_root_path,
+    )
+    ok = build_containers.process_container(build_context_1)
+    if not ok:
+        logger.log("ERROR: Build failed.")
+        sys.exit(1)
+
+    if verbose:
+        logger.log(f"Base container {base_container} build finished.")
 
     # Now build the target webapp.  We use the same build script, but with a different Dockerfile and work dir.
     container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true"
@@ -76,9 +93,25 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
                                                                           "Dockerfile.webapp")
     if not tag:
         webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1]
-        container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local"
+        tag = f"cerc/{webapp_name}:local"
-    else:
-        container_build_env["CERC_CONTAINER_BUILD_TAG"] = tag
 
-    build_containers.process_container(None, base_container, container_build_dir, container_build_env, dev_root_path, quiet,
+    container_build_env["CERC_CONTAINER_BUILD_TAG"] = tag
-                                       verbose, dry_run, continue_on_error)
+
+    if verbose:
+        logger.log(f"Building app container: {tag}")
+
+    build_context_2 = BuildContext(
+        stack,
+        base_container,
+        container_build_dir,
+        container_build_env,
+        dev_root_path,
+    )
+    ok = build_containers.process_container(build_context_2)
+    if not ok:
+        logger.log("ERROR: Build failed.")
+        sys.exit(1)
+
+    if verbose:
+        logger.log(f"App container {base_container} build finished.")
+        logger.log("build-webapp complete", show_step_time=False, show_total_time=True)

stack_orchestrator/build/fetch_containers.py

@@ -0,0 +1,195 @@
+# Copyright © 2024 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+import click
+from dataclasses import dataclass
+import json
+import platform
+from python_on_whales import DockerClient
+from python_on_whales.components.manifest.cli_wrapper import ManifestCLI, ManifestList
+from python_on_whales.utils import run
+import requests
+from typing import List
+
+from stack_orchestrator.opts import opts
+from stack_orchestrator.util import include_exclude_check, error_exit
+from stack_orchestrator.build.build_util import get_containers_in_scope
+
+# Experimental fetch-container command
+
+
+@dataclass
+class RegistryInfo:
+    registry: str
+    registry_username: str
+    registry_token: str
+
+
+# Extending this code to support the --verbose option, cnosider contributing upstream
+# https://github.com/gabrieldemarmiesse/python-on-whales/blob/master/python_on_whales/components/manifest/cli_wrapper.py#L129
+class ExtendedManifestCLI(ManifestCLI):
+    def inspect_verbose(self, x: str) -> ManifestList:
+        """Returns a Docker manifest list object."""
+        json_str = run(self.docker_cmd + ["manifest", "inspect", "--verbose", x])
+        return json.loads(json_str)
+
+
+def _local_tag_for(container: str):
+    return f"{container}:local"
+
+
+# See: https://docker-docs.uclv.cu/registry/spec/api/
+# Emulate this:
+# $ curl -u "my-username:my-token" -X GET "https://<container-registry-hostname>/v2/cerc-io/cerc/test-container/tags/list"
+# {"name":"cerc-io/cerc/test-container","tags":["202402232130","202402232208"]}
+def _get_tags_for_container(container: str, registry_info: RegistryInfo) -> List[str]:
+    # registry looks like: git.vdb.to/cerc-io
+    registry_parts = registry_info.registry.split("/")
+    url = f"https://{registry_parts[0]}/v2/{registry_parts[1]}/{container}/tags/list"
+    if opts.o.debug:
+        print(f"Fetching tags from: {url}")
+    response = requests.get(url, auth=(registry_info.registry_username, registry_info.registry_token))
+    if response.status_code == 200:
+        tag_info = response.json()
+        if opts.o.debug:
+            print(f"container tags list: {tag_info}")
+        tags_array = tag_info["tags"]
+        return tags_array
+    else:
+        error_exit(f"failed to fetch tags from image registry, status code: {response.status_code}")
+
+
+def _find_latest(candidate_tags: List[str]):
+    # Lex sort should give us the latest first
+    sorted_candidates = sorted(candidate_tags)
+    if opts.o.debug:
+        print(f"sorted candidates: {sorted_candidates}")
+    return sorted_candidates[-1]
+
+
+def _filter_for_platform(container: str, 
+                         registry_info: RegistryInfo,
+                         tag_list: List[str]) -> List[str] :
+    filtered_tags = []
+    this_machine = platform.machine()
+    # Translate between Python and docker platform names
+    if this_machine == "x86_64":
+        this_machine = "amd64"
+    if this_machine == "aarch64":
+        this_machine = "arm64"
+    if opts.o.debug:
+        print(f"Python says the architecture is: {this_machine}")
+    docker = DockerClient()
+    for tag in tag_list:
+        remote_tag = f"{registry_info.registry}/{container}:{tag}"
+        manifest_cmd = ExtendedManifestCLI(docker.client_config)
+        manifest = manifest_cmd.inspect_verbose(remote_tag)
+        if opts.o.debug:
+            print(f"manifest: {manifest}")
+        image_architecture =  manifest["Descriptor"]["platform"]["architecture"]
+        if opts.o.debug:
+            print(f"image_architecture: {image_architecture}")
+        if this_machine == image_architecture:
+            filtered_tags.append(tag)
+    if opts.o.debug:
+        print(f"Tags filtered for platform: {filtered_tags}")
+    return filtered_tags
+
+
+def _get_latest_image(container: str, registry_info: RegistryInfo):
+    all_tags = _get_tags_for_container(container, registry_info)
+    tags_for_platform = _filter_for_platform(container, registry_info, all_tags)
+    if len(tags_for_platform) > 0:
+        latest_tag = _find_latest(tags_for_platform)
+        return f"{container}:{latest_tag}"
+    else:
+        return None
+
+
+def _fetch_image(tag: str, registry_info: RegistryInfo):
+    docker = DockerClient()
+    remote_tag = f"{registry_info.registry}/{tag}"
+    if opts.o.debug:
+        print(f"Attempting to pull this image: {remote_tag}")
+    docker.image.pull(remote_tag)
+
+
+def _exists_locally(container: str):
+    docker = DockerClient()
+    return docker.image.exists(_local_tag_for(container))
+
+
+def _add_local_tag(remote_tag: str, registry: str, local_tag: str):
+    docker = DockerClient()
+    docker.image.tag(f"{registry}/{remote_tag}", local_tag)
+
+
+@click.command()
+@click.option('--include', help="only fetch these containers")
+@click.option('--exclude', help="don\'t fetch these containers")
+@click.option("--force-local-overwrite", is_flag=True, default=False, help="Overwrite a locally built image, if present")
+@click.option("--image-registry", required=True, help="Specify the image registry to fetch from")
+@click.option("--registry-username", required=True, help="Specify the image registry username")
+@click.option("--registry-token", required=True, help="Specify the image registry access token")
+@click.pass_context
+def command(ctx, include, exclude, force_local_overwrite, image_registry, registry_username, registry_token):
+    '''EXPERIMENTAL: fetch the images for a stack from remote registry'''
+
+    registry_info = RegistryInfo(image_registry, registry_username, registry_token)
+    docker = DockerClient()
+    if not opts.o.quiet:
+        print("Logging into container registry:")
+    docker.login(registry_info.registry, registry_info.registry_username, registry_info.registry_token)
+    # Generate list of target containers
+    stack = ctx.obj.stack
+    containers_in_scope = get_containers_in_scope(stack)
+    all_containers_found = True
+    for container in containers_in_scope:
+        local_tag = _local_tag_for(container)
+        if include_exclude_check(container, include, exclude):
+            if opts.o.debug:
+                print(f"Processing: {container}")
+            # For each container, attempt to find the latest of a set of
+            # images with the correct name and platform in the specified registry
+            image_to_fetch = _get_latest_image(container, registry_info)
+            if not image_to_fetch:
+                print(f"Warning: no image found to fetch for container: {container}")
+                all_containers_found = False
+                continue
+            if opts.o.debug:
+                print(f"Fetching: {image_to_fetch}")
+            _fetch_image(image_to_fetch, registry_info)
+            # Now check if the target container already exists exists locally already
+            if (_exists_locally(container)):
+                if not opts.o.quiet:
+                    print(f"Container image {container} already exists locally")
+                # if so, fail unless the user specified force-local-overwrite
+                if (force_local_overwrite):
+                    # In that case remove the existing :local tag
+                    if not opts.o.quiet:
+                        print(f"Warning: overwriting local tag from this image: {container} because "
+                              "--force-local-overwrite was specified")
+                else:
+                    if not opts.o.quiet:
+                        print(f"Skipping local tagging for this image: {container} because that would "
+                              "overwrite an existing :local tagged image, use --force-local-overwrite to do so.")
+                    continue
+            # Tag the fetched image with the :local tag
+            _add_local_tag(image_to_fetch, image_registry, local_tag)
+        else:
+            if opts.o.verbose:
+                print(f"Excluding: {container}")
+    if not all_containers_found:
+        print("Warning: couldn't find usable images for one or more containers, this stack will not deploy")

stack_orchestrator/build/publish.py

@@ -0,0 +1,48 @@
+# Copyright © 2024 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+from datetime import datetime
+from python_on_whales import DockerClient
+
+from stack_orchestrator.opts import opts
+from stack_orchestrator.util import error_exit
+
+
+def _publish_tag_for_image(local_image_tag: str, remote_repo: str, version: str):
+    # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
+    (image_name, image_version) = local_image_tag.split(":")
+    if image_version == "local":
+        return f"{remote_repo}/{image_name}:{version}"
+    else:
+        error_exit("Asked to publish a non-locally built image")
+
+
+def publish_image(local_tag, registry):
+    if opts.o.verbose:
+        print(f"Publishing this image: {local_tag} to this registry: {registry}")
+    docker = DockerClient()
+    # Figure out the target image tag
+    # Eventually this version will be generated from the source repo state
+    # Using a timestemp is an intermediate step
+    version = datetime.now().strftime("%Y%m%d%H%M")
+    remote_tag = _publish_tag_for_image(local_tag, registry, version)
+    # Tag the image thus
+    if opts.o.debug:
+        print(f"Tagging {local_tag} to {remote_tag}")
+    docker.image.tag(local_tag, remote_tag)
+    # Push it to the desired registry
+    if opts.o.verbose:
+        print(f"Pushing image {remote_tag}")
+    docker.image.push(remote_tag)

stack_orchestrator/data/compose/docker-compose-container-registry.yml

@@ -5,9 +5,11 @@ services:
     environment:
       REGISTRY_LOG_LEVEL: ${REGISTRY_LOG_LEVEL}
     volumes:
+      - config:/config:ro
       - registry-data:/var/lib/registry
     ports:
       - "5000"
 
 volumes:
+  config:
   registry-data:

stack_orchestrator/data/compose/docker-compose-fixturenet-blast.yml

@@ -0,0 +1,80 @@
+
+# From: https://raw.githubusercontent.com/blast-io/deployment/master/docker-compose.yml
+services:
+  # generate jwt.txt if it's absent
+  generate-jwt:
+    image: blastio/openssl
+    volumes:
+      - blast-data:/blast:rw
+    command: >
+      sh -c "[ ! -f /blast/jwt.txt ] && openssl rand -hex 32 | tr -d '\n' > /blast/jwt.txt || exit 0"
+  # initialise geth db
+  geth-init:
+    image: blastio/blast-geth:${NETWORK:-testnet-sepolia}
+    volumes:
+      - blast-data:/blast:rw
+      - ../config/fixturenet-blast/genesis.json:/blast/genesis.json
+    entrypoint: /bin/sh
+    command: >
+      -c "[ ! -d /blast/${GETH_DATA_DIR:-blast-geth-data}/geth ] && /usr/local/bin/geth init --datadir=/blast/${GETH_DATA_DIR:-blast-geth-data} /blast/genesis.json || exit 0"
+    depends_on:
+      generate-jwt:
+        condition: service_completed_successfully
+    env_file: 
+      - ../config/fixturenet-blast/${NETWORK:-fixturenet}.config
+  blast-geth:
+    image: blastio/blast-geth:${NETWORK:-testnet-sepolia}
+    volumes:
+      - blast-data:/blast
+    ports:
+      - "9545"
+      - "9546"
+    command: >
+      --datadir=/blast/${GETH_DATA_DIR:-blast-geth-data}
+      --http
+      --http.corsdomain="*"
+      --http.vhosts="*"
+      --http.addr=0.0.0.0
+      --http.port=9545
+      --http.api=web3,debug,eth,txpool,net,engine
+      --ws
+      --ws.addr=0.0.0.0
+      --ws.port=9546
+      --ws.origins="*"
+      --ws.api=debug,eth,txpool,net,engine
+      --authrpc.addr="0.0.0.0"
+      --authrpc.port="8551"
+      --authrpc.vhosts="*"
+      --authrpc.jwtsecret=/blast/jwt.txt
+      --syncmode=full
+      --gcmode=archive
+      --nodiscover
+      --maxpeers=0
+      --rollup.disabletxpoolgossip=true
+    env_file: 
+      - ../config/fixturenet-blast/${NETWORK:-fixturenet}.config
+    depends_on:
+      geth-init:
+        condition: service_completed_successfully
+  op-node:
+    image: blastio/blast-optimism:${NETWORK:-testnet-sepolia}
+    volumes:
+      - blast-data:/blast
+      - ../config/fixturenet-blast/rollup.json:/blast/rollup.json
+    ports:
+      - "9003"
+    command: >
+      op-node
+      --l1="${CERC_L1_RPC}"
+      --l1.rpckind="any"
+      --l1.trustrpc=true
+      --l2="http://blast-geth:8551"
+      --l2.jwt-secret=/blast/jwt.txt
+      --rollup.config="/blast/rollup.json"
+    depends_on:
+      - blast-geth
+    env_file: 
+      - ../config/fixturenet-blast/${NETWORK:-fixturenet}.config
+
+volumes:
+  blast-data:

stack_orchestrator/data/compose/docker-compose-fixturenet-laconicd.yml

@@ -3,6 +3,9 @@ services:
     restart: unless-stopped
     image: cerc/laconicd:local
     command: ["sh", "/docker-entrypoint-scripts.d/create-fixturenet.sh"]
+    environment:
+      TEST_AUCTION_ENABLED: ${TEST_AUCTION_ENABLED}
+      TEST_REGISTRY_EXPIRY: ${TEST_REGISTRY_EXPIRY}
     volumes:
       # The cosmos-sdk node's database directory:
       - laconicd-data:/root/.laconicd
@@ -25,6 +28,7 @@ services:
     image: cerc/laconic-registry-cli:local
     volumes:
       - ../config/fixturenet-laconicd/registry-cli-config-template.yml:/registry-cli-config-template.yml
+      - ${BASE_DIR:-~/cerc}/laconic-registry-cli:/laconic-registry-cli
 
 volumes:
   laconicd-data:

stack_orchestrator/data/compose/docker-compose-fixturenet-plugeth.yml

@@ -6,6 +6,7 @@ services:
       - ../config/fixturenet-eth/fixturenet-eth.env
     environment:
       RUN_BOOTNODE: "true"
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
     image: cerc/fixturenet-plugeth-plugeth:local
     volumes:
       - fixturenet_plugeth_bootnode_geth_data:/root/ethdata

stack_orchestrator/data/compose/docker-compose-grafana.yml

@@ -2,14 +2,20 @@ version: "3.7"
 
 services:
   grafana:
-    image: grafana/grafana:10.2.2
+    image: grafana/grafana:10.2.3
     restart: always
     environment:
       GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL}
+      CERC_GRAFANA_ALERTS_SUBGRAPH_IDS: ${CERC_GRAFANA_ALERTS_SUBGRAPH_IDS}
     volumes:
       - ../config/monitoring/grafana/provisioning:/etc/grafana/provisioning
       - ../config/monitoring/grafana/dashboards:/etc/grafana/dashboards
+      - ../config/monitoring/update-grafana-alerts-config.sh:/update-grafana-alerts-config.sh
       - grafana_storage:/var/lib/grafana
+    user: root
+    entrypoint: ["bash", "-c"]
+    command: |
+      "/update-grafana-alerts-config.sh && /run.sh"
     ports:
       - "3000"
     healthcheck:

stack_orchestrator/data/compose/docker-compose-graph-node.yml

@@ -16,8 +16,12 @@ services:
       postgres_pass: password
       postgres_db: graph-node
       ethereum: ${ETH_NETWORKS:-lotus-fixturenet:http://lotus-node-1:1234/rpc/v1}
+      # Env varaibles reference: https://git.vdb.to/cerc-io/graph-node/src/branch/master/docs/environment-variables.md
       GRAPH_LOG: debug
       ETHEREUM_REORG_THRESHOLD: 3
+      GRAPH_ETHEREUM_JSON_RPC_TIMEOUT: ${GRAPH_ETHEREUM_JSON_RPC_TIMEOUT:-180}
+      GRAPH_ETHEREUM_REQUEST_RETRIES: ${GRAPH_ETHEREUM_REQUEST_RETRIES:-10}
+      GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE: ${GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE:-2000}
     entrypoint: ["bash", "-c"]
     # Wait for ETH RPC endpoint to be up when running with fixturenet-lotus
     command: |

stack_orchestrator/data/compose/docker-compose-mainnet-blast.yml

@@ -0,0 +1,83 @@
+
+# From: https://raw.githubusercontent.com/blast-io/deployment/master/docker-compose.yml
+services:
+  # generate jwt.txt if it's absent
+  generate-jwt:
+    image: blastio/openssl
+    volumes:
+      - blast-data:/blast:rw
+    command: >
+      sh -c "[ ! -f /blast/jwt.txt ] && openssl rand -hex 32 | tr -d '\n' > /blast/jwt.txt || exit 0"
+  # initialise geth db
+  geth-init:
+    image: blastio/blast-geth:${NETWORK:-mainnet}
+    volumes:
+      - blast-data:/blast:rw
+    entrypoint: /bin/sh
+    command: >
+      -c "[ ! -d /blast/${GETH_DATA_DIR:-blast-geth-data}/geth ] && /usr/local/bin/geth init --datadir=/blast/${GETH_DATA_DIR:-blast-geth-data} /blast/genesis.json || exit 0"
+    depends_on:
+      generate-jwt:
+        condition: service_completed_successfully
+    env_file: 
+      - ../config/mainnet-blast/${NETWORK:-mainnet}.config
+  blast-geth:
+    image: blastio/blast-geth:${NETWORK:-mainnet}
+    volumes:
+      - blast-data:/blast
+    ports:
+      - "9545"
+      - "9546"
+      - "6060"
+    command: >
+      --datadir=/blast/${GETH_DATA_DIR:-blast-geth-data}
+      --http
+      --http.corsdomain="*"
+      --http.vhosts="*"
+      --http.addr=0.0.0.0
+      --http.port=9545
+      --http.api=web3,debug,eth,txpool,net,engine
+      --ws
+      --ws.addr=0.0.0.0
+      --ws.port=9546
+      --ws.origins="*"
+      --ws.api=debug,eth,txpool,net,engine
+      --authrpc.addr="0.0.0.0"
+      --authrpc.port="8551"
+      --authrpc.vhosts="*"
+      --authrpc.jwtsecret=/blast/jwt.txt
+      --syncmode=full
+      --metrics
+      --metrics.addr=0.0.0.0
+      --gcmode=archive
+      --nodiscover
+      --maxpeers=0
+      --rollup.disabletxpoolgossip=true
+    env_file: 
+      - ../config/mainnet-blast/${NETWORK:-mainnet}.config
+    depends_on:
+      geth-init:
+        condition: service_completed_successfully
+  op-node:
+    image: blastio/blast-optimism:${NETWORK:-mainnet}
+    volumes:
+      - blast-data:/blast
+    ports:
+      - "9003"
+      - "7300"
+    command: >
+      op-node
+      --l1="https://eth-mainnet-1.vdb.to/"
+      --metrics.enabled
+      --l1.rpckind="any"
+      --l1.trustrpc=true
+      --l2="http://blast-geth:8551"
+      --l2.jwt-secret=/blast/jwt.txt
+      --rollup.config="/blast/rollup.json"
+    depends_on:
+      - blast-geth
+    env_file: 
+      - ../config/mainnet-blast/${NETWORK:-mainnet}.config
+
+volumes:
+  blast-data:

stack_orchestrator/data/compose/docker-compose-prom-server.yml

@@ -18,6 +18,14 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
+  alert-manager:
+    image: prom/alertmanager
+    volumes:
+      - ../config/monitoring/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml
+      - alertmanager_data:/data
+    ports:
+      - "9093"
+
   blackbox:
     image: prom/blackbox-exporter:latest
     restart: always
@@ -55,3 +63,4 @@ services:
 
 volumes:
   prometheus_data:
+  alertmanager_data:

stack_orchestrator/data/compose/docker-compose-snowballtools-base-backend.yml

@@ -0,0 +1,13 @@
+services:
+  snowballtools-base-backend:
+    image: cerc/snowballtools-base-backend:local
+    restart: always
+    volumes:
+      - data:/data
+      - config:/config:ro
+    ports:
+      - 8000
+
+volumes:
+  data:
+  config:

stack_orchestrator/data/compose/docker-compose-test.yml

@@ -6,6 +6,7 @@ services:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
       CERC_TEST_PARAM_1: ${CERC_TEST_PARAM_1:-FAILED}
       CERC_TEST_PARAM_2: "CERC_TEST_PARAM_2_VALUE"
+      CERC_TEST_PARAM_3: ${CERC_TEST_PARAM_3:-FAILED}
     volumes:
       - test-data-bind:/data
       - test-data-auto:/data2

stack_orchestrator/data/compose/docker-compose-watcher-ajna.yml

@@ -0,0 +1,76 @@
+version: '3.2'
+
+services:
+  ajna-watcher-db:
+    restart: unless-stopped
+    image: postgres:14-alpine
+    environment:
+      - POSTGRES_USER=vdbm
+      - POSTGRES_MULTIPLE_DATABASES=ajna-watcher,ajna-watcher-job-queue
+      - POSTGRES_EXTENSION=ajna-watcher-job-queue:pgcrypto
+      - POSTGRES_PASSWORD=password
+    volumes:
+      - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
+      - ajna_watcher_db_data:/var/lib/postgresql/data
+    ports:
+      - "5432"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+
+  ajna-watcher-job-runner:
+    restart: unless-stopped
+    depends_on:
+      ajna-watcher-db:
+        condition: service_healthy
+    image: cerc/watcher-ajna:local
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+    command: ["bash", "./start-job-runner.sh"]
+    volumes:
+      - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
+      - ../config/watcher-ajna/start-job-runner.sh:/app/start-job-runner.sh
+    ports:
+      - "9000"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  ajna-watcher-server:
+    restart: unless-stopped
+    depends_on:
+      ajna-watcher-db:
+        condition: service_healthy
+      ajna-watcher-job-runner:
+        condition: service_healthy
+    image: cerc/watcher-ajna:local
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+    command: ["bash", "./start-server.sh"]
+    volumes:
+      - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
+      - ../config/watcher-ajna/start-server.sh:/app/start-server.sh
+    ports:
+      - "3008"
+      - "9001"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+volumes:
+  ajna_watcher_db_data:

stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml

@@ -32,8 +32,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
       CERC_HISTORICAL_BLOCK_RANGE: 500
       CONTRACT_ADDRESS: 0x223c067F8CF28ae173EE5CafEa60cA44C335fecB
       CONTRACT_NAME: Azimuth
@@ -47,7 +47,7 @@ services:
     ports:
       - "9000"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -66,8 +66,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
     working_dir: /app/packages/azimuth-watcher
     command: "./start-server.sh"
     volumes:
@@ -77,7 +77,7 @@ services:
     ports:
       - "3001"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3001"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3001"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -94,8 +94,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
       CONTRACT_ADDRESS: 0x325f68d32BdEe6Ed86E7235ff2480e2A433D6189
       CONTRACT_NAME: Censures
       STARTING_BLOCK: 6784954
@@ -108,7 +108,7 @@ services:
     ports:
       - "9002"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9002"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9002"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -127,8 +127,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
     working_dir: /app/packages/censures-watcher
     command: "./start-server.sh"
     volumes:
@@ -138,7 +138,7 @@ services:
     ports:
       - "3002"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3002"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3002"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -155,8 +155,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
       CONTRACT_ADDRESS: 0xe7e7f69b34D7d9Bd8d61Fb22C33b22708947971A
       CONTRACT_NAME: Claims
       STARTING_BLOCK: 6784941
@@ -169,7 +169,7 @@ services:
     ports:
       - "9004"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9004"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9004"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -188,8 +188,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
     working_dir: /app/packages/claims-watcher
     command: "./start-server.sh"
     volumes:
@@ -199,7 +199,7 @@ services:
     ports:
       - "3003"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3003"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3003"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -216,8 +216,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
       CONTRACT_ADDRESS: 0x8C241098C3D3498Fe1261421633FD57986D74AeA
       CONTRACT_NAME: ConditionalStarRelease
       STARTING_BLOCK: 6828004
@@ -230,7 +230,7 @@ services:
     ports:
       - "9006"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9006"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9006"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -249,8 +249,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
     working_dir: /app/packages/conditional-star-release-watcher
     command: "./start-server.sh"
     volumes:
@@ -260,7 +260,7 @@ services:
     ports:
       - "3004"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3004"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3004"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -277,8 +277,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
       CONTRACT_ADDRESS: 0xf6b461fE1aD4bd2ce25B23Fe0aff2ac19B3dFA76
       CONTRACT_NAME: DelegatedSending
       STARTING_BLOCK: 6784956
@@ -291,7 +291,7 @@ services:
     ports:
       - "9008"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9008"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -310,8 +310,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
     working_dir: /app/packages/delegated-sending-watcher
     command: "./start-server.sh"
     volumes:
@@ -321,7 +321,7 @@ services:
     ports:
       - "3005"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3005"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3005"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -338,8 +338,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
       CONTRACT_ADDRESS: 0x33EeCbf908478C10614626A9D304bfe18B78DD73
       CONTRACT_NAME: Ecliptic
       STARTING_BLOCK: 13692129
@@ -352,7 +352,7 @@ services:
     ports:
       - "9010"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9010"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9010"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -371,8 +371,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
     working_dir: /app/packages/ecliptic-watcher
     command: "./start-server.sh"
     volumes:
@@ -382,7 +382,7 @@ services:
     ports:
       - "3006"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3006"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3006"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -399,8 +399,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
       CONTRACT_ADDRESS: 0x86cd9cd0992F04231751E3761De45cEceA5d1801
       CONTRACT_NAME: LinearStarRelease
       STARTING_BLOCK: 6784943
@@ -413,7 +413,7 @@ services:
     ports:
       - "9012"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9012"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9012"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -432,8 +432,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
     working_dir: /app/packages/linear-star-release-watcher
     command: "./start-server.sh"
     volumes:
@@ -443,7 +443,7 @@ services:
     ports:
       - "3007"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3007"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3007"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -460,8 +460,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
       CONTRACT_ADDRESS: 0x7fEcaB617c868Bb5996d99D95200D2Fa708218e4
       CONTRACT_NAME: Polls
       STARTING_BLOCK: 6784912
@@ -474,7 +474,7 @@ services:
     ports:
       - "9014"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9014"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9014"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -493,8 +493,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
     working_dir: /app/packages/polls-watcher
     command: "./start-server.sh"
     volumes:
@@ -504,7 +504,7 @@ services:
     ports:
       - "3008"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -542,7 +542,7 @@ services:
     ports:
       - "0.0.0.0:4000:4000"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "4000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "4000"]
       interval: 20s
       timeout: 5s
       retries: 15

stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml

@@ -29,7 +29,7 @@ services:
     image: cerc/watcher-merkl-sushiswap-v3:local
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
     command: ["bash", "./start-job-runner.sh"]
     volumes:
       - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -37,7 +37,7 @@ services:
     ports:
       - "9002:9000"
     healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -55,7 +55,7 @@ services:
     image: cerc/watcher-merkl-sushiswap-v3:local
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
     command: ["bash", "./start-server.sh"]
     volumes:
       - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -64,7 +64,7 @@ services:
       - "127.0.0.1:3007:3008"
       - "9003:9001"
     healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
       interval: 20s
       timeout: 5s
       retries: 15

stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml

@@ -29,7 +29,7 @@ services:
     image: cerc/watcher-sushiswap-v3:local
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
     command: ["bash", "./start-job-runner.sh"]
     volumes:
       - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -37,7 +37,7 @@ services:
     ports:
       - "9000:9000"
     healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -55,7 +55,7 @@ services:
     image: cerc/watcher-sushiswap-v3:local
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
     command: ["bash", "./start-server.sh"]
     volumes:
       - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -64,7 +64,7 @@ services:
       - "127.0.0.1:3008:3008"
       - "9001:9001"
     healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
       interval: 20s
       timeout: 5s
       retries: 15

stack_orchestrator/data/config/fixturenet-blast/fixturenet.config

@@ -0,0 +1,2 @@
+GETH_ROLLUP_SEQUENCERHTTP=https://sequencer.s2.testblast.io
+OP_NODE_P2P_BOOTNODES=enr:-J-4QM3GLUFfKMSJQuP1UvuKQe8DyovE7Eaiit0l6By4zjTodkR4V8NWXJxNmlg8t8rP-Q-wp3jVmeAOml8cjMj__ROGAYznzb_HgmlkgnY0gmlwhA-cZ_eHb3BzdGFja4X947FQAIlzZWNwMjU2azGhAiuDqvB-AsVSRmnnWr6OHfjgY8YfNclFy9p02flKzXnOg3RjcIJ2YYN1ZHCCdmE,enr:-J-4QDCVpByqQ8nFqCS9aHicqwUfXgzFDslvpEyYz19lvkHLIdtcIGp2d4q5dxHdjRNTO6HXCsnIKxUeuZSPcEbyVQCGAYznzz0RgmlkgnY0gmlwhANiQfuHb3BzdGFja4X947FQAIlzZWNwMjU2azGhAy3AtF2Jh_aPdOohg506Hjmtx-fQ1AKmu71C7PfkWAw9g3RjcIJ2YYN1ZHCCdmE

stack_orchestrator/data/config/fixturenet-blast/genesis.json

@@ -0,0 +1,57 @@
+{
+  "config": {
+    "chainId": 608943043,
+    "homesteadBlock": 0,
+    "eip150Block": 0,
+    "eip155Block": 0,
+    "eip158Block": 0,
+    "byzantiumBlock": 0,
+    "constantinopleBlock": 0,
+    "petersburgBlock": 0,
+    "istanbulBlock": 0,
+    "muirGlacierBlock": 0,
+    "berlinBlock": 0,
+    "londonBlock": 0,
+    "arrowGlacierBlock": 0,
+    "grayGlacierBlock": 0,
+    "mergeNetsplitBlock": 0,
+    "shanghaiTime": 0,
+    "bedrockBlock": 0,
+    "regolithTime": 0,
+    "canyonTime": 0,
+    "terminalTotalDifficulty": 0,
+    "terminalTotalDifficultyPassed": true,
+    "optimism": {
+      "eip1559Elasticity": 6,
+      "eip1559Denominator": 50,
+      "eip1559DenominatorCanyon": 250
+    }
+  },
+  "alloc": {
+    "0000000000000000000000000000000000000000": {
+      "balance": "0x1"
+    },
+    "4200000000000000000000000000000000000000": {
+      "code": "0x60806040526004361061004e5760003560e01c80633659cfe6146100655780634f1ef286146100855780635c60da1b146100ae5780638f283970146100db578063f851a440146100fb5761005d565b3661005d5761005b610110565b005b61005b610110565b34801561007157600080fd5b5061005b610080366004610521565b6101c8565b61009861009336600461053c565b61020e565b6040516100a591906105bf565b60405180910390f35b3480156100ba57600080fd5b506100c361033e565b6040516001600160a01b0390911681526020016100a5565b3480156100e757600080fd5b5061005b6100f6366004610521565b6103a9565b34801561010757600080fd5b506100c36103e4565b600061013a7f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc5490565b90506001600160a01b0381166101a55760405162461bcd60e51b815260206004820152602560248201527f50726f78793a20696d706c656d656e746174696f6e206e6f7420696e697469616044820152641b1a5e995960da1b60648201526084015b60405180910390fd5b3660008037600080366000845af43d6000803e806101c2573d6000fd5b503d6000f35b600080516020610625833981519152546001600160a01b0316336001600160a01b031614806101f5575033155b156102065761020381610432565b50565b610203610110565b60606102266000805160206106258339815191525490565b6001600160a01b0316336001600160a01b03161480610243575033155b1561032f5761025184610432565b600080856001600160a01b0316858560405161026e929190610614565b600060405180830381855af49150503d80600081146102a9576040519150601f19603f3d011682016040523d82523d6000602084013e6102ae565b606091505b5091509150816103265760405162461bcd60e51b815260206004820152603960248201527f50726f78793a2064656c656761746563616c6c20746f206e657720696d706c6560448201527f6d656e746174696f6e20636f6e7472616374206661696c656400000000000000606482015260840161019c565b91506103379050565b610337610110565b9392505050565b60006103566000805160206106258339815191525490565b6001600160a01b0316336001600160a01b03161480610373575033155b1561039e57507f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc5490565b6103a6610110565b90565b600080516020610625833981519152546001600160a01b0316336001600160a01b031614806103d6575033155b15610206576102038161048e565b60006103fc6000805160206106258339815191525490565b6001600160a01b0316336001600160a01b03161480610419575033155b1561039e57506000805160206106258339815191525490565b7f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc8181556040516001600160a01b038316907fbc7cd75a20ee27fd9adebab32041f755214dbc6bffa90cc0225b39da2e5c2d3b90600090a25050565b60006104a66000805160206106258339815191525490565b600080516020610625833981519152838155604080516001600160a01b0380851682528616602082015292935090917f7e644d79422f17c01e4894b5f4f588d331ebfa28653d42ae832dc59e38c9798f910160405180910390a1505050565b80356001600160a01b038116811461051c57600080fd5b919050565b60006020828403121561053357600080fd5b61033782610505565b60008060006040848603121561055157600080fd5b61055a84610505565b9250602084013567ffffffffffffffff8082111561057757600080fd5b818601915086601f83011261058b57600080fd5b81358181111561059a57600080fd5b8760208285010111156105ac57600080fd5b6020830194508093505050509250925092565b600060208083528351808285015260005b818110156105ec578581018301518582016040015282016105d0565b818111156105fe576000604083870101525b50601f01601f1916929092016040019392505050565b818382376000910190815291905056feb53127684a568b3173ae13b9f8a6016e243e63b6e8ee1178d6a717850b5d6103a164736f6c634300080f000a",
+      "storage": {
+        "0x360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc": "0x000000000000000000000000c0d3c0d3c0d3c0d3c0d3c0d3c0d3c0d3c0d30000",
+        "0xb53127684a568b3173ae13b9f8a6016e243e63b6e8ee1178d6a717850b5d6103": "0x0000000000000000000000004200000000000000000000000000000000000018"
+      },
+      "balance": "0x0",
+      "flags": 1
+    }
+  },
+  "nonce": "0x0",
+  "timestamp": "0x659b7460",
+  "extraData": "0x424544524f434b",
+  "gasLimit": "0x1c9c380",
+  "difficulty": "0x0",
+  "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+  "coinbase": "0x4200000000000000000000000000000000000011",
+  "number": "0x0",
+  "gasUsed": "0x0",
+  "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+  "baseFeePerGas": "0x3b9aca00",
+  "excessBlobGas": null,
+  "blobGasUsed": null
+}

stack_orchestrator/data/config/fixturenet-blast/rollup.json

@@ -0,0 +1,31 @@
+{
+  "genesis": {
+    "l1": {
+      "hash": "0x17728cf4d8e0b4f292d2390a869fd7c632d39e72efb00ca3462b4387c6aa2437",
+      "number": 5044255
+    },
+    "l2": {
+      "hash": "0x26a1c0faad7b041f34569a1bb383f00ab74b335883a44bed53e9f41ced5fd906",
+      "number": 0
+    },
+    "l2_time": 1704686688,
+    "system_config": {
+      "batcherAddr": "0xba26fee2fa917443e05e65de8d4350bcd2f59222",
+      "overhead": "0x00000000000000000000000000000000000000000000000000000000000000bc",
+      "scalar": "0x00000000000000000000000000000000000000000000000000000000000a6fe0",
+      "gasLimit": 30000000
+    }
+  },
+  "block_time": 2,
+  "max_sequencer_drift": 600,
+  "seq_window_size": 3600,
+  "channel_timeout": 300,
+  "l1_chain_id": 11155111,
+  "l2_chain_id": 608943043,
+  "regolith_time": 0,
+  "canyon_time": 0,
+  "batch_inbox_address": "0x1c3b85a2108784eab6a4bf56cdd6f722e415b331",
+  "deposit_contract_address": "0x2757e4430e694f27b73ec9c02257cab3a498c8c5",
+  "l1_system_config_address": "0x329faf078c364a316e08bf6a17b7eee6ae75a613",
+  "protocol_versions_address": "0x0000000000000000000000000000000000000000"
+}

stack_orchestrator/data/config/fixturenet-eth/fixturenet-eth.env

@@ -23,3 +23,6 @@ CERC_STATEDIFF_WORKERS=2
 
 CERC_GETH_VMODULE="statediff/*=5,rpc/*=5"
 CERC_GETH_VERBOSITY=${CERC_GETH_VERBOSITY:-3}
+
+# Used by Lighthouse
+SECONDS_PER_ETH1_BLOCK=${SECONDS_PER_ETH1_BLOCK:-3}

stack_orchestrator/data/config/fixturenet-laconicd/create-fixturenet.sh

@@ -102,6 +102,17 @@ if [ "$1" == "clean" ] || [ ! -d "$HOME/.laconicd/data/blockstore.db" ]; then
     fi
   fi
 
+  # Enable telemetry (prometheus metrics: http://localhost:1317/metrics?format=prometheus)
+  if [[ "$OSTYPE" == "darwin"* ]]; then
+    sed -i '' 's/enabled = false/enabled = true/g' $HOME/.laconicd/config/app.toml
+    sed -i '' 's/prometheus-retention-time = 0/prometheus-retention-time = 60/g' $HOME/.laconicd/config/app.toml
+    sed -i '' 's/prometheus = false/prometheus = true/g' $HOME/.laconicd/config/config.toml
+  else
+    sed -i 's/enabled = false/enabled = true/g' $HOME/.laconicd/config/app.toml
+    sed -i 's/prometheus-retention-time = 0/prometheus-retention-time = 60/g' $HOME/.laconicd/config/app.toml
+    sed -i 's/prometheus = false/prometheus = true/g' $HOME/.laconicd/config/config.toml
+  fi
+
   # Allocate genesis accounts (cosmos formatted addresses)
   laconicd add-genesis-account $KEY 100000000000000000000000000aphoton --keyring-backend $KEYRING
 

stack_orchestrator/data/config/mainnet-blast/keycloak.env

@@ -0,0 +1,32 @@
+POSTGRES_DB=keycloak
+POSTGRES_USER=keycloak
+POSTGRES_PASSWORD=keycloak
+# Don't change this unless you also change the healthcheck in docker-compose-mainnet-eth-keycloak.yml
+PGPORT=35432
+KC_DB=postgres
+KC_DB_URL_HOST=keycloak-db
+KC_DB_URL_PORT=${PGPORT}
+KC_DB_URL_DATABASE=${POSTGRES_DB}
+KC_DB_USERNAME=${POSTGRES_USER}
+KC_DB_PASSWORD=${POSTGRES_PASSWORD}
+KC_DB_SCHEMA=public
+KC_HOSTNAME=localhost
+KC_HTTP_ENABLED="true"
+KC_HTTP_RELATIVE_PATH="/auth"
+KC_HOSTNAME_STRICT_HTTPS="false"
+KEYCLOAK_ADMIN=admin
+KEYCLOAK_ADMIN_PASSWORD=admin
+X_API_CHECK_REALM=cerc
+X_API_CHECK_CLIENT_ID="%user_id%"
+
+
+# keycloak-reg-api
+CERC_KCUSERREG_LISTEN_PORT=9292
+CERC_KCUSERREG_LISTEN_ADDR='0.0.0.0'
+CERC_KCUSERREG_API_URL='http://keycloak:8080/auth'
+CERC_KCUSERREG_REG_USER="${KEYCLOAK_ADMIN}"
+CERC_KCUSERREG_REG_PW="${KEYCLOAK_ADMIN_PASSWORD}"
+CERC_KCUSERREG_REG_CLIENT_ID='admin-cli'
+CERC_KCUSERREG_TARGET_REALM=cerc
+CERC_KCUSERREG_TARGET_GROUPS=eth
+CERC_KCUSERREG_CREATE_ENABLED=true

stack_orchestrator/data/config/mainnet-blast/lighthouse.env

@@ -0,0 +1,33 @@
+# Enable startup script debug output.
+CERC_SCRIPT_DEBUG=false
+
+# Specify any other lighthouse CLI options.
+LIGHTHOUSE_OPTS=""
+
+# Override the advertised public IP (optional)
+# --enr-address
+#LIGHTHOUSE_ENR_ADDRESS=""
+
+# --checkpoint-sync-url
+LIGHTHOUSE_CHECKPOINT_SYNC_URL="https://beaconstate.ethstaker.cc"
+
+# --checkpoint-sync-url-timeout
+LIGHTHOUSE_CHECKPOINT_SYNC_URL_TIMEOUT=300
+
+# --datadir
+LIGHTHOUSE_DATADIR=/data
+
+# --debug-level
+LIGHTHOUSE_DEBUG_LEVEL=info
+
+# --http-port
+LIGHTHOUSE_HTTP_PORT=5052
+
+# --execution-jwt
+LIGHTHOUSE_JWTSECRET=/etc/mainnet-eth/jwtsecret
+
+# --metrics-port
+LIGHTHOUSE_METRICS_PORT=5054
+
+# --port --enr-udp-port --enr-tcp-port
+LIGHTHOUSE_NETWORK_PORT=9000

stack_orchestrator/data/config/mainnet-blast/mainnet.config

@@ -0,0 +1,2 @@
+GETH_ROLLUP_SEQUENCERHTTP=https://sequencer.blast.io
+OP_NODE_P2P_BOOTNODES=enr:-J64QGwHl9uYLfC_cnmxSA6wQH811nkOWJDWjzxqkEUlJoZHWvI66u-BXgVcPCeMUmg0dBpFQAPotFchG67FHJMZ9OSGAY3d6wevgmlkgnY0gmlwhANizeSHb3BzdGFja4Sx_AQAiXNlY3AyNTZrMaECg4pk0cskPAyJ7pOmo9E6RqGBwV-Lex4VS9a3MQvu7PWDdGNwgnZhg3VkcIJ2YQ,enr:-J64QDge2jYBQtcNEpRqmKfci5E5BHAhNBjgv4WSdwH1_wPqbueq2bDj38-TSW8asjy5lJj1Xftui6Or8lnaYFCqCI-GAY3d6wf3gmlkgnY0gmlwhCO2D9yHb3BzdGFja4Sx_AQAiXNlY3AyNTZrMaEDo4aCTq7pCEN8om9U5n_VyWdambGnQhwHNwKc8o-OicaDdGNwgnZhg3VkcIJ2YQ

stack_orchestrator/data/config/mainnet-blast/rollup.json

@@ -0,0 +1,32 @@
+{
+    "genesis": {
+      "l1": {
+        "hash": "0xfcfb8d586bdae763f1189988789211c69eb893a895e7ba48be3ca6289f0941b7",
+        "number": 19300102
+      },
+      "l2": {
+        "hash": "0xb689b35ef29d0bec5816938e0e52683c7257d2e325420ea69b739a2be4754b89",
+        "number": 0
+      },
+      "l2_time": 1708809815,
+      "system_config": {
+        "batcherAddr": "0x415c8893d514f9bc5211d36eeda4183226b84aa7",
+        "overhead": "0x00000000000000000000000000000000000000000000000000000000000000bc",
+        "scalar": "0x00000000000000000000000000000000000000000000000000000000000a6fe0",
+        "gasLimit": 30000000
+      }
+    },
+    "block_time": 2,
+    "max_sequencer_drift": 600,
+    "seq_window_size": 3600,
+    "channel_timeout": 300,
+    "l1_chain_id": 1,
+    "l2_chain_id": 81457,
+    "regolith_time": 0,
+    "canyon_time": 0,
+    "batch_inbox_address": "0xff00000000000000000000000000000000081457",
+    "deposit_contract_address": "0x0ec68c5b10f21effb74f2a5c61dfe6b08c0db6cb",
+    "l1_system_config_address": "0x5531dcff39ec1ec727c4c5d2fc49835368f805a9",
+    "protocol_versions_address": "0x0000000000000000000000000000000000000000"
+  }
+  

stack_orchestrator/data/config/mainnet-blast/run-lighthouse.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+if [[ "true" == "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi
+
+ENR_OPTS=""
+if [[ -n "$LIGHTHOUSE_ENR_ADDRESS" ]]; then
+  ENR_OPTS="--enr-address $LIGHTHOUSE_ENR_ADDRESS"
+fi
+
+exec lighthouse bn \
+  --checkpoint-sync-url "$LIGHTHOUSE_CHECKPOINT_SYNC_URL" \
+  --checkpoint-sync-url-timeout ${LIGHTHOUSE_CHECKPOINT_SYNC_URL_TIMEOUT} \
+  --datadir "$LIGHTHOUSE_DATADIR" \
+  --debug-level $LIGHTHOUSE_DEBUG_LEVEL \
+  --disable-deposit-contract-sync \
+  --disable-upnp \
+  --enr-tcp-port $LIGHTHOUSE_NETWORK_PORT \
+  --enr-udp-port $LIGHTHOUSE_NETWORK_PORT \
+  --execution-endpoint "$LIGHTHOUSE_EXECUTION_ENDPOINT" \
+  --execution-jwt /etc/mainnet-eth/jwtsecret \
+  --http \
+  --http-address 0.0.0.0 \
+  --http-port $LIGHTHOUSE_HTTP_PORT \
+  --metrics \
+  --metrics-address=0.0.0.0 \
+  --metrics-port $LIGHTHOUSE_METRICS_PORT \
+  --network mainnet \
+  --port $LIGHTHOUSE_NETWORK_PORT \
+  $ENR_OPTS $LIGHTHOUSE_OPTS

stack_orchestrator/data/config/monitoring/alertmanager/alertmanager.yml

@@ -0,0 +1,10 @@
+route:
+  receiver: 'slack'
+  # group_by: [ alertname ]
+
+receivers:
+  - name: 'slack'
+    slack_configs:
+      - send_resolved: true
+        api_url: <YOUR_SLACK_WEBHOOK_URL>
+        channel: <YOUR_SLACK_CHANNEL>

stack_orchestrator/data/config/monitoring/grafana/dashboards/watcher-dashboard.json

@@ -18,7 +18,7 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "id": 4,
+  "id": 6,
   "links": [
     {
       "asDropdown": false,
@@ -55,7 +55,7 @@
         "x": 0,
         "y": 0
       },
-      "id": 1,
+      "id": 29,
       "panels": [
         {
           "datasource": {
@@ -103,7 +103,7 @@
             "textMode": "auto",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
           "targets": [
             {
               "datasource": {
@@ -173,7 +173,7 @@
             "textMode": "auto",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
           "targets": [
             {
               "datasource": {
@@ -242,7 +242,7 @@
             "textMode": "auto",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
           "targets": [
             {
               "datasource": {
@@ -438,7 +438,7 @@
             "textMode": "auto",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
           "targets": [
             {
               "datasource": {
@@ -518,7 +518,7 @@
             "textMode": "auto",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
           "targets": [
             {
               "datasource": {
@@ -629,7 +629,7 @@
             "textMode": "auto",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
           "targets": [
             {
               "datasource": {
@@ -730,7 +730,7 @@
             "textMode": "auto",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
           "targets": [
             {
               "datasource": {
@@ -854,7 +854,7 @@
             "textMode": "auto",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
           "targets": [
             {
               "datasource": {
@@ -1025,7 +1025,7 @@
             "textMode": "auto",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
           "targets": [
             {
               "datasource": {
@@ -1111,7 +1111,7 @@
             "textMode": "auto",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
           "targets": [
             {
               "datasource": {
@@ -1146,6 +1146,230 @@
           "title": "DB size (MB)",
           "type": "stat"
         },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "thresholds"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "dark-green",
+                    "value": null
+                  },
+                  {
+                    "color": "dark-orange",
+                    "value": 8
+                  },
+                  {
+                    "color": "dark-red",
+                    "value": 16
+                  }
+                ]
+              }
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "diff_external"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "mode": "thresholds",
+                      "seriesBy": "last"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "diff_external"
+                },
+                "properties": [
+                  {
+                    "id": "thresholds",
+                    "value": {
+                      "mode": "absolute",
+                      "steps": [
+                        {
+                          "color": "green",
+                          "value": null
+                        },
+                        {
+                          "color": "orange",
+                          "value": 8
+                        },
+                        {
+                          "color": "red",
+                          "value": 16
+                        }
+                      ]
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 6,
+            "w": 12,
+            "x": 0,
+            "y": 12
+          },
+          "id": 33,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "10.2.3",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "disableTextWrap": false,
+              "editorMode": "code",
+              "expr": "latest_upstream_block_number{job=~\"$job\", instance=~\"$watcher\"}",
+              "fullMetaSearch": false,
+              "hide": false,
+              "includeNullMetadata": true,
+              "instant": false,
+              "legendFormat": "{{__name__}}",
+              "range": true,
+              "refId": "A",
+              "useBackend": false
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "editorMode": "code",
+              "expr": "sync_status_block_number{job=~\"$job\", instance=~\"$watcher\", kind=\"latest_indexed\"}",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{__name__}}",
+              "range": true,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "editorMode": "code",
+              "expr": "latest_block_number{job=\"chain_heads\", chain=\"$target_chain\"}",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{__name__}}",
+              "range": true,
+              "refId": "C"
+            }
+          ],
+          "title": "diff",
+          "transformations": [
+            {
+              "id": "calculateField",
+              "options": {
+                "alias": "diff_upstream",
+                "binary": {
+                  "left": "latest_upstream_block_number",
+                  "operator": "-",
+                  "right": "sync_status_block_number"
+                },
+                "mode": "binary",
+                "reduce": {
+                  "reducer": "sum"
+                },
+                "replaceFields": false
+              }
+            },
+            {
+              "id": "calculateField",
+              "options": {
+                "alias": "diff_external",
+                "binary": {
+                  "left": "latest_block_number",
+                  "operator": "-",
+                  "right": "sync_status_block_number"
+                },
+                "mode": "binary",
+                "reduce": {
+                  "reducer": "sum"
+                },
+                "replaceFields": false
+              }
+            },
+            {
+              "id": "organize",
+              "options": {
+                "excludeByName": {
+                  "Time": false,
+                  "latest_block_number": true,
+                  "latest_upstream_block_number": true,
+                  "sync_status_block_number": true
+                },
+                "includeByName": {},
+                "indexByName": {},
+                "renameByName": {}
+              }
+            }
+          ],
+          "type": "timeseries"
+        },
         {
           "datasource": {
             "type": "prometheus",
@@ -1208,8 +1432,8 @@
           },
           "gridPos": {
             "h": 6,
-            "w": 12,
+            "w": 6,
-            "x": 0,
+            "x": 12,
             "y": 12
           },
           "id": 5,
@@ -1310,8 +1534,8 @@
           },
           "gridPos": {
             "h": 6,
-            "w": 12,
+            "w": 6,
-            "x": 12,
+            "x": 18,
             "y": 12
           },
           "id": 15,
@@ -1350,6 +1574,436 @@
           "title": "Num event-processing jobs",
           "type": "timeseries"
         },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "description": "Total number of failed ETH RPC requests by method and provider endpoint ",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 18
+          },
+          "id": 28,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "10.2.3",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "disableTextWrap": false,
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "watcher_eth_rpc_errors{job=~\"$job\", instance=~\"$watcher\"}",
+              "format": "time_series",
+              "fullMetaSearch": false,
+              "includeNullMetadata": true,
+              "instant": false,
+              "legendFormat": "{{method}}, {{provider}}",
+              "range": true,
+              "refId": "A",
+              "useBackend": false
+            }
+          ],
+          "title": "ETH RPC request failures",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "description": "Time taken by last ETH RPC requests by method and provider endpoint",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 18
+          },
+          "id": 26,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "editorMode": "code",
+              "expr": "watcher_eth_rpc_request_duration{job=~\"$job\", instance=~\"$watcher\"}",
+              "instant": false,
+              "legendFormat": "{{method}}, {{provider}}",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "ETH RPC request durations",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "description": "Total number of failed ETH RPC requests by method and provider endpoint  (across all watchers)",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 25
+          },
+          "id": 30,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "10.2.3",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "disableTextWrap": false,
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "sum by (method, provider) (watcher_eth_rpc_errors{chain=\"$target_chain\"})",
+              "format": "time_series",
+              "fullMetaSearch": false,
+              "includeNullMetadata": true,
+              "instant": false,
+              "legendFormat": "{{method}}, {{provider}}",
+              "range": true,
+              "refId": "A",
+              "useBackend": false
+            }
+          ],
+          "title": "Total ETH RPC request failures (across all watchers)",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "description": "Configured upstream ETH RPC endpoints",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "fixedColor": "blue",
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "align": "auto",
+                "cellOptions": {
+                  "type": "auto"
+                },
+                "filterable": false,
+                "inspect": false
+              },
+              "mappings": [
+                {
+                  "options": {
+                    "0": {
+                      "color": "red",
+                      "index": 0,
+                      "text": "inactive"
+                    },
+                    "1": {
+                      "color": "green",
+                      "index": 1,
+                      "text": "active"
+                    }
+                  },
+                  "type": "value"
+                }
+              ],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "blue",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "status"
+                },
+                "properties": [
+                  {
+                    "id": "custom.cellOptions",
+                    "value": {
+                      "type": "color-text"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 25
+          },
+          "id": 32,
+          "options": {
+            "cellHeight": "sm",
+            "footer": {
+              "countRows": false,
+              "enablePagination": false,
+              "fields": "",
+              "reducer": [
+                "sum"
+              ],
+              "show": false
+            },
+            "frameIndex": 0,
+            "showHeader": true,
+            "sortBy": [
+              {
+                "desc": true,
+                "displayName": "status"
+              }
+            ]
+          },
+          "pluginVersion": "10.2.3",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "watcher_config_upstream_endpoints{job=~\"$job\", instance=~\"$watcher\"}",
+              "format": "table",
+              "instant": true,
+              "legendFormat": "__auto",
+              "range": false,
+              "refId": "A"
+            }
+          ],
+          "title": "Configured upstream endpoints",
+          "transformations": [
+            {
+              "id": "labelsToFields",
+              "options": {
+                "mode": "columns"
+              }
+            },
+            {
+              "id": "organize",
+              "options": {
+                "excludeByName": {
+                  "Time": true,
+                  "__name__": true,
+                  "chain": true,
+                  "instance": true,
+                  "job": true
+                },
+                "includeByName": {},
+                "indexByName": {},
+                "renameByName": {
+                  "Value": "status"
+                }
+              }
+            }
+          ],
+          "type": "table"
+        },
         {
           "datasource": {
             "type": "prometheus",
@@ -1377,7 +2031,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   }
                 ]
               }
@@ -1388,7 +2043,7 @@
             "h": 5,
             "w": 8,
             "x": 0,
-            "y": 18
+            "y": 32
           },
           "id": 18,
           "options": {
@@ -1410,7 +2065,7 @@
             "textMode": "auto",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
           "targets": [
             {
               "datasource": {
@@ -1522,7 +2177,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   }
                 ]
               }
@@ -1533,7 +2189,7 @@
             "h": 5,
             "w": 8,
             "x": 8,
-            "y": 18
+            "y": 32
           },
           "id": 17,
           "options": {
@@ -1555,7 +2211,7 @@
             "textMode": "value_and_name",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
           "targets": [
             {
               "datasource": {
@@ -1669,7 +2325,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   }
                 ]
               }
@@ -1680,7 +2337,7 @@
             "h": 5,
             "w": 8,
             "x": 16,
-            "y": 18
+            "y": 32
           },
           "id": 19,
           "options": {
@@ -1702,7 +2359,7 @@
             "textMode": "auto",
             "wideLayout": true
           },
-          "pluginVersion": "10.2.2",
+          "pluginVersion": "10.2.3",
           "targets": [
             {
               "datasource": {
@@ -1792,7 +2449,6 @@
             "uid": "PBFA97CFB590B2093"
           },
           "description": "Total number of subgraph entities loaded in event processing",
-          "hide": true,
           "fieldConfig": {
             "defaults": {
               "color": {
@@ -1836,7 +2492,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   }
                 ]
               }
@@ -1847,8 +2504,9 @@
             "h": 6,
             "w": 8,
             "x": 0,
-            "y": 23
+            "y": 37
           },
+          "hide": true,
           "id": 8,
           "options": {
             "legend": {
@@ -1932,7 +2590,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   }
                 ]
               }
@@ -1943,7 +2602,7 @@
             "h": 6,
             "w": 8,
             "x": 8,
-            "y": 23
+            "y": 37
           },
           "id": 9,
           "options": {
@@ -2028,7 +2687,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   }
                 ]
               }
@@ -2039,7 +2699,7 @@
             "h": 6,
             "w": 8,
             "x": 16,
-            "y": 23
+            "y": 37
           },
           "id": 10,
           "options": {
@@ -2084,7 +2744,7 @@
     }
   ],
   "refresh": "10s",
-  "schemaVersion": 38,
+  "schemaVersion": 39,
   "tags": [
     "watcher"
   ],
@@ -2182,6 +2842,6 @@
   "timepicker": {},
   "timezone": "",
   "title": "Watchers",
-  "version": 1,
+  "version": 3,
   "weekStart": ""
 }

stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml

@@ -3,8 +3,13 @@ global:
   evaluation_interval: 15s
 
 rule_files:
-  # - "first.rules"
+  - rules.yml
-  # - "second.rules"
+
+alerting:
+  alertmanagers:
+   - static_configs:
+     - targets:
+        - alert-manager:9093
 
 scrape_configs:
   - job_name: prometheus
@@ -65,3 +70,12 @@ scrape_configs:
         target_label: instance
       - target_label: __address__
         replacement: postgres-exporter:9187
+
+  - job_name: laconicd
+    metrics_path: /metrics
+    scrape_interval: 30s
+    static_configs:
+      # Add laconicd REST endpoint target with host and port (1317)
+      # - targets: ['example-host:1317']
+    params:
+      format: ['prometheus']

stack_orchestrator/data/config/monitoring/prometheus/rules.yml

@@ -0,0 +1,10 @@
+groups:
+  - name: WatcherAlerts
+    rules:
+      - alert: SushiswapWatcherHeadTracking
+        expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"} > 16
+        for: 5m
+        labels:
+          severity: 'critical'
+        annotations:
+          summary: 'Watcher {{ $labels.instance }} of job {{ $labels.job }} is falling behind external head'

stack_orchestrator/data/config/monitoring/subgraph-alert-rules.yml

@@ -0,0 +1,64 @@
+apiVersion: 1
+groups:
+  - orgId: 1
+    name: subgraph
+    folder: SubgraphAlerts
+    interval: 30s
+    rules:
+      - uid: b2a9144b-6104-46fc-92b5-352f4e643c4c
+        title: subgraph_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                  type: prometheus
+                  uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: ethereum_chain_head_number - on(network) group_right deployment_head{deployment=~"REPLACE_WITH_SUBGRAPH_IDS"}
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                  - evaluator:
+                      params:
+                          - 15
+                          - 0
+                      type: gt
+                    operator:
+                      type: and
+                    query:
+                      params: []
+                    reducer:
+                      params: []
+                      type: avg
+                    type: query
+              datasource:
+                  name: Expression
+                  type: __expr__
+                  uid: __expr__
+              expression: diff
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: threshold
+        noDataState: OK
+        execErrState: Alerting
+        for: 5m
+        annotations:
+          summary: Subgraph deployment {{ index $labels "deployment" }} is falling behind head by {{ index $values "diff" }}
+        labels: {}
+        isPaused: false

stack_orchestrator/data/config/monitoring/update-grafana-alerts-config.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo Using CERC_GRAFANA_ALERTS_SUBGRAPH_IDS ${CERC_GRAFANA_ALERTS_SUBGRAPH_IDS}
+
+# Replace subgraph ids in subgraph alerting config
+# Note: Requires the grafana container to be run with user root
+sed -i "s/REPLACE_WITH_SUBGRAPH_IDS/$CERC_GRAFANA_ALERTS_SUBGRAPH_IDS/g" /etc/grafana/provisioning/alerting/subgraph-alert-rules.yml

stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml

@@ -50,22 +50,6 @@ groups:
               legendFormat: __auto
               range: false
               refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
           - refId: condition
             relativeTimeRange:
               from: 600
@@ -142,22 +126,6 @@ groups:
               legendFormat: __auto
               range: false
               refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
           - refId: condition
             relativeTimeRange:
               from: 600
@@ -234,22 +202,6 @@ groups:
               legendFormat: __auto
               range: false
               refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
           - refId: condition
             relativeTimeRange:
               from: 600
@@ -326,22 +278,6 @@ groups:
               legendFormat: __auto
               range: false
               refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
           - refId: condition
             relativeTimeRange:
               from: 600
@@ -418,22 +354,6 @@ groups:
               legendFormat: __auto
               range: false
               refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
           - refId: condition
             relativeTimeRange:
               from: 600
@@ -510,22 +430,6 @@ groups:
               legendFormat: __auto
               range: false
               refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
           - refId: condition
             relativeTimeRange:
               from: 600
@@ -602,22 +506,6 @@ groups:
               legendFormat: __auto
               range: false
               refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
           - refId: condition
             relativeTimeRange:
               from: 600
@@ -694,22 +582,6 @@ groups:
               legendFormat: __auto
               range: false
               refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
           - refId: condition
             relativeTimeRange:
               from: 600
@@ -788,22 +660,6 @@ groups:
               legendFormat: __auto
               range: false
               refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
           - refId: condition
             relativeTimeRange:
               from: 600
@@ -880,22 +736,84 @@ groups:
               legendFormat: __auto
               range: false
               refId: latest_external
-          - refId: latest_indexed
+          - refId: condition
-            relativeTimeRange:
+            relativeTimeRange:
-              from: 600
+              from: 600
-              to: 0
+              to: 0
-            datasourceUid: PBFA97CFB590B2093
+            datasourceUid: __expr__
-            model:
+            model:
-              datasource:
+              conditions:
-                type: prometheus
+                - evaluator:
-                uid: PBFA97CFB590B2093
+                    params:
-              editorMode: code
+                      - 0
-              expr: sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
+                      - 0
-              hide: false
+                    type: gt
-              instant: true
+                  operator:
-              legendFormat: __auto
+                    type: and
-              range: false
+                  query:
-              refId: latest_indexed
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+
+      # Ajna
+      - uid: ajna_diff_external
+        title: ajna_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="ajna", instance="ajna", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
           - refId: condition
             relativeTimeRange:
               from: 600

stack_orchestrator/data/config/watcher-ajna/start-job-runner.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+set -u
+
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
+
+# Read in the config template TOML file and modify it
+WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
+WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
+
+# Write the modified content to a new file
+echo "$WATCHER_CONFIG" > environments/local.toml
+
+echo "Running job-runner..."
+DEBUG=vulcanize:* exec node --enable-source-maps dist/job-runner.js

stack_orchestrator/data/config/watcher-ajna/start-server.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+set -u
+
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
+
+# Read in the config template TOML file and modify it
+WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
+WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
+
+# Write the modified content to a new file
+echo "$WATCHER_CONFIG" > environments/local.toml
+
+echo "Running server..."
+DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js

stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml

@@ -0,0 +1,101 @@
+[server]
+  host = "0.0.0.0"
+  port = 3008
+  kind = "active"
+  gqlPath = "/"
+
+  # Checkpointing state.
+  checkpointing = true
+
+  # Checkpoint interval in number of blocks.
+  checkpointInterval = 2000
+
+  # Enable state creation
+  # CAUTION: Disable only if state creation is not desired or can be filled subsequently
+  enableState = false
+
+  subgraphPath = "./subgraph-build"
+
+  # Interval to restart wasm instance periodically
+  wasmRestartBlocksInterval = 20
+
+  # Interval in number of blocks at which to clear entities cache.
+  clearEntitiesCacheInterval = 1000
+
+  # Max block range for which to return events in eventsInRange GQL query.
+  # Use -1 for skipping check on block range.
+  maxEventsBlockRange = 1000
+
+  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
+  rpcSupportsBlockHashParam = false
+
+  # GQL cache settings
+  [server.gqlCache]
+    enabled = true
+
+    # Max in-memory cache size (in bytes) (default 8 MB)
+    # maxCacheSize
+
+    # GQL cache-control max-age settings (in seconds)
+    maxAge = 15
+    timeTravelMaxAge = 86400 # 1 day
+
+[metrics]
+  host = "0.0.0.0"
+  port = 9000
+  [metrics.gql]
+    port = 9001
+
+[database]
+  type = "postgres"
+  host = "ajna-watcher-db"
+  port = 5432
+  database = "ajna-watcher"
+  username = "vdbm"
+  password = "password"
+  synchronize = true
+  logging = false
+
+[upstream]
+  [upstream.ethServer]
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS
+
+    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
+    rpcClient = true
+
+    # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint
+    isFEVM = true
+
+    # Boolean flag to filter event logs by contracts
+    filterLogsByAddresses = true
+    # Boolean flag to filter event logs by topics
+    filterLogsByTopics = true
+
+  [upstream.cache]
+    name = "requests"
+    enabled = false
+    deleteOnStart = false
+
+[jobQueue]
+  dbConnectionString = "postgres://vdbm:password@ajna-watcher-db/ajna-watcher-job-queue"
+  maxCompletionLagInSecs = 300
+  jobDelayInMilliSecs = 100
+  eventsInBatch = 50
+  subgraphEventsOrder = true
+  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
+  blockDelayInMilliSecs = 30000
+
+  # Boolean to switch between modes of processing events when starting the server.
+  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
+  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
+  useBlockRanges = true
+
+  # Block range in which logs are fetched during historical blocks processing
+  historicalLogsBlockRange = 2000
+
+  # Max block range of historical processing after which it waits for completion of events processing
+  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
+  historicalMaxFetchAhead = 10000
+
+  # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
+  maxNewBlockRetries = 3

stack_orchestrator/data/config/watcher-azimuth/start-job-runner.sh

@@ -4,16 +4,19 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
   set -x
 fi
 
-echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
-echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}"
+echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL_ENDPOINT}"
 echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}"
 
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 # Replace env variables in template TOML file
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}|g; \
-          s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \
+          s|REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT|${CERC_IPLD_ETH_GQL_ENDPOINT}|g; \
           s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ")
 
 # Write the modified content to a new file

stack_orchestrator/data/config/watcher-azimuth/start-server.sh

@@ -4,16 +4,19 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
   set -x
 fi
 
-echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
-echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}"
+echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL_ENDPOINT}"
 echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}"
 
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 # Replace env variables in template TOML file
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}|g; \
-          s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \
+          s|REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT|${CERC_IPLD_ETH_GQL_ENDPOINT}|g; \
           s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ")
 
 # Write the modified content to a new file

stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml

@@ -13,8 +13,8 @@
 
 [upstream]
   [upstream.ethServer]
-    gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL"
+    gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT"
-    rpcProviderEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_RPC"
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS
 
 [jobQueue]
   historicalLogsBlockRange = REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE

stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-job-runner.sh

@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u
 
-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
 
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
 
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml

stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-server.sh

@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u
 
-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
 
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
 
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml

stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml

@@ -58,7 +58,7 @@
 
 [upstream]
   [upstream.ethServer]
-    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS
 
     # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
     rpcClient = true
@@ -69,7 +69,7 @@
     # Boolean flag to filter event logs by contracts
     filterLogsByAddresses = true
     # Boolean flag to filter event logs by topics
-    filterLogsByTopics = false
+    filterLogsByTopics = true
 
   [upstream.cache]
     name = "requests"
@@ -96,3 +96,6 @@
   # Max block range of historical processing after which it waits for completion of events processing
   # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
   historicalMaxFetchAhead = 10000
+
+  # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
+  maxNewBlockRetries = 3

stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh

@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u
 
-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
 
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
 
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml

stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh

@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u
 
-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
 
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
 
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml

stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml

@@ -58,7 +58,7 @@
 
 [upstream]
   [upstream.ethServer]
-    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS
 
     # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
     rpcClient = true
@@ -96,3 +96,6 @@
   # Max block range of historical processing after which it waits for completion of events processing
   # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
   historicalMaxFetchAhead = 10000
+
+  # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
+  maxNewBlockRetries = 3

stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/Dockerfile

@@ -1,26 +1,23 @@
-FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen
+FROM ethpandaops/ethereum-genesis-generator:3.0.0 AS ethgen
 
 FROM golang:1.20-alpine as builder
 
-RUN apk add --no-cache python3 py3-pip
+RUN apk add --no-cache python3 py3-pip make bash envsubst jq
-
-COPY genesis /opt/genesis
 
 # Install ethereum-genesis-generator tools
-COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/
-COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/
 COPY --from=ethgen /apps /apps
 RUN cd /apps/el-gen && pip3 install --break-system-packages -r requirements.txt
-# web3==5.24.0 used by el-gen is broken on python 3.11
+RUN pip3 install --break-system-packages --upgrade "web3==v6.15.1"
-RUN pip3 install --break-system-packages --upgrade "web3==6.5.0"
 RUN pip3 install --break-system-packages --upgrade "typing-extensions"
 
+# Install tool to generate initial block
+RUN go install github.com/cerc-io/eth-dump-genblock@b29516740fc01cf1d1d623acbfd0e9a2b6440a96
+
 # Build genesis config
-RUN apk add --no-cache make bash envsubst jq
+COPY genesis /opt/genesis
 RUN cd /opt/genesis && make genesis-el
 
 # Snag the genesis block info.
-RUN go install github.com/cerc-io/eth-dump-genblock@latest
 RUN eth-dump-genblock /opt/genesis/build/el/geth.json > /opt/genesis/build/el/genesis_block.json
 
 FROM alpine:latest

stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/genesis/el/build_el.sh

@@ -9,32 +9,8 @@ mkdir -p ../build/el
 tmp_dir=$(mktemp -d -t ci-XXXXXXXXXX)
 envsubst < el-config.yaml > $tmp_dir/genesis-config.yaml
 
-ttd=`cat $tmp_dir/genesis-config.yaml | grep terminal_total_difficulty | awk '{ print $2 }'`
-homestead_block=`cat $tmp_dir/genesis-config.yaml | grep homestead_block | awk '{ print $2 }'`
-eip150_block=`cat $tmp_dir/genesis-config.yaml | grep eip150_block | awk '{ print $2 }'`
-eip155_block=`cat $tmp_dir/genesis-config.yaml | grep eip155_block | awk '{ print $2 }'`
-eip158_block=`cat $tmp_dir/genesis-config.yaml | grep eip158_block | awk '{ print $2 }'`
-byzantium_block=`cat $tmp_dir/genesis-config.yaml | grep byzantium_block | awk '{ print $2 }'`
-constantinople_block=`cat $tmp_dir/genesis-config.yaml | grep constantinople_block | awk '{ print $2 }'`
-petersburg_block=`cat $tmp_dir/genesis-config.yaml | grep petersburg_block | awk '{ print $2 }'`
-istanbul_block=`cat $tmp_dir/genesis-config.yaml | grep istanbul_block | awk '{ print $2 }'`
-berlin_block=`cat $tmp_dir/genesis-config.yaml | grep berlin_block | awk '{ print $2 }'`
-london_block=`cat $tmp_dir/genesis-config.yaml | grep london_block | awk '{ print $2 }'`
-merge_fork_block=`cat $tmp_dir/genesis-config.yaml | grep merge_fork_block | awk '{ print $2 }'`
-
 python3 /apps/el-gen/genesis_geth.py $tmp_dir/genesis-config.yaml | \
-  jq ".config.terminalTotalDifficulty=$ttd" | \
+  jq 'del(.config.pragueTime)' \
-  jq ".config.homesteadBlock=$homestead_block" | \
-  jq ".config.eip150Block=$eip150_block" | \
-  jq ".config.eip155Block=$eip155_block" | \
-  jq ".config.eip158Block=$eip158_block" | \
-  jq ".config.byzantiumBlock=$byzantium_block" | \
-  jq ".config.constantinopleBlock=$constantinople_block" | \
-  jq ".config.petersburgBlock=$petersburg_block" | \
-  jq ".config.istanbulBlock=$istanbul_block" | \
-  jq ".config.berlinBlock=$berlin_block" | \
-  jq ".config.londonBlock=$london_block" | \
-  jq ".config.mergeForkBlock=$merge_fork_block" | \
-  jq ".config.mergeNetsplitBlock=$merge_fork_block" \
   > ../build/el/geth.json
+
 python3 ../accounts/mnemonic_to_csv.py $tmp_dir/genesis-config.yaml > ../build/el/accounts.csv

stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/genesis/el/el-config.yaml

@@ -10,22 +10,8 @@ el_premine_addrs: {}
 chain_id: 1212
 deposit_contract_address: "0x1212121212121212121212121212121212121212"
 genesis_timestamp: 0
-terminal_total_difficulty: 1000
+genesis_delay: 0
-homestead_block: 1
+deneb_fork_epoch: 0
-eip150_block: 1
+# note: only needed as workaround https://github.com/ethpandaops/ethereum-genesis-generator/pull/105
-eip155_block: 1
+electra_fork_epoch: 0
-eip158_block: 1
+slot_duration_in_seconds: 3
-byzantium_block: 1
-constantinople_block: 1
-petersburg_block: 1
-istanbul_block: 1
-berlin_block: 1
-london_block: 1
-merge_fork_block: 1
-
-clique:
-  enabled: false
-  signers:
-    - 36d56343bc308d4ffaac2f793d121aba905fa6cc
-    - 5e762d4a3847cadaf40a4b0c39574b0ff6698c78
-    - 15d7acc1019fdf8ab4f0f7bd31ec1487ecb5a2bd

stack_orchestrator/data/container-build/cerc-fixturenet-eth-geth/run-el.sh

@@ -6,7 +6,7 @@ fi
 
 ETHERBASE=`cat /opt/testnet/build/el/accounts.csv | head -1 | cut -d',' -f2`
 NETWORK_ID=`cat /opt/testnet/el/el-config.yaml | grep 'chain_id' | awk '{ print $2 }'`
-NETRESTRICT=`ip addr | grep inet | grep -v '127.0' | awk '{print $2}'`
+NETRESTRICT=`ip addr | grep -w inet | grep -v '127.0' | awk '{print $2}'`
 CERC_ETH_DATADIR="${CERC_ETH_DATADIR:-$HOME/ethdata}"
 CERC_PLUGINS_DIR="${CERC_PLUGINS_DIR:-/usr/local/lib/plugeth}"
 
@@ -102,6 +102,13 @@ else
       fi
     fi
 
+    OTHER_OPTS=""
+    # miner options were removed in v1.12
+    GETH_VERSION=$(geth --version | grep -io '[0-9][0-9a-z.-]*')
+    if echo -e "$GETH_VERSION\n1.12" | sort -Vc; then
+      OTHER_OPTS="--miner.threads=1"
+    fi
+
     $START_CMD \
       --datadir="${CERC_ETH_DATADIR}" \
       --bootnodes="${ENODE}" \
@@ -126,12 +133,12 @@ else
       --cache.preimages \
       --syncmode=full \
       --mine \
-      --miner.threads=1 \
       --metrics \
       --metrics.addr="0.0.0.0" \
       --verbosity=${CERC_GETH_VERBOSITY:-3} \
       --log.vmodule="${CERC_GETH_VMODULE:-statediff/*=5}" \
       --miner.etherbase="${ETHERBASE}" \
+      ${OTHER_OPTS} \
       ${STATEDIFF_OPTS} \
       &
 

stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/Dockerfile

@@ -1,5 +1,4 @@
 FROM cerc/lighthouse-cli:local AS lcli
-FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen
 FROM cerc/fixturenet-eth-genesis:local AS fnetgen
 
 FROM cerc/lighthouse:local
@@ -12,16 +11,13 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-reco
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 
-COPY genesis /opt/testnet
-COPY run-cl.sh /opt/testnet/run.sh
-
 COPY --from=lcli /usr/local/bin/lcli /usr/local/bin/lcli
-COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/eth2-testnet-genesis
-COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/eth2-val-tools
-COPY --from=ethgen /apps /apps
 COPY --from=fnetgen /opt/genesis/el /opt/testnet/el
 COPY --from=fnetgen /opt/genesis/build/el /opt/testnet/build/el
 
+COPY genesis /opt/testnet
+COPY run-cl.sh /opt/testnet/run.sh
+
 RUN cd /opt/testnet && make genesis-cl
 
 # Work around some bugs in lcli where the default path is always used.

stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/beacon_node.sh

@@ -10,7 +10,6 @@ set -Eeuo pipefail
 source ./vars.env
 
 SUBSCRIBE_ALL_SUBNETS=
-DEBUG_LEVEL=${DEBUG_LEVEL:-debug}
 
 # Get positional arguments
 data_dir=$DATADIR/node_${NODE_NUMBER}

stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/bootnode.sh

@@ -9,8 +9,6 @@ set -Eeuo pipefail
 
 source ./vars.env
 
-DEBUG_LEVEL=${1:-info}
-
 echo "Starting bootnode"
 
 # Clean up existing ENR dir to avoid node connectivity issues on a restart

stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/build_cl.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+# See https://github.com/sigp/lighthouse/scripts/local_testnet/setup.sh
 #
 # Deploys the deposit contract and makes deposits for $VALIDATOR_COUNT insecure deterministic validators.
 # Produces a testnet specification and a genesis state where the genesis time
@@ -24,23 +25,26 @@ echo "(Note: errors of the form 'WARN: Scrypt parameters are too weak...' below
 lcli \
   new-testnet \
   --spec $SPEC_PRESET \
-  --deposit-contract-address $ETH1_DEPOSIT_CONTRACT_ADDRESS \
   --testnet-dir $TESTNET_DIR \
+  --deposit-contract-address $ETH1_DEPOSIT_CONTRACT_ADDRESS \
   --min-genesis-active-validator-count $GENESIS_VALIDATOR_COUNT \
   --validator-count $VALIDATOR_COUNT \
   --min-genesis-time $GENESIS_TIME \
   --genesis-delay $GENESIS_DELAY \
   --genesis-fork-version $GENESIS_FORK_VERSION \
   --altair-fork-epoch $ALTAIR_FORK_EPOCH \
-  --bellatrix-fork-epoch $MERGE_FORK_EPOCH \
+  --bellatrix-fork-epoch $BELLATRIX_FORK_EPOCH \
+  --capella-fork-epoch $CAPELLA_FORK_EPOCH \
+  --deneb-fork-epoch $DENEB_FORK_EPOCH \
   --eth1-id $ETH1_CHAIN_ID \
   --eth1-block-hash $ETH1_BLOCK_HASH \
   --eth1-follow-distance 1 \
   --seconds-per-slot $SECONDS_PER_SLOT \
   --seconds-per-eth1-block $SECONDS_PER_ETH1_BLOCK \
+  --interop-genesis-state \
   --force
 
-echo Specification generated at $TESTNET_DIR.
+echo Specification and genesis.ssz generated at $TESTNET_DIR.
 echo "Generating $VALIDATOR_COUNT validators concurrently... (this may take a while)"
 
 lcli \
@@ -50,13 +54,3 @@ lcli \
   --node-count $BN_COUNT
 
 echo Validators generated with keystore passwords at $DATADIR.
-echo "Building genesis state... (this might take a while)"
-
-lcli \
-  interop-genesis \
-  --spec $SPEC_PRESET \
-  --genesis-time $GENESIS_TIME \
-  --testnet-dir $TESTNET_DIR \
-  $GENESIS_VALIDATOR_COUNT
-
-echo Created genesis state in $TESTNET_DIR

stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/validator_client.sh

@@ -8,8 +8,6 @@ set -Eeuo pipefail
 
 source ./vars.env
 
-DEBUG_LEVEL=info
-
 BUILDER_PROPOSALS=
 
 # Get options

stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/vars.env

@@ -25,7 +25,9 @@ BOOTNODE_PORT=${BOOTNODE_PORT:-4242}
 
 # Hard fork configuration
 ALTAIR_FORK_EPOCH=${ALTAIR_FORK_EPOCH:-0}
-MERGE_FORK_EPOCH=${MERGE_FORK_EPOCH:-0}
+BELLATRIX_FORK_EPOCH=${BELLATRIX_FORK_EPOCH:-0}
+CAPELLA_FORK_EPOCH=${CAPELLA_FORK_EPOCH:-0}
+DENEB_FORK_EPOCH=${DENEB_FORK_EPOCH:-0}
 
 # Spec version (mainnet or minimal)
 SPEC_PRESET=${SPEC_PRESET:-mainnet}
@@ -51,3 +53,6 @@ ETH1_TTD=${ETH1_TTD:-`cat $ETH1_GENESIS_JSON | jq -r '.config.terminalTotalDiffi
 ETH1_DEPOSIT_CONTRACT_ADDRESS=${ETH1_DEPOSIT_CONTRACT_ADDRESS:-`cat $ETH1_CONFIG_YAML | grep 'deposit_contract_address' | awk '{ print $2 }' | sed 's/"//g'`}
 ETH1_DEPOSIT_CONTRACT_BLOCK=${ETH1_DEPOSIT_CONTRACT_BLOCK:-0x0}
 SUGGESTED_FEE_RECIPIENT=`cat ../build/el/accounts.csv | head -1 | cut -d',' -f2`
+
+# --debug-level
+DEBUG_LEVEL=${LIGHTHOUSE_DEBUG_LEVEL:-debug}

stack_orchestrator/data/container-build/cerc-lighthouse/Dockerfile

@@ -1,9 +1,10 @@
-ARG TAG_SUFFIX="-modern"
+FROM sigp/lighthouse:v5.1.2
-FROM sigp/lighthouse:v4.3.0${TAG_SUFFIX}
 
-RUN apt-get update; apt-get install bash netcat curl less jq wget -y;
+RUN apt-get update && apt-get -y upgrade \
+    && apt-get -y install bash netcat curl less jq wget \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
 
-WORKDIR /root/
+WORKDIR /root
 ADD start-lighthouse.sh .
 
 ENTRYPOINT [ "./start-lighthouse.sh" ]

stack_orchestrator/data/container-build/cerc-lighthouse/build.sh

@@ -6,4 +6,4 @@ source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 # See: https://stackoverflow.com/a/246128/1701505
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-docker build -t cerc/lighthouse:local ${build_command_args} --build-arg TAG_SUFFIX="" ${SCRIPT_DIR}
+docker build -t cerc/lighthouse:local ${build_command_args} ${SCRIPT_DIR}

stack_orchestrator/data/container-build/cerc-nextjs-base/Dockerfile

@@ -26,6 +26,8 @@ RUN \
     && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \
     # Install semver
     && su ${USERNAME} -c "umask 0002 && npm install -g semver" \
+    # Install pnpm
+    && su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \
     && npm cache clean --force > /dev/null 2>&1
 
 # [Optional] Uncomment this section to install additional OS packages.
@@ -35,6 +37,9 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
 # [Optional] Uncomment if you want to install more global node modules
 # RUN su node -c "npm install -g <your-package-list-here>"
 
+# We do this to get a yq binary from the published container, for the correct architecture we're building here
+COPY --from=docker.io/mikefarah/yq:latest /usr/bin/yq /usr/local/bin/yq
+
 # Expose port for http
 EXPOSE 80
 

stack_orchestrator/data/container-build/cerc-nextjs-base/build.sh

@@ -11,8 +11,14 @@ CERC_CONTAINER_BUILD_DOCKERFILE=${CERC_CONTAINER_BUILD_DOCKERFILE:-$SCRIPT_DIR/D
 CERC_CONTAINER_BUILD_TAG=${CERC_CONTAINER_BUILD_TAG:-cerc/nextjs-base:local}
 
 docker build -t $CERC_CONTAINER_BUILD_TAG ${build_command_args} -f $CERC_CONTAINER_BUILD_DOCKERFILE $CERC_CONTAINER_BUILD_WORK_DIR
+rc=$?
 
-if [ $? -eq 0 ] && [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/nextjs-base:local" ]; then
+if [ $rc -ne 0 ]; then
+  echo "BUILD FAILED" 1>&2
+  exit $rc
+fi
+
+if [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/nextjs-base:local" ]; then
   cat <<EOF
 
 #################################################################

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/apply-runtime-env.sh

@@ -10,10 +10,12 @@ TRG_DIR="${3:-.next-r}"
 
 CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
 if [ -z "$CERC_BUILD_TOOL" ]; then
-  if [ -f "yarn.lock" ]; then
+  if [ -f "pnpm-lock.yaml" ]; then
-    CERC_BUILD_TOOL=npm
+    CERC_BUILD_TOOL=pnpm
-  else
+  elif [ -f "yarn.lock" ]; then
     CERC_BUILD_TOOL=yarn
+  else
+    CERC_BUILD_TOOL=npm
   fi
 fi
 

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/build-app.sh

@@ -9,7 +9,9 @@ CERC_MIN_NEXTVER=13.4.2
 CERC_NEXT_VERSION="${CERC_NEXT_VERSION:-keep}"
 CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
 if [ -z "$CERC_BUILD_TOOL" ]; then
-  if [ -f "yarn.lock" ]; then
+  if [ -f "pnpm-lock.yaml" ]; then
+    CERC_BUILD_TOOL=pnpm
+  elif [ -f "yarn.lock" ]; then
     CERC_BUILD_TOOL=yarn
   else
     CERC_BUILD_TOOL=npm
@@ -113,7 +115,7 @@ if [ "$CERC_NEXT_VERSION" != "keep" ] && [ "$CUR_NEXT_VERSION" != "$CERC_NEXT_VE
   mv package.json.$$ package.json
 fi
 
-$CERC_BUILD_TOOL install || exit 1
+time $CERC_BUILD_TOOL install || exit 1
 
 CUR_NEXT_VERSION=`jq -r '.version' node_modules/next/package.json`
 
@@ -136,9 +138,9 @@ to use for the build with:
 EOF
   cat package.json | jq ".dependencies.next = \"^$CERC_MIN_NEXTVER\"" > package.json.$$
   mv package.json.$$ package.json
-  $CERC_BUILD_TOOL install || exit 1
+  time $CERC_BUILD_TOOL install || exit 1
 fi
 
-$CERC_BUILD_TOOL run cerc_compile || exit 1
+time $CERC_BUILD_TOOL run cerc_compile || exit 1
 
 exit 0

stack_orchestrator/data/container-build/cerc-nextjs-base/scripts/start-serving-app.sh

@@ -16,7 +16,9 @@ trap ctrl_c INT
 
 CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
 if [ -z "$CERC_BUILD_TOOL" ]; then
-  if [ -f "yarn.lock" ] && [ ! -f "package-lock.json" ]; then
+  if [ -f "pnpm-lock.yaml" ]; then
+    CERC_BUILD_TOOL=pnpm
+  elif [ -f "yarn.lock" ]; then
     CERC_BUILD_TOOL=yarn
   else
     CERC_BUILD_TOOL=npm

stack_orchestrator/data/container-build/cerc-snowballtools-base-backend/Dockerfile

@@ -0,0 +1,6 @@
+FROM cerc/snowballtools-base-backend-base:local
+
+WORKDIR /app/packages/backend
+COPY run.sh .
+
+ENTRYPOINT ["./run.sh"]

stack_orchestrator/data/container-build/cerc-snowballtools-base-backend/Dockerfile-base

@@ -0,0 +1,26 @@
+FROM ubuntu:22.04 as builder
+
+RUN apt update && \
+      apt install -y --no-install-recommends --no-install-suggests \
+      ca-certificates curl gnupg
+
+# Node
+ARG NODE_MAJOR=20
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
+      echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \
+      apt update && apt install -y nodejs
+
+# npm setup
+RUN npm config set @cerc-io:registry https://git.vdb.to/api/packages/cerc-io/npm/ && npm install -g yarn
+
+COPY . /app/
+WORKDIR /app/
+
+RUN find . -name 'node_modules' | xargs -n1 rm -rf
+RUN yarn && yarn build --ignore frontend
+
+FROM cerc/webapp-base:local
+
+COPY --from=builder /app /app
+
+WORKDIR /app/packages/backend

stack_orchestrator/data/container-build/cerc-snowballtools-base-backend/build.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# Build cerc/webapp-deployer-backend
+
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/snowballtools-base-backend-base:local ${build_command_args} -f ${SCRIPT_DIR}/Dockerfile-base ${CERC_REPO_BASE_DIR}/snowballtools-base
+docker build -t cerc/snowballtools-base-backend:local ${build_command_args} ${SCRIPT_DIR} 

stack_orchestrator/data/container-build/cerc-snowballtools-base-backend/run.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+
+LACONIC_HOSTED_CONFIG_FILE=${LACONIC_HOSTED_CONFIG_FILE}
+if [ -z "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
+  if [ -f "/config/laconic-hosted-config.yml" ]; then
+    LACONIC_HOSTED_CONFIG_FILE="/config/laconic-hosted-config.yml"
+  elif [ -f "/config/config.yml" ]; then
+    LACONIC_HOSTED_CONFIG_FILE="/config/config.yml"
+  fi
+fi
+
+if [ -f "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
+  /scripts/apply-webapp-config.sh $LACONIC_HOSTED_CONFIG_FILE "`pwd`/dist"
+fi
+
+/scripts/apply-runtime-env.sh "`pwd`/dist"
+
+yarn start

stack_orchestrator/data/container-build/cerc-test-container/run.sh

@@ -39,6 +39,15 @@ fi
 if [ -n "$CERC_TEST_PARAM_2" ]; then
   echo "Test-param-2: ${CERC_TEST_PARAM_2}"
 fi
+if [ -n "$CERC_TEST_PARAM_3" ]; then
+  echo "Test-param-3: ${CERC_TEST_PARAM_3}"
+fi
+if [ -n "$CERC_TEST_PARAM_4" ]; then
+  echo "Test-param-4: ${CERC_TEST_PARAM_4}"
+fi
+if [ -n "$CERC_TEST_PARAM_5" ]; then
+  echo "Test-param-5: ${CERC_TEST_PARAM_5}"
+fi
 
 if [ -d "/config" ]; then
   echo "/config: EXISTS"

stack_orchestrator/data/container-build/cerc-watcher-ajna/Dockerfile

@@ -0,0 +1,10 @@
+FROM node:18.17.1-alpine3.18
+
+RUN apk --update --no-cache add git python3 alpine-sdk bash curl jq
+
+WORKDIR /app
+
+COPY . .
+
+RUN echo "Installing dependencies and building ajna-watcher-ts" && \
+  yarn && yarn build

stack_orchestrator/data/container-build/cerc-watcher-ajna/build.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# Build cerc/watcher-ajna
+
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/watcher-ajna:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/ajna-watcher-ts

stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile

@@ -1,6 +1,6 @@
 # Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile
 # [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
-ARG VARIANT=20-bullseye
+ARG VARIANT=20-bullseye-slim
 FROM node:${VARIANT}
 
 ARG USERNAME=node
@@ -24,6 +24,10 @@ RUN \
     && su ${USERNAME} -c "npm config -g set prefix ${NPM_GLOBAL}" \
     # Install eslint
     && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \
+    # Install semver
+    && su ${USERNAME} -c "umask 0002 && npm install -g semver" \
+    # Install pnpm
+    && su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \
     && npm cache clean --force > /dev/null 2>&1
 
 # [Optional] Uncomment this section to install additional OS packages.
@@ -43,7 +47,6 @@ COPY scripts /scripts
 # RUN su node -c "npm install -g <your-package-list-here>"
 
 RUN mkdir -p /config
-COPY ./config.yml /config
 
 # Install simple web server for now (use nginx perhaps later)
 RUN yarn global add http-server

stack_orchestrator/data/container-build/cerc-webapp-base/Dockerfile.webapp

@@ -1,11 +1,12 @@
 FROM cerc/webapp-base:local as builder
 
 ARG CERC_BUILD_TOOL
+ARG CERC_BUILD_OUTPUT_DIR
 
 WORKDIR /app
 COPY . .
-RUN rm -rf node_modules build .next*
+RUN rm -rf node_modules build dist .next*
-RUN /scripts/build-app.sh /app build /data
+RUN /scripts/build-app.sh /app /data
 
 FROM cerc/webapp-base:local
 COPY --from=builder /data /data

stack_orchestrator/data/container-build/cerc-webapp-base/build.sh

@@ -11,8 +11,14 @@ CERC_CONTAINER_BUILD_DOCKERFILE=${CERC_CONTAINER_BUILD_DOCKERFILE:-$SCRIPT_DIR/D
 CERC_CONTAINER_BUILD_TAG=${CERC_CONTAINER_BUILD_TAG:-cerc/webapp-base:local}
 
 docker build -t $CERC_CONTAINER_BUILD_TAG ${build_command_args} -f $CERC_CONTAINER_BUILD_DOCKERFILE $CERC_CONTAINER_BUILD_WORK_DIR
+rc=$?
 
-if [ $? -eq 0 ] && [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/webapp-base:local" ]; then
+if [ $rc -ne 0 ]; then
+  echo "BUILD FAILED" 1>&2
+  exit $rc
+fi
+
+if [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/webapp-base:local" ]; then
   cat <<EOF
 
 #################################################################

stack_orchestrator/data/container-build/cerc-webapp-base/config.yml

@@ -1 +0,0 @@
-# Put config here.

stack_orchestrator/data/container-build/cerc-webapp-base/scripts/build-app.sh

@@ -7,27 +7,46 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 
 CERC_BUILD_TOOL="${CERC_BUILD_TOOL}"
-WORK_DIR="${1:-/app}"
+CERC_BUILD_OUTPUT_DIR="${CERC_BUILD_OUTPUT_DIR}"
-OUTPUT_DIR="${2:-build}"
-DEST_DIR="${3:-/data}"
 
-if [ -f "${WORK_DIR}/package.json" ]; then
+WORK_DIR="${1:-/app}"
+DEST_DIR="${2:-/data}"
+
+if [ -f "${WORK_DIR}/build-webapp.sh" ]; then
+  echo "Building webapp with ${WORK_DIR}/build-webapp.sh ..."
+  cd "${WORK_DIR}" || exit 1
+
+  rm -rf "${DEST_DIR}"
+  ./build-webapp.sh "${DEST_DIR}" || exit 1
+elif [ -f "${WORK_DIR}/package.json" ]; then
   echo "Building node-based webapp ..."
   cd "${WORK_DIR}" || exit 1
 
   if [ -z "$CERC_BUILD_TOOL" ]; then
-    if [ -f "yarn.lock" ]; then
+    if [ -f "pnpm-lock.yaml" ]; then
+      CERC_BUILD_TOOL=pnpm
+    elif [ -f "yarn.lock" ]; then
       CERC_BUILD_TOOL=yarn
     else
       CERC_BUILD_TOOL=npm
     fi
   fi
 
-  $CERC_BUILD_TOOL install || exit 1
+  time $CERC_BUILD_TOOL install || exit 1
-  $CERC_BUILD_TOOL build || exit 1
+  time $CERC_BUILD_TOOL build || exit 1
 
   rm -rf "${DEST_DIR}"
-  mv "${WORK_DIR}/${OUTPUT_DIR}" "${DEST_DIR}"
+  if [ -z "${CERC_BUILD_OUTPUT_DIR}" ]; then
+    if [ -d "${WORK_DIR}/dist" ]; then
+      CERC_BUILD_OUTPUT_DIR="${WORK_DIR}/dist"
+    elif [ -d "${WORK_DIR}/build" ]; then
+      CERC_BUILD_OUTPUT_DIR="${WORK_DIR}/build"
+    else
+      echo "ERROR: Unable to locate build output.  Set with --extra-build-args \"--build-arg CERC_BUILD_OUTPUT_DIR=path\"" 1>&2
+      exit 1
+    fi
+  fi
+  mv "${CERC_BUILD_OUTPUT_DIR}" "${DEST_DIR}"
 else
   echo "Copying static app ..."
   mv "${WORK_DIR}" "${DEST_DIR}"