env

Otherwise we sometimes see errors like:

```
cerc-webapp-deployer:   File "/root/.shiv/laconic-so_0f937aa98c2748ef9af8585d6f441dbc01546ace0d6660cbb159d1e5040aeddf/site-packages/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py", line 671, in command
cerc-webapp-deployer:     shutil.rmtree(tempdir)
cerc-webapp-deployer:   File "/usr/lib/python3.10/shutil.py", line 725, in rmtree
cerc-webapp-deployer:     _rmtree_safe_fd(fd, path, onerror)
cerc-webapp-deployer:   File "/usr/lib/python3.10/shutil.py", line 681, in _rmtree_safe_fd
cerc-webapp-deployer:     onerror(os.unlink, fullname, sys.exc_info())
cerc-webapp-deployer:   File "/usr/lib/python3.10/shutil.py", line 679, in _rmtree_safe_fd
cerc-webapp-deployer:     os.unlink(entry.name, dir_fd=topfd)
cerc-webapp-deployer: FileNotFoundError: [Errno 2] No such file or directory: 'S.gpg-agent.extra'
```

Reviewed-on: cerc-io/stack-orchestrator#941
Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com>
Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>

.gitea/workflows/fixturenet-eth-plugeth-arm-test.yml

@@ -1,45 +0,0 @@
-name: Fixturenet-Eth-Plugeth-Arm-Test
-
-on:
-  push:
-    branches: '*'
-    paths:
-      - '!**'
-      - '.gitea/workflows/triggers/fixturenet-eth-plugeth-arm-test'
-  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
-    - cron: '2 14 * * *'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
-
-jobs:
-  test:
-    name: "Run an Ethereum plugeth fixturenet test"
-    runs-on: ubuntu-latest-arm
-    steps:
-      - name: "Clone project repository"
-        uses: actions/checkout@v3
-      # At present the stock setup-python action fails on Linux/aarch64
-      # Conditional steps below workaroud this by using deadsnakes for that case only
-      - name: "Install Python for ARM on Linux"
-        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
-        uses: deadsnakes/action@v3.0.1
-        with:
-          python-version: '3.8'
-      - name: "Install Python cases other than ARM on Linux"
-        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
-      - name: "Print Python version"
-        run: python3 --version
-      - name: "Install shiv"
-        run: pip install shiv
-      - name: "Generate build version file"
-        run: ./scripts/create_build_tag_file.sh
-      - name: "Build local shiv package"
-        run: ./scripts/build_shiv_package.sh
-      - name: "Run fixturenet-eth tests"
-        run: ./tests/fixturenet-eth-plugeth/run-test.sh

.gitea/workflows/fixturenet-eth-test.yml

@@ -1,48 +0,0 @@
-name: Fixturenet-Eth-Test
-
-on:
-  push:
-    branches: '*'
-    paths:
-      - '!**'
-      - '.gitea/workflows/triggers/fixturenet-eth-test'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
-
-jobs:
-  test:
-    name: "Run an Ethereum fixturenet test"
-    runs-on: ubuntu-latest
-    steps:
-      - name: "Clone project repository"
-        uses: actions/checkout@v3
-      # At present the stock setup-python action fails on Linux/aarch64
-      # Conditional steps below workaroud this by using deadsnakes for that case only
-      - name: "Install Python for ARM on Linux"
-        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
-        uses: deadsnakes/action@v3.0.1
-        with:
-          python-version: '3.8'
-      - name: "Install Python cases other than ARM on Linux"
-        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
-      - name: "Print Python version"
-        run: python3 --version
-      - name: "Install shiv"
-        run: pip install shiv
-      - name: "Generate build version file"
-        run: ./scripts/create_build_tag_file.sh
-      - name: "Build local shiv package"
-        run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
-      - name: "Run fixturenet-eth tests"
-        run: ./tests/fixturenet-eth/run-test.sh
-

.gitea/workflows/fixturenet-laconicd-test.yml

@@ -11,7 +11,7 @@ on:
 
 jobs:
   test:
-    name: "Run an Laconicd fixturenet test"
+    name: "Run Laconicd fixturenet and Laconic CLI tests"
     runs-on: ubuntu-latest
     steps:
       - name: 'Update'
@@ -46,3 +46,21 @@ jobs:
         run: ./scripts/build_shiv_package.sh
       - name: "Run fixturenet-laconicd tests"
         run: ./tests/fixturenet-laconicd/run-test.sh
+      - name: "Run laconic CLI tests"
+        run: ./tests/fixturenet-laconicd/run-cli-test.sh
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/lint.yml

@@ -19,3 +19,19 @@ jobs:
           python-version: '3.8'
       - name : "Run flake8"
         uses: py-actions/flake8@v2
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/publish.yml

@@ -54,3 +54,19 @@ jobs:
           # Hack using endsWith to workaround Gitea sometimes sending "publish-test" vs "refs/heads/publish-test"
           draft: ${{ endsWith('publish-test', github.ref ) }}
           files: ./laconic-so
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/test-container-registry.yml

@@ -51,4 +51,19 @@ jobs:
           source /opt/bash-utils/cgroup-helper.sh
           join_cgroup
           ./tests/container-registry/run-test.sh
-
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/test-database.yml

@@ -49,4 +49,19 @@ jobs:
           source /opt/bash-utils/cgroup-helper.sh
           join_cgroup
           ./tests/database/run-test.sh
-
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/test-deploy.yml

@@ -10,9 +10,6 @@ on:
     paths-ignore:
       - '.gitea/workflows/triggers/*'
 
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
 
 jobs:
   test:
@@ -41,9 +38,21 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
       - name: "Run deploy tests"
         run: ./tests/deploy/run-deploy-test.sh
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/test-external-stack.yml

@@ -1,22 +1,19 @@
-name: Fixturenet-Eth-Plugeth-Test
+name: External Stack Test
 
 on:
   push:
     branches: '*'
     paths:
       - '!**'
-      - '.gitea/workflows/triggers/fixturenet-eth-plugeth-test'
+      - '.gitea/workflows/triggers/test-external-stack'
+      - '.gitea/workflows/test-external-stack.yml'
+      - 'tests/external-stack/run-test.sh'
   schedule: # Note: coordinate with other tests to not overload runners at the same time of day
-    - cron: '2 14 * * *'
-
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
+    - cron: '8 19 * * *'
 
 jobs:
   test:
-    name: "Run an Ethereum plugeth fixturenet test"
+    name: "Run external stack test suite"
     runs-on: ubuntu-latest
     steps:
       - name: "Clone project repository"
@@ -41,9 +38,21 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
-      - name: "Run fixturenet-eth tests"
-        run: ./tests/fixturenet-eth-plugeth/run-test.sh
+      - name: "Run external stack tests"
+        run: ./tests/external-stack/run-test.sh
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/test-k8s-deploy.yml

@@ -51,4 +51,19 @@ jobs:
           source /opt/bash-utils/cgroup-helper.sh
           join_cgroup
           ./tests/k8s-deploy/run-deploy-test.sh
-
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/test-k8s-deployment-control.yml

@@ -0,0 +1,69 @@
+name: K8s Deployment Control Test
+
+on:
+  pull_request:
+    branches: '*'
+  push:
+    branches: '*'
+    paths:
+      - '!**'
+      - '.gitea/workflows/triggers/test-k8s-deployment-control'
+      - '.gitea/workflows/test-k8s-deployment-control.yml'
+      - 'tests/k8s-deployment-control/run-test.sh'
+  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
+    - cron: '3 30 * * *'
+
+jobs:
+  test:
+    name: "Run deployment control suite on kind/k8s"
+    runs-on: ubuntu-22.04
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      # At present the stock setup-python action fails on Linux/aarch64
+      # Conditional steps below workaroud this by using deadsnakes for that case only
+      - name: "Install Python for ARM on Linux"
+        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
+        uses: deadsnakes/action@v3.0.1
+        with:
+          python-version: '3.8'
+      - name: "Install Python cases other than ARM on Linux"
+        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: "Check cgroups version"
+        run: mount | grep cgroup
+      - name: "Install kind"
+        run: ./tests/scripts/install-kind.sh
+      - name: "Install Kubectl"
+        run: ./tests/scripts/install-kubectl.sh
+      - name: "Run k8s deployment control test"
+        run: |
+          source /opt/bash-utils/cgroup-helper.sh
+          join_cgroup
+          ./tests/k8s-deployment-control/run-test.sh
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/test-mobymask-v3-demo-arm.yml

@@ -1,44 +0,0 @@
-name: Mobymask Demo Build
-
-on:
-  pull_request:
-    branches: '*'
-  push:
-    branches:
-      - ci-test
-      - dboreham/mobymask-v3-demo-test
-    paths-ignore:
-      - '.gitea/workflows/triggers/*'
-
-jobs:
-  test:
-    name: "Run mobymask-v3-demo test suite"
-    runs-on: ubuntu-latest-arm
-    steps:
-      - name: "Clone project repository"
-        uses: actions/checkout@v3
-      # At present the stock setup-python action fails on Linux/aarch64
-      # Conditional steps below workaroud this by using deadsnakes for that case only
-      - name: "Install Python for ARM on Linux"
-        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
-        uses: deadsnakes/action@v3.0.1
-        with:
-          python-version: '3.8'
-      - name: "Install Python cases other than ARM on Linux"
-        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
-      - name: "Print Python version"
-        run: python3 --version
-      - name: "Install shiv"
-        run: pip install shiv
-      - name: "Generate build version file"
-        run: ./scripts/create_build_tag_file.sh
-      - name: "Build local shiv package"
-        run: ./scripts/build_shiv_package.sh
-      - name: "Login to image registry"
-        run: |
-          echo ${{ secrets.CICD_PUBLISH_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
-      - name: "Run mobymask v3 tests"
-        run: ./tests/mobymask-v3-demo/run-test.sh

.gitea/workflows/test-mobymask-v3-demo.yml

@@ -1,44 +0,0 @@
-name: Mobymask Demo Build
-
-on:
-  pull_request:
-    branches: '*'
-  push:
-    branches:
-      - ci-test
-      - dboreham/mobymask-v3-demo-test
-    paths-ignore:
-      - '.gitea/workflows/triggers/*'
-
-jobs:
-  test:
-    name: "Run mobymask-v3-demo test suite"
-    runs-on: ubuntu-latest
-    steps:
-      - name: "Clone project repository"
-        uses: actions/checkout@v3
-      # At present the stock setup-python action fails on Linux/aarch64
-      # Conditional steps below workaroud this by using deadsnakes for that case only
-      - name: "Install Python for ARM on Linux"
-        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
-        uses: deadsnakes/action@v3.0.1
-        with:
-          python-version: '3.8'
-      - name: "Install Python cases other than ARM on Linux"
-        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
-      - name: "Print Python version"
-        run: python3 --version
-      - name: "Install shiv"
-        run: pip install shiv
-      - name: "Generate build version file"
-        run: ./scripts/create_build_tag_file.sh
-      - name: "Build local shiv package"
-        run: ./scripts/build_shiv_package.sh
-      - name: "Login to image registry"
-        run: |
-          echo ${{ secrets.CICD_PUBLISH_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
-      - name: "Run mobymask v3 tests"
-        run: ./tests/mobymask-v3-demo/run-test.sh

.gitea/workflows/test-webapp.yml

@@ -10,10 +10,6 @@ on:
     paths-ignore:
       - '.gitea/workflows/triggers/*'
 
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
 jobs:
   test:
     name: "Run webapp test suite"
@@ -43,9 +39,21 @@ jobs:
         run: ./scripts/build_shiv_package.sh
       - name: "Install wget" # 20240109 - Only needed until the executors are updated.
         run: apt update && apt install -y wget
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
       - name: "Run webapp tests"
         run: ./tests/webapp-test/run-webapp-test.sh
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/test.yml

@@ -10,9 +10,6 @@ on:
     paths-ignore:
       - '.gitea/workflows/triggers/*'
 
-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
 
 jobs:
   test:
@@ -41,11 +38,21 @@ jobs:
         run: ./scripts/create_build_tag_file.sh
       - name: "Build local shiv package"
         run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
       - name: "Run smoke tests"
         run: ./tests/smoke-test/run-smoke-test.sh
-
-
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/triggers/fixturenet-eth-plugeth-arm-test

@@ -1,2 +0,0 @@
-Change this file to trigger running the fixturenet-eth-plugeth-arm-test CI job
-

.gitea/workflows/triggers/fixturenet-eth-plugeth-test

@@ -1,3 +0,0 @@
-Change this file to trigger running the fixturenet-eth-plugeth-test CI job
-trigger
-trigger

.gitea/workflows/triggers/fixturenet-eth-test

@@ -1,2 +0,0 @@
-Change this file to trigger running the fixturenet-eth-test CI job
-

.gitea/workflows/triggers/fixturenet-laconicd-test

@@ -1,3 +1,10 @@
 Change this file to trigger running the fixturenet-laconicd-test CI job
 Trigger
 Trigger
+Trigger
+Trigger
+Trigger
+Trigger
+Trigger
+Trigger
+Trigger

.gitea/workflows/triggers/test-external-stack

@@ -0,0 +1,2 @@
+Change this file to trigger running the external-stack CI job
+trigger

docs/cli.md

@@ -51,7 +51,7 @@ $ laconic-so build-npms --include <package-name>
 ```
 e.g.
 ```
-$ laconic-so build-npms --include laconic-sdk
+$ laconic-so build-npms --include registry-sdk
 ```
 Build the packages for a stack:
 ```

docs/fetching-containers.md

@@ -0,0 +1,9 @@
+# Fetching pre-built container images
+When Stack Orchestrator deploys a stack containing a suite of one or more containers it expects images for those containers to be on the local machine with a tag of the form `<image-name>:local` Images for these containers can be built from source (and optionally base container images from public registries) with the `build-containers` subcommand. 
+
+However, the task of building a large number of containers from source may consume considerable time and machine resources. This is where the `fetch-containers` subcommand steps in. It is designed to work exactly like `build-containers` but instead the images, pre-built, are fetched from an image registry then re-tagged for deployment. It can be used in place of `build-containers` for any stack provided the necessary containers, built for the local machine architecture (e.g. arm64 or x86-64) have already been published in an image registry.
+## Usage
+To use `fetch-containers`, provide an image registry path, a username and token/password with read access to the registry, and optionally specify `--force-local-overwrite`. If this argument is not specified, if there is already a locally built or previously fetched image for a stack container on the machine, it will not be overwritten and a warning issued.
+```
+$ laconic-so --stack mobymask-v3-demo fetch-containers --image-registry git.vdb.to/cerc-io --registry-username <registry-user> --registry-token <registry-token> --force-local-overwrite
+```

docs/gitea-with-laconicd-fixturenet.md

@@ -56,7 +56,7 @@ laconic-so --stack fixturenet-laconicd build-npms
 Navigate to the Gitea console and switch to the `cerc-io` user then find the `Packages` tab to confirm that these two npm packages have been published:
 
 - `@cerc-io/laconic-registry-cli`
-- `@cerc-io/laconic-sdk`
+- `@cerc-io/registry-sdk`
 
 ### Build and deploy fixturenet containers
 
@@ -74,7 +74,7 @@ laconic-so --stack fixturenet-laconicd deploy logs
 ### Test with the registry CLI
 
 ```bash
-laconic-so --stack fixturenet-laconicd deploy exec cli "laconic cns status"
+laconic-so --stack fixturenet-laconicd deploy exec cli "laconic registry status"
 ```
 
 Try additional CLI commands, documented [here](https://github.com/cerc-io/laconic-registry-cli#operations).

docs/k8s-deployment-enhancements.md

@@ -0,0 +1,27 @@
+# K8S Deployment Enhancements
+## Controlling pod placement
+The placement of pods created as part of a stack deployment can be controlled to either avoid certain nodes, or require certain nodes.
+### Pod/Node Affinity
+Node affinity rules applied to pods target node labels. The effect is that a pod can only be placed on a node having the specified label value. Note that other pods that do not have any node affinity rules can also be placed on those same nodes. Thus node affinity for a pod controls where that pod can be placed, but does not control where other pods are placed.
+
+Node affinity for stack pods is specified in the deployment's `spec.yml` file as follows:
+```
+node-affinities:
+  - label: nodetype
+    value: typeb
+```
+This example denotes that the stack's pods should only be placed on nodes that have the label `nodetype` with value `typeb`.
+### Node Taint Toleration
+K8s nodes can be given one or more "taints". These are special fields (distinct from labels) with a name (key) and optional value.
+When placing pods, the k8s scheduler will only assign a pod to a tainted node if the pod posesses a corresponding "toleration".
+This is metadata associated with the pod that specifies that the pod "tolerates" a given taint.
+Therefore taint toleration provides a mechanism by which only certain pods can be placed on specific nodes, and provides a complementary mechanism to node affinity.
+
+Taint toleration for stack pods is specified in the deployment's `spec.yml` file as follows:
+```
+node-tolerations:
+  - key: nodetype
+    value: typeb
+```
+This example denotes that the stack's pods will tolerate a taint: `nodetype=typeb`
+

docs/laconicd-with-console.md

@@ -1,9 +1,8 @@
 # Running a laconicd fixturenet with console
 
-The following tutorial explains the steps to run a laconicd fixturenet with CLI and web console that displays records in the registry. It is designed as an introduction to Stack Orchestrator and to showcase one component of the Laconic Stack. Prior to Stack Orchestrator, the following 4 repositories had to be cloned and setup manually:
+The following tutorial explains the steps to run a laconicd fixturenet with CLI and web console that displays records in the registry. It is designed as an introduction to Stack Orchestrator and to showcase one component of the Laconic Stack. Prior to Stack Orchestrator, the following repositories had to be cloned and setup manually:
 
 - https://git.vdb.to/cerc-io/laconicd
-- https://git.vdb.to/cerc-io/laconic-sdk
 - https://git.vdb.to/cerc-io/laconic-registry-cli
 - https://git.vdb.to/cerc-io/laconic-console
 
@@ -51,7 +50,7 @@ To avoid hiccups on Mac M1/M2 and any local machine nuances that may affect the
 1. Get the repositories
 
     ```
-    laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd,git.vdb.to/cerc-io/laconic-sdk,git.vdb.to/cerc-io/laconic-registry-cli,git.vdb.to/cerc-io/laconic-console
+    laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd
     ```
 
 1. Build the containers:
@@ -76,6 +75,8 @@ To avoid hiccups on Mac M1/M2 and any local machine nuances that may affect the
 1. Create a deployment directory for the stack:
     ```
     laconic-so --stack fixturenet-laconic-loaded deploy init --output laconic-loaded.spec --map-ports-to-host any-same --config LACONIC_HOSTED_ENDPOINT=$BACKEND_ENDPOINT
+
+    # Update port mapping in the laconic-loaded.spec file to resolve port conflicts on host if any
     ```
     ```
     laconic-so --stack fixturenet-laconic-loaded deploy create --deployment-dir laconic-loaded-deployment --spec-file laconic-loaded.spec
@@ -95,52 +96,51 @@ To avoid hiccups on Mac M1/M2 and any local machine nuances that may affect the
     You'll see output from `laconicd` and the block height should be >1 to confirm it is running:
 
     ```
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:29PM INF indexed block exents height=12 module=txindex server=node
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF Timed out dur=4976.960115 height=13 module=consensus round=0 server=node step=1
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF received proposal module=consensus proposal={"Type":32,"block_id":{"hash":"D26C088A711F912ADB97888C269F628DA33153795621967BE44DCB43C3D03CA4","parts":{"hash":"22411A20B7F14CDA33244420FBDDAF24450C0628C7A06034FF22DAC3699DDCC8","total":1}},"height":13,"pol_round":-1,"round":0,"signature":"DEuqnaQmvyYbUwckttJmgKdpRu6eVm9i+9rQ1pIrV2PidkMNdWRZBLdmNghkIrUzGbW8Xd7UVJxtLRmwRASgBg==","timestamp":"2023-04-18T21:30:01.49450663Z"} server=node
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF received complete proposal block hash=D26C088A711F912ADB97888C269F628DA33153795621967BE44DCB43C3D03CA4 height=13 module=consensus server=node
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF finalizing commit of block hash={} height=13 module=consensus num_txs=0 root=1A8CA1AF139CCC80EC007C6321D8A63A46A793386EE2EDF9A5CA0AB2C90728B7 server=node
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF minted coins from module account amount=2059730459416582643aphoton from=mint module=x/bank
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF executed block height=13 module=state num_invalid_txs=0 num_valid_txs=0 server=node
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF commit synced commit=436F6D6D697449447B5B363520313037203630203232372039352038352032303820313334203231392032303520313433203130372031343920313431203139203139322038362031323720362031383520323533203137362031333820313735203135392031383620323334203135382031323120313431203230342037335D3A447D
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF committed state app_hash=416B3CE35F55D086DBCD8F6B958D13C0567F06B9FDB08AAF9FBAEA9E798DCC49 height=13 module=state num_txs=0 server=node
-    laconic-5cd0a80c1442c3044c8b295d26426bae-laconicd-1         | 9:30PM INF indexed block exents height=13 module=txindex server=node
+    laconicd-1         | 6:12AM INF indexed block events height=16 module=txindex
+    laconicd-1         | 6:12AM INF Timed out dur=2993.893332 height=17 module=consensus round=0 step=RoundStepNewHeight
+    laconicd-1         | 6:12AM INF received proposal module=consensus proposal="Proposal{17/0 (E15D03C180CE607AE8340A1325A0C134DFB4E1ADD992E173C701EBD362523267:1:DF138772FEF0, -1) 6A6F3B0A42B3 @ 2024-07-25T06:12:31.952967053Z}" proposer=86970D950BC9C16F3991A52D9C6DC55BA478A7C6
+    laconicd-1         | 6:12AM INF received complete proposal block hash=E15D03C180CE607AE8340A1325A0C134DFB4E1ADD992E173C701EBD362523267 height=17 module=consensus
+    laconicd-1         | 6:12AM INF finalizing commit of block hash=E15D03C180CE607AE8340A1325A0C134DFB4E1ADD992E173C701EBD362523267 height=17 module=consensus num_txs=0 root=AF4941107DC718ED1425E77A3DC7F1154FB780B7A7DE20288DC43442203527E3
+    laconicd-1         | 6:12AM INF finalized block block_app_hash=26A665360BB1EE64E54F97F2A5AB7F621B33A86D9896574000C05DE63F43F788 height=17 module=state num_txs_res=0 num_val_updates=0
+    laconicd-1         | 6:12AM INF executed block app_hash=26A665360BB1EE64E54F97F2A5AB7F621B33A86D9896574000C05DE63F43F788 height=17 module=state
+    laconicd-1         | 6:12AM INF committed state block_app_hash=AF4941107DC718ED1425E77A3DC7F1154FB780B7A7DE20288DC43442203527E3 height=17 module=state
+    laconicd-1         | 6:12AM INF indexed block events height=17 module=txindex
     ```
 
 4. Confirm operation of the registry CLI:
 
    ```
-   laconic-so deployment --dir laconic-loaded-deployment exec cli "laconic cns status"
+   laconic-so deployment --dir laconic-loaded-deployment exec cli "laconic registry status"
    ```
 
    ```
    {
      "version": "0.3.0",
      "node": {
-       "id": "4216af2ac9f68bda33a38803fc1b5c9559312c1d",
+       "id": "6e072894aa1f5d9535a1127a0d7a7f8e65100a2c",
        "network": "laconic_9000-1",
        "moniker": "localtestnet"
      },
      "sync": {
-       "latest_block_hash": "1BDF4CB9AE2390DA65BCF997C83133C18014FCDDCAE03708488F0B56FCEEA429",
-       "latest_block_height": "5",
-       "latest_block_time": "2023-08-09 16:00:30.386903172 +0000 UTC",
-       "catching_up": false
+       "latestBlockHash": "260102C283D0411CFBA0270F7DC182650FFCA737A2F6F652A985F6065696F590",
+       "latestBlockHeight": "49",
+       "latestBlockTime": "2024-07-25 06:14:05.626744215 +0000 UTC",
+       "catchingUp": false
      },
      "validator": {
-       "address": "651FBC700B747C76E90ACFC18CC9508C3D0905B9",
-       "voting_power": "1000000000000000"
+       "address": "86970D950BC9C16F3991A52D9C6DC55BA478A7C6",
+       "votingPower": "1000000000000000"
      },
      "validators": [
        {
-         "address": "651FBC700B747C76E90ACFC18CC9508C3D0905B9",
-         "voting_power": "1000000000000000",
-         "proposer_priority": "0"
+         "address": "86970D950BC9C16F3991A52D9C6DC55BA478A7C6",
+         "votingPower": "1000000000000000",
+         "proposerPriority": "0"
        }
      ],
-     "num_peers": "0",
+     "numPeers": "0",
      "peers": [],
-     "disk_usage": "292.0K"
+     "diskUsage": "688K"
    }
    ```
 
@@ -186,13 +186,13 @@ wns
 1. The following command will create a bond and publish a record:
 
 ```
-laconic-so --stack fixturenet-laconic-loaded deploy exec cli ./scripts/create-demo-records.sh
+laconic-so deployment --dir laconic-loaded-deployment exec cli ./scripts/create-demo-records.sh
 ```
 
 You'll get an output like:
 
 ```
-Balance is: 99998999999999998999600000
+Balance is: 9.9999e+25
 Created bond with id: dd88e8d6f9567b32b28e70552aea4419c5dd3307ebae85a284d1fe38904e301a
 Published demo-record-1.yml with id: bafyreierh3xnfivexlscdwubvczmddsnf46uytyfvrbdhkjzztvsz6ruly
 ```
@@ -223,5 +223,5 @@ record:
 - e.g,:
 
 ```
-laconic-so --stack fixturenet-laconic-loaded deploy exec cli "laconic cns record list"
+laconic-so deployment --dir laconic-loaded-deployment exec cli "laconic registry record list"
 ```

requirements.txt

@@ -11,3 +11,5 @@ tomli==2.0.1
 validators==0.22.0
 kubernetes>=28.1.0
 humanfriendly>=10.0
+python-gnupg>=0.5.2
+requests>=2.3.2

setup.py

@@ -4,9 +4,11 @@ with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 with open("requirements.txt", "r", encoding="utf-8") as fh:
     requirements = fh.read()
+with open("stack_orchestrator/data/version.txt", "r", encoding="utf-8") as fh:
+    version = fh.readlines()[-1].strip(" \n")
 setup(
     name='laconic-stack-orchestrator',
-    version='1.0.12',
+    version=version,
     author='Cerc',
     author_email='info@cerc.io',
     license='GNU Affero General Public License',

stack_orchestrator/build/build_containers.py

@@ -71,7 +71,7 @@ def process_container(build_context: BuildContext) -> bool:
 
     # Check if this is in an external stack
     if stack_is_external(build_context.stack):
-        container_parent_dir = Path(build_context.stack).joinpath("container-build")
+        container_parent_dir = Path(build_context.stack).parent.parent.joinpath("container-build")
         temp_build_dir = container_parent_dir.joinpath(build_context.container.replace("/", "-"))
         temp_build_script_filename = temp_build_dir.joinpath("build.sh")
         # Now check if the container exists in the external stack.

stack_orchestrator/build/build_util.py

@@ -21,11 +21,6 @@ from stack_orchestrator.util import get_parsed_stack_config, warn_exit
 
 def get_containers_in_scope(stack: str):
 
-    # See: https://stackoverflow.com/a/20885799/1701505
-    from stack_orchestrator import data
-    with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
-        all_containers = container_list_file.read().splitlines()
-
     containers_in_scope = []
     if stack:
         stack_config = get_parsed_stack_config(stack)
@@ -33,7 +28,10 @@ def get_containers_in_scope(stack: str):
             warn_exit(f"stack {stack} does not define any containers")
         containers_in_scope = stack_config['containers']
     else:
-        containers_in_scope = all_containers
+        # See: https://stackoverflow.com/a/20885799/1701505
+        from stack_orchestrator import data
+        with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
+            containers_in_scope = container_list_file.read().splitlines()
 
     if opts.o.verbose:
         print(f'Containers: {containers_in_scope}')

stack_orchestrator/build/build_webapp.py

@@ -27,9 +27,15 @@ from decouple import config
 import click
 from pathlib import Path
 from stack_orchestrator.build import build_containers
-from stack_orchestrator.deploy.webapp.util import determine_base_container
+from stack_orchestrator.deploy.webapp.util import determine_base_container, TimedLogger
 from stack_orchestrator.build.build_types import BuildContext
 
+def create_env_file(env_vars, repo_root):
+    env_file_path = os.path.join(repo_root, '.env')
+    with open(env_file_path, 'w') as env_file:
+        for key, value in env_vars.items():
+            env_file.write(f"{key}={value}\n")
+    return env_file_path
 
 @click.command()
 @click.option('--base-container')
@@ -37,12 +43,15 @@ from stack_orchestrator.build.build_types import BuildContext
 @click.option("--force-rebuild", is_flag=True, default=False, help="Override dependency checking -- always rebuild")
 @click.option("--extra-build-args", help="Supply extra arguments to build")
 @click.option("--tag", help="Container tag (default: cerc/<app_name>:local)")
+@click.option("--env", help="Environment variables for webapp (format: KEY1=VALUE1,KEY2=VALUE2)", default="")
 @click.pass_context
-def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, tag):
+def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, tag, env):
     '''build the specified webapp container'''
+    logger = TimedLogger()
 
     quiet = ctx.obj.quiet
     debug = ctx.obj.debug
+    verbose = ctx.obj.verbose
     local_stack = ctx.obj.local_stack
     stack = ctx.obj.stack
 
@@ -51,12 +60,12 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
 
     if local_stack:
         dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
-        print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
+        logger.log(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
     else:
         dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))
 
-    if not quiet:
-        print(f'Dev Root is: {dev_root_path}')
+    if verbose:
+        logger.log(f'Dev Root is: {dev_root_path}')
 
     if not base_container:
         base_container = determine_base_container(source_repo)
@@ -65,6 +74,9 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
     container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug,
                                                                     force_rebuild, extra_build_args)
 
+    if verbose:
+        logger.log(f"Building base container: {base_container}")
+
     build_context_1 = BuildContext(
         stack,
         base_container,
@@ -74,20 +86,45 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
     )
     ok = build_containers.process_container(build_context_1)
     if not ok:
-        print("ERROR: Build failed.", file=sys.stderr)
+        logger.log("ERROR: Build failed.")
         sys.exit(1)
 
+    if verbose:
+        logger.log(f"Base container {base_container} build finished.")
+
     # Now build the target webapp.  We use the same build script, but with a different Dockerfile and work dir.
     container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true"
     container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"] = os.path.abspath(source_repo)
-    container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = os.path.join(container_build_dir,
-                                                                          base_container.replace("/", "-"),
-                                                                          "Dockerfile.webapp")
+
+    # Check if Dockerfile exists in the repository
+    repo_dockerfile = os.path.join(container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"], "Dockerfile")
+    default_dockerfile = os.path.join(container_build_dir,
+                                    base_container.replace("/", "-"),
+                                    "Dockerfile.webapp")
+
+    if os.path.isfile(repo_dockerfile):
+        env_vars = {}
+        if env:
+            for pair in env.split(','):
+                key, value = pair.split('=')
+                env_vars[key.strip()] = value.strip()
+        
+        container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = repo_dockerfile
+        
+        # Create .env file with environment variables
+        env_file_path = create_env_file(env_vars, container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"])
+        container_build_env["CERC_CONTAINER_BUILD_ENV_FILE"] = env_file_path
+    else:
+        container_build_env["CERC_CONTAINER_BUILD_DOCKERFILE"] = default_dockerfile
+
     if not tag:
         webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1]
-        container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local"
-    else:
-        container_build_env["CERC_CONTAINER_BUILD_TAG"] = tag
+        tag = f"cerc/{webapp_name}:local"
+
+    container_build_env["CERC_CONTAINER_BUILD_TAG"] = tag
+
+    if verbose:
+        logger.log(f"Building app container: {tag}")
 
     build_context_2 = BuildContext(
         stack,
@@ -98,5 +135,9 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
     )
     ok = build_containers.process_container(build_context_2)
     if not ok:
-        print("ERROR: Build failed.", file=sys.stderr)
+        logger.log("ERROR: Build failed.")
         sys.exit(1)
+
+    if verbose:
+        logger.log(f"App container {base_container} build finished.")
+        logger.log("build-webapp complete", show_step_time=False, show_total_time=True)

stack_orchestrator/constants.py

@@ -34,5 +34,8 @@ volumes_key = "volumes"
 security_key = "security"
 annotations_key = "annotations"
 labels_key = "labels"
+replicas_key = "replicas"
+node_affinities_key = "node-affinities"
+node_tolerations_key = "node-tolerations"
 kind_config_filename = "kind-config.yml"
 kube_config_filename = "kubeconfig.yml"

stack_orchestrator/data/compose/docker-compose-fixturenet-blast.yml

@@ -0,0 +1,80 @@
+
+# From: https://raw.githubusercontent.com/blast-io/deployment/master/docker-compose.yml
+services:
+  # generate jwt.txt if it's absent
+  generate-jwt:
+    image: blastio/openssl
+    volumes:
+      - blast-data:/blast:rw
+    command: >
+      sh -c "[ ! -f /blast/jwt.txt ] && openssl rand -hex 32 | tr -d '\n' > /blast/jwt.txt || exit 0"
+  # initialise geth db
+  geth-init:
+    image: blastio/blast-geth:${NETWORK:-testnet-sepolia}
+    volumes:
+      - blast-data:/blast:rw
+      - ../config/fixturenet-blast/genesis.json:/blast/genesis.json
+    entrypoint: /bin/sh
+    command: >
+      -c "[ ! -d /blast/${GETH_DATA_DIR:-blast-geth-data}/geth ] && /usr/local/bin/geth init --datadir=/blast/${GETH_DATA_DIR:-blast-geth-data} /blast/genesis.json || exit 0"
+    depends_on:
+      generate-jwt:
+        condition: service_completed_successfully
+    env_file: 
+      - ../config/fixturenet-blast/${NETWORK:-fixturenet}.config
+  blast-geth:
+    image: blastio/blast-geth:${NETWORK:-testnet-sepolia}
+    volumes:
+      - blast-data:/blast
+    ports:
+      - "9545"
+      - "9546"
+    command: >
+      --datadir=/blast/${GETH_DATA_DIR:-blast-geth-data}
+      --http
+      --http.corsdomain="*"
+      --http.vhosts="*"
+      --http.addr=0.0.0.0
+      --http.port=9545
+      --http.api=web3,debug,eth,txpool,net,engine
+      --ws
+      --ws.addr=0.0.0.0
+      --ws.port=9546
+      --ws.origins="*"
+      --ws.api=debug,eth,txpool,net,engine
+      --authrpc.addr="0.0.0.0"
+      --authrpc.port="8551"
+      --authrpc.vhosts="*"
+      --authrpc.jwtsecret=/blast/jwt.txt
+      --syncmode=full
+      --gcmode=archive
+      --nodiscover
+      --maxpeers=0
+      --rollup.disabletxpoolgossip=true
+    env_file: 
+      - ../config/fixturenet-blast/${NETWORK:-fixturenet}.config
+    depends_on:
+      geth-init:
+        condition: service_completed_successfully
+  op-node:
+    image: blastio/blast-optimism:${NETWORK:-testnet-sepolia}
+    volumes:
+      - blast-data:/blast
+      - ../config/fixturenet-blast/rollup.json:/blast/rollup.json
+    ports:
+      - "9003"
+    command: >
+      op-node
+      --l1="${CERC_L1_RPC}"
+      --l1.rpckind="any"
+      --l1.trustrpc=true
+      --l2="http://blast-geth:8551"
+      --l2.jwt-secret=/blast/jwt.txt
+      --rollup.config="/blast/rollup.json"
+    depends_on:
+      - blast-geth
+    env_file: 
+      - ../config/fixturenet-blast/${NETWORK:-fixturenet}.config
+
+volumes:
+  blast-data:

stack_orchestrator/data/compose/docker-compose-fixturenet-laconicd.yml

@@ -2,7 +2,11 @@ services:
   laconicd:
     restart: unless-stopped
     image: cerc/laconicd:local
-    command: ["sh", "/docker-entrypoint-scripts.d/create-fixturenet.sh"]
+    command: ["bash", "/docker-entrypoint-scripts.d/create-fixturenet.sh"]
+    environment:
+      TEST_AUCTION_ENABLED: ${TEST_AUCTION_ENABLED:-false}
+      TEST_REGISTRY_EXPIRY: ${TEST_REGISTRY_EXPIRY:-false}
+      ONBOARDING_ENABLED: ${ONBOARDING_ENABLED:-false}
     volumes:
       # The cosmos-sdk node's database directory:
       - laconicd-data:/root/.laconicd
@@ -16,15 +20,14 @@ services:
       - "26657"
       - "26656"
       - "9473"
-      - "8545"
-      - "8546"
       - "9090"
-      - "9091"
       - "1317"
+
   cli:
     image: cerc/laconic-registry-cli:local
     volumes:
       - ../config/fixturenet-laconicd/registry-cli-config-template.yml:/registry-cli-config-template.yml
+      - ${BASE_DIR:-~/cerc}/laconic-registry-cli:/laconic-registry-cli
 
 volumes:
   laconicd-data:

stack_orchestrator/data/compose/docker-compose-fixturenet-plugeth.yml

@@ -6,6 +6,7 @@ services:
       - ../config/fixturenet-eth/fixturenet-eth.env
     environment:
       RUN_BOOTNODE: "true"
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
     image: cerc/fixturenet-plugeth-plugeth:local
     volumes:
       - fixturenet_plugeth_bootnode_geth_data:/root/ethdata

stack_orchestrator/data/compose/docker-compose-grafana.yml

@@ -2,16 +2,24 @@ version: "3.7"
 
 services:
   grafana:
-    image: grafana/grafana:10.2.2
+    image: grafana/grafana:10.2.3
     restart: always
     environment:
       GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL}
+      CERC_GRAFANA_ALERTS_SUBGRAPH_IDS: ${CERC_GRAFANA_ALERTS_SUBGRAPH_IDS}
     volumes:
       - ../config/monitoring/grafana/provisioning:/etc/grafana/provisioning
       - ../config/monitoring/grafana/dashboards:/etc/grafana/dashboards
+      - ../config/monitoring/update-grafana-alerts-config.sh:/update-grafana-alerts-config.sh
       - grafana_storage:/var/lib/grafana
+    user: root
+    entrypoint: ["bash", "-c"]
+    command: |
+      "/update-grafana-alerts-config.sh && /run.sh"
     ports:
       - "3000"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     healthcheck:
       test: ["CMD", "nc", "-vz", "localhost", "3000"]
       interval: 30s

stack_orchestrator/data/compose/docker-compose-graph-node.yml

@@ -16,8 +16,13 @@ services:
       postgres_pass: password
       postgres_db: graph-node
       ethereum: ${ETH_NETWORKS:-lotus-fixturenet:http://lotus-node-1:1234/rpc/v1}
+      # Env varaibles reference: https://git.vdb.to/cerc-io/graph-node/src/branch/master/docs/environment-variables.md
       GRAPH_LOG: debug
       ETHEREUM_REORG_THRESHOLD: 3
+      GRAPH_ETHEREUM_JSON_RPC_TIMEOUT: ${GRAPH_ETHEREUM_JSON_RPC_TIMEOUT:-180}
+      GRAPH_ETHEREUM_REQUEST_RETRIES: ${GRAPH_ETHEREUM_REQUEST_RETRIES:-10}
+      GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE: ${GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE:-2000}
+      GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS: ${GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS:-1000}
     entrypoint: ["bash", "-c"]
     # Wait for ETH RPC endpoint to be up when running with fixturenet-lotus
     command: |
@@ -27,6 +32,7 @@ services:
       - "8001"
       - "8020"
       - "8030"
+      - "8040"
     healthcheck:
       test: ["CMD", "nc", "-vz", "localhost", "8020"]
       interval: 30s

stack_orchestrator/data/compose/docker-compose-laconic-explorer.yml

@@ -0,0 +1,10 @@
+services:
+  laconic-explorer:
+    restart: unless-stopped
+    image: cerc/ping-pub:local
+    environment:
+      - LACONIC_LACONICD_API_URL=${LACONIC_LACONICD_API_URL:-http://localhost:1317}
+      - LACONIC_LACONICD_RPC_URL=${LACONIC_LACONICD_RPC_URL:-http://localhost:26657}
+      - LACONIC_LACONICD_CHAIN_ID=${LACONIC_LACONICD_CHAIN_ID:-chain-id-not-set}
+    ports:
+      - "5173"

stack_orchestrator/data/compose/docker-compose-mainnet-blast.yml

@@ -0,0 +1,83 @@
+
+# From: https://raw.githubusercontent.com/blast-io/deployment/master/docker-compose.yml
+services:
+  # generate jwt.txt if it's absent
+  generate-jwt:
+    image: blastio/openssl
+    volumes:
+      - blast-data:/blast:rw
+    command: >
+      sh -c "[ ! -f /blast/jwt.txt ] && openssl rand -hex 32 | tr -d '\n' > /blast/jwt.txt || exit 0"
+  # initialise geth db
+  geth-init:
+    image: blastio/blast-geth:${NETWORK:-mainnet}
+    volumes:
+      - blast-data:/blast:rw
+    entrypoint: /bin/sh
+    command: >
+      -c "[ ! -d /blast/${GETH_DATA_DIR:-blast-geth-data}/geth ] && /usr/local/bin/geth init --datadir=/blast/${GETH_DATA_DIR:-blast-geth-data} /blast/genesis.json || exit 0"
+    depends_on:
+      generate-jwt:
+        condition: service_completed_successfully
+    env_file: 
+      - ../config/mainnet-blast/${NETWORK:-mainnet}.config
+  blast-geth:
+    image: blastio/blast-geth:${NETWORK:-mainnet}
+    volumes:
+      - blast-data:/blast
+    ports:
+      - "9545"
+      - "9546"
+      - "6060"
+    command: >
+      --datadir=/blast/${GETH_DATA_DIR:-blast-geth-data}
+      --http
+      --http.corsdomain="*"
+      --http.vhosts="*"
+      --http.addr=0.0.0.0
+      --http.port=9545
+      --http.api=web3,debug,eth,txpool,net,engine
+      --ws
+      --ws.addr=0.0.0.0
+      --ws.port=9546
+      --ws.origins="*"
+      --ws.api=debug,eth,txpool,net,engine
+      --authrpc.addr="0.0.0.0"
+      --authrpc.port="8551"
+      --authrpc.vhosts="*"
+      --authrpc.jwtsecret=/blast/jwt.txt
+      --syncmode=full
+      --metrics
+      --metrics.addr=0.0.0.0
+      --gcmode=archive
+      --nodiscover
+      --maxpeers=0
+      --rollup.disabletxpoolgossip=true
+    env_file: 
+      - ../config/mainnet-blast/${NETWORK:-mainnet}.config
+    depends_on:
+      geth-init:
+        condition: service_completed_successfully
+  op-node:
+    image: blastio/blast-optimism:${NETWORK:-mainnet}
+    volumes:
+      - blast-data:/blast
+    ports:
+      - "9003"
+      - "7300"
+    command: >
+      op-node
+      --l1="https://eth-mainnet-1.vdb.to/"
+      --metrics.enabled
+      --l1.rpckind="any"
+      --l1.trustrpc=true
+      --l2="http://blast-geth:8551"
+      --l2.jwt-secret=/blast/jwt.txt
+      --rollup.config="/blast/rollup.json"
+    depends_on:
+      - blast-geth
+    env_file: 
+      - ../config/mainnet-blast/${NETWORK:-mainnet}.config
+
+volumes:
+  blast-data:

stack_orchestrator/data/compose/docker-compose-mobymask-app-v3-demo.yml

@@ -1,32 +0,0 @@
-version: '3.2'
-
-services:
-  # Builds and serves the MobyMask v3 react-app
-  mobymask-v3-app:
-    restart: unless-stopped
-    image: cerc/mobymask-ui:local
-    environment:
-      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_APP_WATCHER_URL: ${CERC_APP_WATCHER_URL}
-      CERC_SNAP_URL: ${CERC_SNAP_URL}
-    working_dir: /app
-    command: ["bash", "/scripts/mobymask-app-start.sh"]
-    volumes:
-      - ../config/watcher-mobymask-v3-demo/mobymask-app-start.sh:/scripts/mobymask-app-start.sh
-      - ../config/watcher-mobymask-v3-demo/local/keys:/server/keys
-      - mobymask_deployment:/server
-      - nitro_deployment:/nitro
-    ports:
-      - "127.0.0.1:3004:80"
-    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "80"]
-      interval: 20s
-      timeout: 5s
-      retries: 15
-      start_period: 10s
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-
-volumes:
-  mobymask_deployment:
-  nitro_deployment:

stack_orchestrator/data/compose/docker-compose-prom-server.yml

@@ -28,15 +28,37 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
-  chain-head-exporter:
+  ethereum-chain-head-exporter:
     image: cerc/watcher-ts:local
     restart: always
     working_dir: /app/packages/cli
     environment:
-      ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
-      FIL_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT}
+      ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT:-https://mainnet.infura.io/v3}
       ETH_RPC_API_KEY: ${CERC_INFURA_KEY}
-      PORT: ${CERC_METRICS_PORT}
+    command: ["sh", "-c", "yarn export-metrics:chain-heads"]
+    ports:
+      - '5000'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  filecoin-chain-head-exporter:
+    image: cerc/watcher-ts:local
+    restart: always
+    working_dir: /app/packages/cli
+    environment:
+      ETH_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT:-https://api.node.glif.io/rpc/v1}
+    command: ["sh", "-c", "yarn export-metrics:chain-heads"]
+    ports:
+      - '5000'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  graph-node-upstream-head-exporter:
+    image: cerc/watcher-ts:local
+    restart: always
+    working_dir: /app/packages/cli
+    environment:
+      ETH_RPC_ENDPOINT: ${GRAPH_NODE_RPC_ENDPOINT}
     command: ["sh", "-c", "yarn export-metrics:chain-heads"]
     ports:
       - '5000'

stack_orchestrator/data/compose/docker-compose-snowballtools-base-backend.yml

@@ -1,13 +0,0 @@
-services:
-  snowballtools-base-backend:
-    image: cerc/snowballtools-base-backend:local
-    restart: always
-    volumes:
-      - data:/data
-      - config:/config:ro
-    ports:
-      - 8000
-
-volumes:
-  data:
-  config:

stack_orchestrator/data/compose/docker-compose-watcher-ajna.yml

@@ -0,0 +1,78 @@
+version: '3.2'
+
+services:
+  ajna-watcher-db:
+    restart: unless-stopped
+    image: postgres:14-alpine
+    environment:
+      - POSTGRES_USER=vdbm
+      - POSTGRES_MULTIPLE_DATABASES=ajna-watcher,ajna-watcher-job-queue
+      - POSTGRES_EXTENSION=ajna-watcher-job-queue:pgcrypto
+      - POSTGRES_PASSWORD=password
+    volumes:
+      - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
+      - ajna_watcher_db_data:/var/lib/postgresql/data
+    ports:
+      - "5432"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+
+  ajna-watcher-job-runner:
+    restart: unless-stopped
+    depends_on:
+      ajna-watcher-db:
+        condition: service_healthy
+    image: cerc/watcher-ajna:local
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+    command: ["bash", "./start-job-runner.sh"]
+    volumes:
+      - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
+      - ../config/watcher-ajna/start-job-runner.sh:/app/start-job-runner.sh
+    ports:
+      - "9000"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  ajna-watcher-server:
+    restart: unless-stopped
+    depends_on:
+      ajna-watcher-db:
+        condition: service_healthy
+      ajna-watcher-job-runner:
+        condition: service_healthy
+    image: cerc/watcher-ajna:local
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+    command: ["bash", "./start-server.sh"]
+    volumes:
+      - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
+      - ../config/watcher-ajna/start-server.sh:/app/start-server.sh
+      - ajna_watcher_gql_logs_data:/app/gql-logs
+    ports:
+      - "3008"
+      - "9001"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+volumes:
+  ajna_watcher_db_data:
+  ajna_watcher_gql_logs_data:

stack_orchestrator/data/compose/docker-compose-watcher-azimuth.yml

@@ -32,8 +32,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
       CERC_HISTORICAL_BLOCK_RANGE: 500
       CONTRACT_ADDRESS: 0x223c067F8CF28ae173EE5CafEa60cA44C335fecB
       CONTRACT_NAME: Azimuth
@@ -47,7 +47,7 @@ services:
     ports:
       - "9000"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -66,18 +66,20 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
     working_dir: /app/packages/azimuth-watcher
     command: "./start-server.sh"
     volumes:
       - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/azimuth-watcher/environments/watcher-config-template.toml
       - ../config/watcher-azimuth/merge-toml.js:/app/packages/azimuth-watcher/merge-toml.js
       - ../config/watcher-azimuth/start-server.sh:/app/packages/azimuth-watcher/start-server.sh
+      - azimuth_watcher_gql_logs_data:/app/packages/azimuth-watcher/gql-logs
     ports:
       - "3001"
+      - "9001"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3001"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3001"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -94,8 +96,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
       CONTRACT_ADDRESS: 0x325f68d32BdEe6Ed86E7235ff2480e2A433D6189
       CONTRACT_NAME: Censures
       STARTING_BLOCK: 6784954
@@ -108,7 +110,7 @@ services:
     ports:
       - "9002"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9002"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9002"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -127,18 +129,20 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
     working_dir: /app/packages/censures-watcher
     command: "./start-server.sh"
     volumes:
       - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/censures-watcher/environments/watcher-config-template.toml
       - ../config/watcher-azimuth/merge-toml.js:/app/packages/censures-watcher/merge-toml.js
       - ../config/watcher-azimuth/start-server.sh:/app/packages/censures-watcher/start-server.sh
+      - censures_watcher_gql_logs_data:/app/packages/censures-watcher/gql-logs
     ports:
       - "3002"
+      - "9003"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3002"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3002"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -155,8 +159,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
       CONTRACT_ADDRESS: 0xe7e7f69b34D7d9Bd8d61Fb22C33b22708947971A
       CONTRACT_NAME: Claims
       STARTING_BLOCK: 6784941
@@ -169,7 +173,7 @@ services:
     ports:
       - "9004"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9004"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9004"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -188,18 +192,20 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
     working_dir: /app/packages/claims-watcher
     command: "./start-server.sh"
     volumes:
       - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/claims-watcher/environments/watcher-config-template.toml
       - ../config/watcher-azimuth/merge-toml.js:/app/packages/claims-watcher/merge-toml.js
       - ../config/watcher-azimuth/start-server.sh:/app/packages/claims-watcher/start-server.sh
+      - claims_watcher_gql_logs_data:/app/packages/claims-watcher/gql-logs
     ports:
       - "3003"
+      - "9005"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3003"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3003"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -216,8 +222,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
       CONTRACT_ADDRESS: 0x8C241098C3D3498Fe1261421633FD57986D74AeA
       CONTRACT_NAME: ConditionalStarRelease
       STARTING_BLOCK: 6828004
@@ -230,7 +236,7 @@ services:
     ports:
       - "9006"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9006"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9006"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -249,18 +255,20 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
     working_dir: /app/packages/conditional-star-release-watcher
     command: "./start-server.sh"
     volumes:
       - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/conditional-star-release-watcher/environments/watcher-config-template.toml
       - ../config/watcher-azimuth/merge-toml.js:/app/packages/conditional-star-release-watcher/merge-toml.js
       - ../config/watcher-azimuth/start-server.sh:/app/packages/conditional-star-release-watcher/start-server.sh
+      - conditional_star_release_watcher_gql_logs_data:/app/packages/conditional-star-release-watcher/gql-logs
     ports:
       - "3004"
+      - "9007"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3004"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3004"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -277,8 +285,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
       CONTRACT_ADDRESS: 0xf6b461fE1aD4bd2ce25B23Fe0aff2ac19B3dFA76
       CONTRACT_NAME: DelegatedSending
       STARTING_BLOCK: 6784956
@@ -291,7 +299,7 @@ services:
     ports:
       - "9008"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9008"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -310,18 +318,20 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
     working_dir: /app/packages/delegated-sending-watcher
     command: "./start-server.sh"
     volumes:
       - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/delegated-sending-watcher/environments/watcher-config-template.toml
       - ../config/watcher-azimuth/merge-toml.js:/app/packages/delegated-sending-watcher/merge-toml.js
       - ../config/watcher-azimuth/start-server.sh:/app/packages/delegated-sending-watcher/start-server.sh
+      - delegated_sending_watcher_gql_logs_data:/app/packages/delegated-sending-watcher/gql-logs
     ports:
       - "3005"
+      - "9009"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3005"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3005"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -338,8 +348,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
       CONTRACT_ADDRESS: 0x33EeCbf908478C10614626A9D304bfe18B78DD73
       CONTRACT_NAME: Ecliptic
       STARTING_BLOCK: 13692129
@@ -352,7 +362,7 @@ services:
     ports:
       - "9010"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9010"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9010"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -371,18 +381,20 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
     working_dir: /app/packages/ecliptic-watcher
     command: "./start-server.sh"
     volumes:
       - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/ecliptic-watcher/environments/watcher-config-template.toml
       - ../config/watcher-azimuth/merge-toml.js:/app/packages/ecliptic-watcher/merge-toml.js
       - ../config/watcher-azimuth/start-server.sh:/app/packages/ecliptic-watcher/start-server.sh
+      - ecliptic_watcher_gql_logs_data:/app/packages/ecliptic-watcher/gql-logs
     ports:
       - "3006"
+      - "9011"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3006"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3006"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -399,8 +411,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
       CONTRACT_ADDRESS: 0x86cd9cd0992F04231751E3761De45cEceA5d1801
       CONTRACT_NAME: LinearStarRelease
       STARTING_BLOCK: 6784943
@@ -413,7 +425,7 @@ services:
     ports:
       - "9012"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9012"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9012"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -432,18 +444,20 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
     working_dir: /app/packages/linear-star-release-watcher
     command: "./start-server.sh"
     volumes:
       - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/linear-star-release-watcher/environments/watcher-config-template.toml
       - ../config/watcher-azimuth/merge-toml.js:/app/packages/linear-star-release-watcher/merge-toml.js
       - ../config/watcher-azimuth/start-server.sh:/app/packages/linear-star-release-watcher/start-server.sh
+      - linear_star_release_watcher_gql_logs_data:/app/packages/linear-star-release-watcher/gql-logs
     ports:
       - "3007"
+      - "9013"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3007"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3007"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -460,8 +474,8 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
       CONTRACT_ADDRESS: 0x7fEcaB617c868Bb5996d99D95200D2Fa708218e4
       CONTRACT_NAME: Polls
       STARTING_BLOCK: 6784912
@@ -474,7 +488,7 @@ services:
     ports:
       - "9014"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9014"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9014"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -493,18 +507,20 @@ services:
         condition: service_healthy
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
     working_dir: /app/packages/polls-watcher
     command: "./start-server.sh"
     volumes:
       - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/polls-watcher/environments/watcher-config-template.toml
       - ../config/watcher-azimuth/merge-toml.js:/app/packages/polls-watcher/merge-toml.js
       - ../config/watcher-azimuth/start-server.sh:/app/packages/polls-watcher/start-server.sh
+      - polls_watcher_gql_logs_data:/app/packages/polls-watcher/gql-logs
     ports:
       - "3008"
+      - "9015"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -542,7 +558,7 @@ services:
     ports:
       - "0.0.0.0:4000:4000"
     healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "4000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "4000"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -552,3 +568,11 @@ services:
 
 volumes:
   watcher_db_data:
+  azimuth_watcher_gql_logs_data:
+  censures_watcher_gql_logs_data:
+  claims_watcher_gql_logs_data:
+  conditional_star_release_watcher_gql_logs_data:
+  delegated_sending_watcher_gql_logs_data:
+  ecliptic_watcher_gql_logs_data:
+  linear_star_release_watcher_gql_logs_data:
+  polls_watcher_gql_logs_data:

stack_orchestrator/data/compose/docker-compose-watcher-merkl-sushiswap-v3.yml

@@ -29,7 +29,7 @@ services:
     image: cerc/watcher-merkl-sushiswap-v3:local
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
     command: ["bash", "./start-job-runner.sh"]
     volumes:
       - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -37,7 +37,7 @@ services:
     ports:
       - "9002:9000"
     healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -55,16 +55,17 @@ services:
     image: cerc/watcher-merkl-sushiswap-v3:local
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
     command: ["bash", "./start-server.sh"]
     volumes:
       - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
       - ../config/watcher-merkl-sushiswap-v3/start-server.sh:/app/start-server.sh
+      - merkl_sushiswap_v3_watcher_gql_logs_data:/app/gql-logs
     ports:
       - "127.0.0.1:3007:3008"
       - "9003:9001"
     healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -74,3 +75,4 @@ services:
 
 volumes:
   merkl_sushiswap_v3_watcher_db_data:
+  merkl_sushiswap_v3_watcher_gql_logs_data:

stack_orchestrator/data/compose/docker-compose-watcher-mobymask-v3-demo.yml

@@ -1,162 +0,0 @@
-version: '3.2'
-
-services:
-  # Starts the PostgreSQL database for watcher
-  mobymask-watcher-db:
-    restart: unless-stopped
-    image: postgres:14-alpine
-    environment:
-      - POSTGRES_USER=vdbm
-      - POSTGRES_MULTIPLE_DATABASES=mobymask-watcher-1,mobymask-watcher-1-job-queue,mobymask-watcher-2,mobymask-watcher-2-job-queue,mobymask-watcher-3,mobymask-watcher-3-job-queue
-      - POSTGRES_EXTENSION=mobymask-watcher-1-job-queue:pgcrypto,mobymask-watcher-2-job-queue:pgcrypto,mobymask-watcher-3-job-queue:pgcrypto
-      - POSTGRES_PASSWORD=password
-    volumes:
-      - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
-      - mobymask_watcher_db_data:/var/lib/postgresql/data
-    ports:
-      - "5432"
-    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "5432"]
-      interval: 20s
-      timeout: 5s
-      retries: 15
-      start_period: 10s
-
-  # Deploys the MobyMask contract and generates an invite link
-  # Deployment is skipped if CERC_DEPLOYED_CONTRACT env is set
-  mobymask:
-    image: cerc/mobymask:local
-    working_dir: /app/packages/server
-    environment:
-      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      # ENV: "PROD" # Check if required
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
-      CERC_PRIVATE_KEY_DEPLOYER: ${CERC_PRIVATE_KEY_DEPLOYER}
-      CERC_MOBYMASK_APP_BASE_URI: ${CERC_MOBYMASK_APP_BASE_URI}
-      CERC_DEPLOYED_CONTRACT: ${CERC_DEPLOYED_CONTRACT}
-    command: ["bash", "-c", "./deploy-and-generate-invite.sh"]
-    volumes:
-      - ../config/watcher-mobymask-v3-demo/deploy-and-generate-invite.sh:/app/packages/server/deploy-and-generate-invite.sh
-      - mobymask_deployment:/app/packages/server
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-
-  # Starts a MobyMask v3 watcher server #1
-  mobymask-watcher-1-server:
-    image: cerc/watcher-mobymask-v3:local
-    restart: unless-stopped
-    depends_on:
-      mobymask-watcher-db:
-        condition: service_healthy
-      mobymask:
-        condition: service_completed_successfully
-      nitro-contracts:
-        condition: service_completed_successfully
-    environment:
-      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_WATCHER_INDEX: 1
-      CERC_ETH_RPC_QUERY_ENDPOINT: ${CERC_ETH_RPC_QUERY_ENDPOINT}
-      CERC_ETH_RPC_MUTATION_ENDPOINT: ${CERC_ETH_RPC_MUTATION_ENDPOINT}
-      CERC_NITRO_CHAIN_URL: ${CERC_NITRO_CHAIN_URL}
-    command: ["bash", "./start-server.sh"]
-    volumes:
-      - ../config/watcher-mobymask-v3-demo/watcher-config-template.toml:/app/environments/watcher-config-template.toml
-      - ../config/watcher-mobymask-v3-demo/watcher-config-rates.toml:/app/environments/rates.toml
-      - ../config/watcher-mobymask-v3-demo/local/keys:/app/keys
-      - ../config/watcher-mobymask-v3-demo/start-server.sh:/app/start-server.sh
-      - watcher_nitro_data:/app/out/nitro-db
-      - nitro_deployment:/nitro
-      - mobymask_deployment:/server
-    # Expose GQL, metrics and relay node ports
-    ports:
-      - "0.0.0.0:3001:3001"
-      - "9001"
-      - "0.0.0.0:9090:9090"
-    healthcheck:
-      test: ["CMD", "busybox", "nc", "localhost", "9090"]
-      interval: 20s
-      timeout: 5s
-      retries: 15
-      start_period: 5s
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-
-  # Starts a MobyMask v3 watcher server #2
-  mobymask-watcher-2-server:
-    image: cerc/watcher-mobymask-v3:local
-    restart: unless-stopped
-    depends_on:
-      mobymask-watcher-1-server:
-        condition: service_healthy
-    environment:
-      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_WATCHER_INDEX: 2
-      CERC_ETH_RPC_QUERY_ENDPOINT: ${CERC_ETH_RPC_QUERY_ENDPOINT}
-      CERC_ETH_RPC_MUTATION_ENDPOINT: ${CERC_ETH_RPC_MUTATION_ENDPOINT}
-      CERC_NITRO_CHAIN_URL: ${CERC_NITRO_CHAIN_URL}
-    command: ["bash", "./start-server.sh"]
-    volumes:
-      - ../config/watcher-mobymask-v3-demo/watcher-config-template.toml:/app/environments/watcher-config-template.toml
-      - ../config/watcher-mobymask-v3-demo/watcher-config-rates.toml:/app/environments/rates.toml
-      - ../config/watcher-mobymask-v3-demo/local/keys:/app/keys
-      - ../config/watcher-mobymask-v3-demo/start-server.sh:/app/start-server.sh
-      - watcher_nitro_data:/app/out/nitro-db
-      - nitro_deployment:/nitro
-      - mobymask_deployment:/server
-    # Expose GQL, metrics and relay node ports
-    ports:
-      - "3001"
-      - "9001"
-      - "0.0.0.0:9091:9090"
-    healthcheck:
-      test: ["CMD", "busybox", "nc", "localhost", "9090"]
-      interval: 20s
-      timeout: 5s
-      retries: 15
-      start_period: 5s
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-
-  # Starts a MobyMask v3 watcher server #3
-  mobymask-watcher-3-server:
-    image: cerc/watcher-mobymask-v3:local
-    restart: unless-stopped
-    depends_on:
-      mobymask-watcher-1-server:
-        condition: service_healthy
-      mobymask-watcher-2-server:
-        condition: service_healthy
-    environment:
-      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_WATCHER_INDEX: 3
-      CERC_ETH_RPC_QUERY_ENDPOINT: ${CERC_ETH_RPC_QUERY_ENDPOINT}
-      CERC_ETH_RPC_MUTATION_ENDPOINT: ${CERC_ETH_RPC_MUTATION_ENDPOINT}
-      CERC_NITRO_CHAIN_URL: ${CERC_NITRO_CHAIN_URL}
-    command: ["bash", "./start-server.sh"]
-    volumes:
-      - ../config/watcher-mobymask-v3-demo/watcher-config-template.toml:/app/environments/watcher-config-template.toml
-      - ../config/watcher-mobymask-v3-demo/watcher-config-rates.toml:/app/environments/rates.toml
-      - ../config/watcher-mobymask-v3-demo/local/keys:/app/keys
-      - ../config/watcher-mobymask-v3-demo/start-server.sh:/app/start-server.sh
-      - watcher_nitro_data:/app/out/nitro-db
-      - nitro_deployment:/nitro
-      - mobymask_deployment:/server
-    # Expose GQL, metrics and relay node ports
-    ports:
-      - "3001"
-      - "9001"
-      - "0.0.0.0:9092:9090"
-    healthcheck:
-      test: ["CMD", "busybox", "nc", "localhost", "9090"]
-      interval: 20s
-      timeout: 5s
-      retries: 15
-      start_period: 5s
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-
-volumes:
-  mobymask_watcher_db_data:
-  mobymask_deployment:
-  nitro_deployment:
-  watcher_nitro_data:

stack_orchestrator/data/compose/docker-compose-watcher-sushiswap-v3.yml

@@ -29,7 +29,7 @@ services:
     image: cerc/watcher-sushiswap-v3:local
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
     command: ["bash", "./start-job-runner.sh"]
     volumes:
       - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -37,7 +37,7 @@ services:
     ports:
       - "9000:9000"
     healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -55,16 +55,17 @@ services:
     image: cerc/watcher-sushiswap-v3:local
     environment:
       CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
     command: ["bash", "./start-server.sh"]
     volumes:
       - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
       - ../config/watcher-sushiswap-v3/start-server.sh:/app/start-server.sh
+      - sushiswap_v3_watcher_gql_logs_data:/app/gql-logs
     ports:
       - "127.0.0.1:3008:3008"
       - "9001:9001"
     healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
       interval: 20s
       timeout: 5s
       retries: 15
@@ -74,3 +75,4 @@ services:
 
 volumes:
   sushiswap_v3_watcher_db_data:
+  sushiswap_v3_watcher_gql_logs_data:

stack_orchestrator/data/config/fixturenet-blast/fixturenet.config

@@ -0,0 +1,2 @@
+GETH_ROLLUP_SEQUENCERHTTP=https://sequencer.s2.testblast.io
+OP_NODE_P2P_BOOTNODES=enr:-J-4QM3GLUFfKMSJQuP1UvuKQe8DyovE7Eaiit0l6By4zjTodkR4V8NWXJxNmlg8t8rP-Q-wp3jVmeAOml8cjMj__ROGAYznzb_HgmlkgnY0gmlwhA-cZ_eHb3BzdGFja4X947FQAIlzZWNwMjU2azGhAiuDqvB-AsVSRmnnWr6OHfjgY8YfNclFy9p02flKzXnOg3RjcIJ2YYN1ZHCCdmE,enr:-J-4QDCVpByqQ8nFqCS9aHicqwUfXgzFDslvpEyYz19lvkHLIdtcIGp2d4q5dxHdjRNTO6HXCsnIKxUeuZSPcEbyVQCGAYznzz0RgmlkgnY0gmlwhANiQfuHb3BzdGFja4X947FQAIlzZWNwMjU2azGhAy3AtF2Jh_aPdOohg506Hjmtx-fQ1AKmu71C7PfkWAw9g3RjcIJ2YYN1ZHCCdmE

stack_orchestrator/data/config/fixturenet-blast/genesis.json

@@ -0,0 +1,57 @@
+{
+  "config": {
+    "chainId": 608943043,
+    "homesteadBlock": 0,
+    "eip150Block": 0,
+    "eip155Block": 0,
+    "eip158Block": 0,
+    "byzantiumBlock": 0,
+    "constantinopleBlock": 0,
+    "petersburgBlock": 0,
+    "istanbulBlock": 0,
+    "muirGlacierBlock": 0,
+    "berlinBlock": 0,
+    "londonBlock": 0,
+    "arrowGlacierBlock": 0,
+    "grayGlacierBlock": 0,
+    "mergeNetsplitBlock": 0,
+    "shanghaiTime": 0,
+    "bedrockBlock": 0,
+    "regolithTime": 0,
+    "canyonTime": 0,
+    "terminalTotalDifficulty": 0,
+    "terminalTotalDifficultyPassed": true,
+    "optimism": {
+      "eip1559Elasticity": 6,
+      "eip1559Denominator": 50,
+      "eip1559DenominatorCanyon": 250
+    }
+  },
+  "alloc": {
+    "0000000000000000000000000000000000000000": {
+      "balance": "0x1"
+    },
+    "4200000000000000000000000000000000000000": {
+      "code": "0x60806040526004361061004e5760003560e01c80633659cfe6146100655780634f1ef286146100855780635c60da1b146100ae5780638f283970146100db578063f851a440146100fb5761005d565b3661005d5761005b610110565b005b61005b610110565b34801561007157600080fd5b5061005b610080366004610521565b6101c8565b61009861009336600461053c565b61020e565b6040516100a591906105bf565b60405180910390f35b3480156100ba57600080fd5b506100c361033e565b6040516001600160a01b0390911681526020016100a5565b3480156100e757600080fd5b5061005b6100f6366004610521565b6103a9565b34801561010757600080fd5b506100c36103e4565b600061013a7f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc5490565b90506001600160a01b0381166101a55760405162461bcd60e51b815260206004820152602560248201527f50726f78793a20696d706c656d656e746174696f6e206e6f7420696e697469616044820152641b1a5e995960da1b60648201526084015b60405180910390fd5b3660008037600080366000845af43d6000803e806101c2573d6000fd5b503d6000f35b600080516020610625833981519152546001600160a01b0316336001600160a01b031614806101f5575033155b156102065761020381610432565b50565b610203610110565b60606102266000805160206106258339815191525490565b6001600160a01b0316336001600160a01b03161480610243575033155b1561032f5761025184610432565b600080856001600160a01b0316858560405161026e929190610614565b600060405180830381855af49150503d80600081146102a9576040519150601f19603f3d011682016040523d82523d6000602084013e6102ae565b606091505b5091509150816103265760405162461bcd60e51b815260206004820152603960248201527f50726f78793a2064656c656761746563616c6c20746f206e657720696d706c6560448201527f6d656e746174696f6e20636f6e7472616374206661696c656400000000000000606482015260840161019c565b91506103379050565b610337610110565b9392505050565b60006103566000805160206106258339815191525490565b6001600160a01b0316336001600160a01b03161480610373575033155b1561039e57507f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc5490565b6103a6610110565b90565b600080516020610625833981519152546001600160a01b0316336001600160a01b031614806103d6575033155b15610206576102038161048e565b60006103fc6000805160206106258339815191525490565b6001600160a01b0316336001600160a01b03161480610419575033155b1561039e57506000805160206106258339815191525490565b7f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc8181556040516001600160a01b038316907fbc7cd75a20ee27fd9adebab32041f755214dbc6bffa90cc0225b39da2e5c2d3b90600090a25050565b60006104a66000805160206106258339815191525490565b600080516020610625833981519152838155604080516001600160a01b0380851682528616602082015292935090917f7e644d79422f17c01e4894b5f4f588d331ebfa28653d42ae832dc59e38c9798f910160405180910390a1505050565b80356001600160a01b038116811461051c57600080fd5b919050565b60006020828403121561053357600080fd5b61033782610505565b60008060006040848603121561055157600080fd5b61055a84610505565b9250602084013567ffffffffffffffff8082111561057757600080fd5b818601915086601f83011261058b57600080fd5b81358181111561059a57600080fd5b8760208285010111156105ac57600080fd5b6020830194508093505050509250925092565b600060208083528351808285015260005b818110156105ec578581018301518582016040015282016105d0565b818111156105fe576000604083870101525b50601f01601f1916929092016040019392505050565b818382376000910190815291905056feb53127684a568b3173ae13b9f8a6016e243e63b6e8ee1178d6a717850b5d6103a164736f6c634300080f000a",
+      "storage": {
+        "0x360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc": "0x000000000000000000000000c0d3c0d3c0d3c0d3c0d3c0d3c0d3c0d3c0d30000",
+        "0xb53127684a568b3173ae13b9f8a6016e243e63b6e8ee1178d6a717850b5d6103": "0x0000000000000000000000004200000000000000000000000000000000000018"
+      },
+      "balance": "0x0",
+      "flags": 1
+    }
+  },
+  "nonce": "0x0",
+  "timestamp": "0x659b7460",
+  "extraData": "0x424544524f434b",
+  "gasLimit": "0x1c9c380",
+  "difficulty": "0x0",
+  "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+  "coinbase": "0x4200000000000000000000000000000000000011",
+  "number": "0x0",
+  "gasUsed": "0x0",
+  "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+  "baseFeePerGas": "0x3b9aca00",
+  "excessBlobGas": null,
+  "blobGasUsed": null
+}

stack_orchestrator/data/config/fixturenet-blast/rollup.json

@@ -0,0 +1,31 @@
+{
+  "genesis": {
+    "l1": {
+      "hash": "0x17728cf4d8e0b4f292d2390a869fd7c632d39e72efb00ca3462b4387c6aa2437",
+      "number": 5044255
+    },
+    "l2": {
+      "hash": "0x26a1c0faad7b041f34569a1bb383f00ab74b335883a44bed53e9f41ced5fd906",
+      "number": 0
+    },
+    "l2_time": 1704686688,
+    "system_config": {
+      "batcherAddr": "0xba26fee2fa917443e05e65de8d4350bcd2f59222",
+      "overhead": "0x00000000000000000000000000000000000000000000000000000000000000bc",
+      "scalar": "0x00000000000000000000000000000000000000000000000000000000000a6fe0",
+      "gasLimit": 30000000
+    }
+  },
+  "block_time": 2,
+  "max_sequencer_drift": 600,
+  "seq_window_size": 3600,
+  "channel_timeout": 300,
+  "l1_chain_id": 11155111,
+  "l2_chain_id": 608943043,
+  "regolith_time": 0,
+  "canyon_time": 0,
+  "batch_inbox_address": "0x1c3b85a2108784eab6a4bf56cdd6f722e415b331",
+  "deposit_contract_address": "0x2757e4430e694f27b73ec9c02257cab3a498c8c5",
+  "l1_system_config_address": "0x329faf078c364a316e08bf6a17b7eee6ae75a613",
+  "protocol_versions_address": "0x0000000000000000000000000000000000000000"
+}

stack_orchestrator/data/config/fixturenet-eth/fixturenet-eth.env

@@ -23,3 +23,6 @@ CERC_STATEDIFF_WORKERS=2
 
 CERC_GETH_VMODULE="statediff/*=5,rpc/*=5"
 CERC_GETH_VERBOSITY=${CERC_GETH_VERBOSITY:-3}
+
+# Used by Lighthouse
+SECONDS_PER_ETH1_BLOCK=${SECONDS_PER_ETH1_BLOCK:-3}

stack_orchestrator/data/config/fixturenet-laconicd/create-fixturenet.sh

@@ -8,68 +8,68 @@ KEY="mykey"
 CHAINID="laconic_9000-1"
 MONIKER="localtestnet"
 KEYRING="test"
-KEYALGO="eth_secp256k1"
-LOGLEVEL="info"
-# trace evm
-TRACE="--trace"
-# TRACE=""
+KEYALGO="secp256k1"
+LOGLEVEL="${LOGLEVEL:-info}"
+DENOM="alnt"
+
 
 if [ "$1" == "clean" ] || [ ! -d "$HOME/.laconicd/data/blockstore.db" ]; then
   # validate dependencies are installed
-  command -v jq > /dev/null 2>&1 || { echo >&2 "jq not installed. More info: https://stedolan.github.io/jq/download/"; exit 1; }
+  command -v jq > /dev/null 2>&1 || {
+    echo >&2 "jq not installed. More info: https://stedolan.github.io/jq/download/"
+    exit 1
+  }
 
   # remove existing daemon and client
   rm -rf $HOME/.laconicd/*
-  rm -rf $HOME/.laconic/*
 
   if [ -n "`which make`" ]; then
     make install
   fi
 
-  laconicd config keyring-backend $KEYRING
-  laconicd config chain-id $CHAINID
+  laconicd config set client chain-id $CHAINID
+  laconicd config set client keyring-backend $KEYRING
 
   # if $KEY exists it should be deleted
   laconicd keys add $KEY --keyring-backend $KEYRING --algo $KEYALGO
 
   # Set moniker and chain-id for Ethermint (Moniker can be anything, chain-id must be an integer)
-  laconicd init $MONIKER --chain-id $CHAINID
+  laconicd init $MONIKER --chain-id $CHAINID --default-denom $DENOM
 
-  # Change parameter token denominations to aphoton
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["staking"]["params"]["bond_denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["crisis"]["constant_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["gov"]["deposit_params"]["min_deposit"][0]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["mint"]["params"]["mint_denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  # Custom modules
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["record_rent"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_commit_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_reveal_fee"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-  cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_minimum_bid"]["denom"]="aphoton"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  update_genesis() {
+    jq "$1" $HOME/.laconicd/config/genesis.json > $HOME/.laconicd/config/tmp_genesis.json &&
+      mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  }
 
   if [[ "$TEST_REGISTRY_EXPIRY" == "true" ]]; then
     echo "Setting timers for expiry tests."
 
-    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["record_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_grace_period"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+    update_genesis '.app_state["registry"]["params"]["record_rent_duration"]="60s"'
+    update_genesis '.app_state["registry"]["params"]["authority_grace_period"]="60s"'
+    update_genesis '.app_state["registry"]["params"]["authority_rent_duration"]="60s"'
   fi
 
   if [[ "$TEST_AUCTION_ENABLED" == "true" ]]; then
     echo "Enabling auction and setting timers."
 
-    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_enabled"]=true' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_rent_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_grace_period"]="300s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_commits_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
-    cat $HOME/.laconicd/config/genesis.json | jq '.app_state["registry"]["params"]["authority_auction_reveals_duration"]="60s"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+    update_genesis '.app_state["registry"]["params"]["authority_auction_enabled"]=true'
+    update_genesis '.app_state["registry"]["params"]["authority_rent_duration"]="60s"'
+    update_genesis '.app_state["registry"]["params"]["authority_grace_period"]="300s"'
+    update_genesis '.app_state["registry"]["params"]["authority_auction_commits_duration"]="60s"'
+    update_genesis '.app_state["registry"]["params"]["authority_auction_reveals_duration"]="60s"'
+  fi
+
+  if [[ "$ONBOARDING_ENABLED" == "true" ]]; then
+    echo "Enabling validator onboarding."
+
+    update_genesis '.app_state["onboarding"]["params"]["onboarding_enabled"]=true'
   fi
 
   # increase block time (?)
-  cat $HOME/.laconicd/config/genesis.json | jq '.consensus_params["block"]["time_iota_ms"]="1000"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  update_genesis '.consensus["params"]["block"]["time_iota_ms"]="1000"'
 
   # Set gas limit in genesis
-  cat $HOME/.laconicd/config/genesis.json | jq '.consensus_params["block"]["max_gas"]="10000000"' > $HOME/.laconicd/config/tmp_genesis.json && mv $HOME/.laconicd/config/tmp_genesis.json $HOME/.laconicd/config/genesis.json
+  update_genesis '.consensus["params"]["block"]["max_gas"]="10000000"'
 
   # disable produce empty block
   if [[ "$OSTYPE" == "darwin"* ]]; then
@@ -78,48 +78,39 @@ if [ "$1" == "clean" ] || [ ! -d "$HOME/.laconicd/data/blockstore.db" ]; then
       sed -i 's/create_empty_blocks = true/create_empty_blocks = false/g' $HOME/.laconicd/config/config.toml
   fi
 
-  if [[ $1 == "pending" ]]; then
-    if [[ "$OSTYPE" == "darwin"* ]]; then
-        sed -i '' 's/create_empty_blocks_interval = "0s"/create_empty_blocks_interval = "30s"/g' $HOME/.laconicd/config/config.toml
-        sed -i '' 's/timeout_propose = "3s"/timeout_propose = "30s"/g' $HOME/.laconicd/config/config.toml
-        sed -i '' 's/timeout_propose_delta = "500ms"/timeout_propose_delta = "5s"/g' $HOME/.laconicd/config/config.toml
-        sed -i '' 's/timeout_prevote = "1s"/timeout_prevote = "10s"/g' $HOME/.laconicd/config/config.toml
-        sed -i '' 's/timeout_prevote_delta = "500ms"/timeout_prevote_delta = "5s"/g' $HOME/.laconicd/config/config.toml
-        sed -i '' 's/timeout_precommit = "1s"/timeout_precommit = "10s"/g' $HOME/.laconicd/config/config.toml
-        sed -i '' 's/timeout_precommit_delta = "500ms"/timeout_precommit_delta = "5s"/g' $HOME/.laconicd/config/config.toml
-        sed -i '' 's/timeout_commit = "5s"/timeout_commit = "150s"/g' $HOME/.laconicd/config/config.toml
-        sed -i '' 's/timeout_broadcast_tx_commit = "10s"/timeout_broadcast_tx_commit = "150s"/g' $HOME/.laconicd/config/config.toml
-    else
-        sed -i 's/create_empty_blocks_interval = "0s"/create_empty_blocks_interval = "30s"/g' $HOME/.laconicd/config/config.toml
-        sed -i 's/timeout_propose = "3s"/timeout_propose = "30s"/g' $HOME/.laconicd/config/config.toml
-        sed -i 's/timeout_propose_delta = "500ms"/timeout_propose_delta = "5s"/g' $HOME/.laconicd/config/config.toml
-        sed -i 's/timeout_prevote = "1s"/timeout_prevote = "10s"/g' $HOME/.laconicd/config/config.toml
-        sed -i 's/timeout_prevote_delta = "500ms"/timeout_prevote_delta = "5s"/g' $HOME/.laconicd/config/config.toml
-        sed -i 's/timeout_precommit = "1s"/timeout_precommit = "10s"/g' $HOME/.laconicd/config/config.toml
-        sed -i 's/timeout_precommit_delta = "500ms"/timeout_precommit_delta = "5s"/g' $HOME/.laconicd/config/config.toml
-        sed -i 's/timeout_commit = "5s"/timeout_commit = "150s"/g' $HOME/.laconicd/config/config.toml
-        sed -i 's/timeout_broadcast_tx_commit = "10s"/timeout_broadcast_tx_commit = "150s"/g' $HOME/.laconicd/config/config.toml
-    fi
+  # Enable telemetry (prometheus metrics: http://localhost:1317/metrics?format=prometheus)
+  if [[ "$OSTYPE" == "darwin"* ]]; then
+    sed -i '' 's/enabled = false/enabled = true/g' $HOME/.laconicd/config/app.toml
+    sed -i '' 's/prometheus-retention-time = 0/prometheus-retention-time = 60/g' $HOME/.laconicd/config/app.toml
+    sed -i '' 's/prometheus = false/prometheus = true/g' $HOME/.laconicd/config/config.toml
+  else
+    sed -i 's/enabled = false/enabled = true/g' $HOME/.laconicd/config/app.toml
+    sed -i 's/prometheus-retention-time = 0/prometheus-retention-time = 60/g' $HOME/.laconicd/config/app.toml
+    sed -i 's/prometheus = false/prometheus = true/g' $HOME/.laconicd/config/config.toml
   fi
 
   # Allocate genesis accounts (cosmos formatted addresses)
-  laconicd add-genesis-account $KEY 100000000000000000000000000aphoton --keyring-backend $KEYRING
+  # 10^30 alnt | 10^12 lnt
+  laconicd genesis add-genesis-account $KEY 1000000000000000000000000000000$DENOM --keyring-backend $KEYRING
 
   # Sign genesis transaction
-  laconicd gentx $KEY 1000000000000000000000aphoton --keyring-backend $KEYRING --chain-id $CHAINID
+  # 10^24 alnt | 10^6 lnt
+  laconicd genesis gentx $KEY 1000000000000000000000000$DENOM --keyring-backend $KEYRING --chain-id $CHAINID
 
   # Collect genesis tx
-  laconicd collect-gentxs
+  laconicd genesis collect-gentxs
 
   # Run this to ensure everything worked and that the genesis file is setup correctly
-  laconicd validate-genesis
-
-  if [[ $1 == "pending" ]]; then
-    echo "pending mode is on, please wait for the first block committed."
-  fi
+  laconicd genesis validate
 else
   echo "Using existing database at $HOME/.laconicd.  To replace, run '`basename $0` clean'"
 fi
 
 # Start the node (remove the --pruning=nothing flag if historical queries are not needed)
-laconicd start --pruning=nothing --evm.tracer=json $TRACE --log_level $LOGLEVEL --minimum-gas-prices=0.0001aphoton --json-rpc.api eth,txpool,personal,net,debug,web3,miner --api.enable --gql-server --gql-playground
+laconicd start \
+  --pruning=nothing \
+  --log_level $LOGLEVEL \
+  --minimum-gas-prices=1$DENOM \
+  --api.enable \
+  --rpc.laddr="tcp://0.0.0.0:26657" \
+  --gql-server --gql-playground

stack_orchestrator/data/config/fixturenet-laconicd/registry-cli-config-template.yml

@@ -1,9 +1,9 @@
 services:
-  cns:
-    restEndpoint: 'http://laconicd:1317'
+  registry:
+    rpcEndpoint: 'http://laconicd:26657'
     gqlEndpoint: 'http://laconicd:9473/api'
     userKey: REPLACE_WITH_MYKEY
     bondId:
     chainId: laconic_9000-1
     gas: 350000
-    fees: 200000aphoton
+    fees: 2000000alnt

stack_orchestrator/data/config/fixturenet-optimism/optimism-contracts/deploy-contracts.sh

@@ -169,4 +169,4 @@ cp -a /app/packages/contracts-bedrock/deployments/$DEPLOYMENT_CONTEXT /l1-deploy
 cp /app/packages/contracts-bedrock/deploy-config/$DEPLOYMENT_CONTEXT.json /l2-config
 openssl rand -hex 32 > /l2-config/l2-jwt.txt
 
-echo "Deployment successful. Exiting."
+echo "Deployment successful. Exiting"

stack_orchestrator/data/config/mainnet-blast/keycloak.env

@@ -0,0 +1,32 @@
+POSTGRES_DB=keycloak
+POSTGRES_USER=keycloak
+POSTGRES_PASSWORD=keycloak
+# Don't change this unless you also change the healthcheck in docker-compose-mainnet-eth-keycloak.yml
+PGPORT=35432
+KC_DB=postgres
+KC_DB_URL_HOST=keycloak-db
+KC_DB_URL_PORT=${PGPORT}
+KC_DB_URL_DATABASE=${POSTGRES_DB}
+KC_DB_USERNAME=${POSTGRES_USER}
+KC_DB_PASSWORD=${POSTGRES_PASSWORD}
+KC_DB_SCHEMA=public
+KC_HOSTNAME=localhost
+KC_HTTP_ENABLED="true"
+KC_HTTP_RELATIVE_PATH="/auth"
+KC_HOSTNAME_STRICT_HTTPS="false"
+KEYCLOAK_ADMIN=admin
+KEYCLOAK_ADMIN_PASSWORD=admin
+X_API_CHECK_REALM=cerc
+X_API_CHECK_CLIENT_ID="%user_id%"
+
+
+# keycloak-reg-api
+CERC_KCUSERREG_LISTEN_PORT=9292
+CERC_KCUSERREG_LISTEN_ADDR='0.0.0.0'
+CERC_KCUSERREG_API_URL='http://keycloak:8080/auth'
+CERC_KCUSERREG_REG_USER="${KEYCLOAK_ADMIN}"
+CERC_KCUSERREG_REG_PW="${KEYCLOAK_ADMIN_PASSWORD}"
+CERC_KCUSERREG_REG_CLIENT_ID='admin-cli'
+CERC_KCUSERREG_TARGET_REALM=cerc
+CERC_KCUSERREG_TARGET_GROUPS=eth
+CERC_KCUSERREG_CREATE_ENABLED=true

stack_orchestrator/data/config/mainnet-blast/lighthouse.env

@@ -0,0 +1,33 @@
+# Enable startup script debug output.
+CERC_SCRIPT_DEBUG=false
+
+# Specify any other lighthouse CLI options.
+LIGHTHOUSE_OPTS=""
+
+# Override the advertised public IP (optional)
+# --enr-address
+#LIGHTHOUSE_ENR_ADDRESS=""
+
+# --checkpoint-sync-url
+LIGHTHOUSE_CHECKPOINT_SYNC_URL="https://beaconstate.ethstaker.cc"
+
+# --checkpoint-sync-url-timeout
+LIGHTHOUSE_CHECKPOINT_SYNC_URL_TIMEOUT=300
+
+# --datadir
+LIGHTHOUSE_DATADIR=/data
+
+# --debug-level
+LIGHTHOUSE_DEBUG_LEVEL=info
+
+# --http-port
+LIGHTHOUSE_HTTP_PORT=5052
+
+# --execution-jwt
+LIGHTHOUSE_JWTSECRET=/etc/mainnet-eth/jwtsecret
+
+# --metrics-port
+LIGHTHOUSE_METRICS_PORT=5054
+
+# --port --enr-udp-port --enr-tcp-port
+LIGHTHOUSE_NETWORK_PORT=9000

stack_orchestrator/data/config/mainnet-blast/mainnet.config

@@ -0,0 +1,2 @@
+GETH_ROLLUP_SEQUENCERHTTP=https://sequencer.blast.io
+OP_NODE_P2P_BOOTNODES=enr:-J64QGwHl9uYLfC_cnmxSA6wQH811nkOWJDWjzxqkEUlJoZHWvI66u-BXgVcPCeMUmg0dBpFQAPotFchG67FHJMZ9OSGAY3d6wevgmlkgnY0gmlwhANizeSHb3BzdGFja4Sx_AQAiXNlY3AyNTZrMaECg4pk0cskPAyJ7pOmo9E6RqGBwV-Lex4VS9a3MQvu7PWDdGNwgnZhg3VkcIJ2YQ,enr:-J64QDge2jYBQtcNEpRqmKfci5E5BHAhNBjgv4WSdwH1_wPqbueq2bDj38-TSW8asjy5lJj1Xftui6Or8lnaYFCqCI-GAY3d6wf3gmlkgnY0gmlwhCO2D9yHb3BzdGFja4Sx_AQAiXNlY3AyNTZrMaEDo4aCTq7pCEN8om9U5n_VyWdambGnQhwHNwKc8o-OicaDdGNwgnZhg3VkcIJ2YQ

stack_orchestrator/data/config/mainnet-blast/rollup.json

@@ -0,0 +1,32 @@
+{
+    "genesis": {
+      "l1": {
+        "hash": "0xfcfb8d586bdae763f1189988789211c69eb893a895e7ba48be3ca6289f0941b7",
+        "number": 19300102
+      },
+      "l2": {
+        "hash": "0xb689b35ef29d0bec5816938e0e52683c7257d2e325420ea69b739a2be4754b89",
+        "number": 0
+      },
+      "l2_time": 1708809815,
+      "system_config": {
+        "batcherAddr": "0x415c8893d514f9bc5211d36eeda4183226b84aa7",
+        "overhead": "0x00000000000000000000000000000000000000000000000000000000000000bc",
+        "scalar": "0x00000000000000000000000000000000000000000000000000000000000a6fe0",
+        "gasLimit": 30000000
+      }
+    },
+    "block_time": 2,
+    "max_sequencer_drift": 600,
+    "seq_window_size": 3600,
+    "channel_timeout": 300,
+    "l1_chain_id": 1,
+    "l2_chain_id": 81457,
+    "regolith_time": 0,
+    "canyon_time": 0,
+    "batch_inbox_address": "0xff00000000000000000000000000000000081457",
+    "deposit_contract_address": "0x0ec68c5b10f21effb74f2a5c61dfe6b08c0db6cb",
+    "l1_system_config_address": "0x5531dcff39ec1ec727c4c5d2fc49835368f805a9",
+    "protocol_versions_address": "0x0000000000000000000000000000000000000000"
+  }
+  

stack_orchestrator/data/config/mainnet-blast/run-lighthouse.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+if [[ "true" == "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi
+
+ENR_OPTS=""
+if [[ -n "$LIGHTHOUSE_ENR_ADDRESS" ]]; then
+  ENR_OPTS="--enr-address $LIGHTHOUSE_ENR_ADDRESS"
+fi
+
+exec lighthouse bn \
+  --checkpoint-sync-url "$LIGHTHOUSE_CHECKPOINT_SYNC_URL" \
+  --checkpoint-sync-url-timeout ${LIGHTHOUSE_CHECKPOINT_SYNC_URL_TIMEOUT} \
+  --datadir "$LIGHTHOUSE_DATADIR" \
+  --debug-level $LIGHTHOUSE_DEBUG_LEVEL \
+  --disable-deposit-contract-sync \
+  --disable-upnp \
+  --enr-tcp-port $LIGHTHOUSE_NETWORK_PORT \
+  --enr-udp-port $LIGHTHOUSE_NETWORK_PORT \
+  --execution-endpoint "$LIGHTHOUSE_EXECUTION_ENDPOINT" \
+  --execution-jwt /etc/mainnet-eth/jwtsecret \
+  --http \
+  --http-address 0.0.0.0 \
+  --http-port $LIGHTHOUSE_HTTP_PORT \
+  --metrics \
+  --metrics-address=0.0.0.0 \
+  --metrics-port $LIGHTHOUSE_METRICS_PORT \
+  --network mainnet \
+  --port $LIGHTHOUSE_NETWORK_PORT \
+  $ENR_OPTS $LIGHTHOUSE_OPTS

stack_orchestrator/data/config/mainnet-laconicd/registry-cli-config-template.yml

@@ -1,9 +1,9 @@
 services:
-  cns:
-    restEndpoint: 'http://laconicd:1317'
+  registry:
+    rpcEndpoint: 'http://laconicd:26657'
     gqlEndpoint: 'http://laconicd:9473/api'
     userKey: REPLACE_WITH_MYKEY
     bondId:
     chainId: laconic_9000-1
     gas: 250000
-    fees: 200000aphoton
+    fees: 2000000alnt

stack_orchestrator/data/config/mainnet-laconicd/scripts/run-laconicd.sh

@@ -1,18 +1,15 @@
 #!/bin/sh
-if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
     set -x
 fi
 
 #TODO: pass these in from the caller
-TRACE="--trace"
 LOGLEVEL="info"
 
 laconicd start \
     --pruning=nothing \
-    --evm.tracer=json $TRACE \
     --log_level $LOGLEVEL \
-    --minimum-gas-prices=0.0001aphoton \
-    --json-rpc.api eth,txpool,personal,net,debug,web3,miner \
+    --minimum-gas-prices=1alnt \
     --api.enable \
     --gql-server \
     --gql-playground

stack_orchestrator/data/config/monitoring/grafana/provisioning/datasources/graph-node-postgres.yml

@@ -0,0 +1,20 @@
+apiVersion: 1
+
+datasources:
+  - name: Graph Node Postgres
+    type: postgres
+    jsonData:
+      database: graph-node
+      sslmode: 'disable'
+      maxOpenConns: 100
+      maxIdleConns: 100
+      maxIdleConnsAuto: true
+      connMaxLifetime: 14400
+      postgresVersion: 1411 # 903=9.3, 1000=10, 1411=14.11
+      timescaledb: false
+    user: graph-node
+    # # Add URL for graph-node database
+    # url: graph-node-db:5432
+    # # Set password for graph-node database
+    # secureJsonData:
+    #   password: 'password'

stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml

@@ -45,7 +45,18 @@ scrape_configs:
     metrics_path: /metrics
     scheme: http
     static_configs:
-      - targets: ['chain-head-exporter:5000']
+      - targets: ['ethereum-chain-head-exporter:5000']
+        labels:
+          instance: 'external'
+          chain: 'ethereum'
+      - targets: ['filecoin-chain-head-exporter:5000']
+        labels:
+          instance: 'external'
+          chain: 'filecoin'
+      - targets: ['graph-node-upstream-head-exporter:5000']
+        labels:
+          instance: 'graph-node'
+          chain: 'filecoin'
 
   - job_name: 'postgres'
     scrape_interval: 30s
@@ -65,3 +76,20 @@ scrape_configs:
         target_label: instance
       - target_label: __address__
         replacement: postgres-exporter:9187
+
+  - job_name: laconicd
+    metrics_path: /metrics
+    scrape_interval: 30s
+    static_configs:
+      # Add laconicd REST endpoint target with host and port (1317)
+      # - targets: ['example-host:1317']
+    params:
+      format: ['prometheus']
+
+  - job_name: graph-node
+    metrics_path: /metrics
+    scrape_interval: 30s
+    scheme: http
+    static_configs:
+      # Add graph-node targets to be monitored below
+      # - targets: ['graph-node:8040']

stack_orchestrator/data/config/monitoring/subgraph-alert-rules.yml

@@ -0,0 +1,64 @@
+apiVersion: 1
+groups:
+  - orgId: 1
+    name: subgraph
+    folder: SubgraphAlerts
+    interval: 30s
+    rules:
+      - uid: b2a9144b-6104-46fc-92b5-352f4e643c4c
+        title: subgraph_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                  type: prometheus
+                  uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: ethereum_chain_head_number - on(network) group_right deployment_head{deployment=~"REPLACE_WITH_SUBGRAPH_IDS"}
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                  - evaluator:
+                      params:
+                          - 15
+                          - 0
+                      type: gt
+                    operator:
+                      type: and
+                    query:
+                      params: []
+                    reducer:
+                      params: []
+                      type: avg
+                    type: query
+              datasource:
+                  name: Expression
+                  type: __expr__
+                  uid: __expr__
+              expression: diff
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: threshold
+        noDataState: OK
+        execErrState: Alerting
+        for: 5m
+        annotations:
+          summary: Subgraph deployment {{ index $labels "deployment" }} is falling behind head by {{ index $values "diff" }}
+        labels: {}
+        isPaused: false

stack_orchestrator/data/config/monitoring/update-grafana-alerts-config.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+echo Using CERC_GRAFANA_ALERTS_SUBGRAPH_IDS ${CERC_GRAFANA_ALERTS_SUBGRAPH_IDS}
+
+# Replace subgraph ids in subgraph alerting config
+# Note: Requires the grafana container to be run with user root
+if [ -n "$CERC_GRAFANA_ALERTS_SUBGRAPH_IDS" ]; then
+  sed -i "s/REPLACE_WITH_SUBGRAPH_IDS/$CERC_GRAFANA_ALERTS_SUBGRAPH_IDS/g" /etc/grafana/provisioning/alerting/subgraph-alert-rules.yml
+fi

stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml

@@ -24,7 +24,7 @@ groups:
                 uid: PBFA97CFB590B2093
               disableTextWrap: false
               editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
               fullMetaSearch: false
               includeNullMetadata: true
               instant: true
@@ -50,22 +50,6 @@ groups:
               legendFormat: __auto
               range: false
               refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
           - refId: condition
             relativeTimeRange:
               from: 600
@@ -116,7 +100,7 @@ groups:
                 uid: PBFA97CFB590B2093
               disableTextWrap: false
               editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
               fullMetaSearch: false
               includeNullMetadata: true
               instant: true
@@ -142,22 +126,6 @@ groups:
               legendFormat: __auto
               range: false
               refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
           - refId: condition
             relativeTimeRange:
               from: 600
@@ -208,7 +176,7 @@ groups:
                 uid: PBFA97CFB590B2093
               disableTextWrap: false
               editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
               fullMetaSearch: false
               includeNullMetadata: true
               instant: true
@@ -234,22 +202,6 @@ groups:
               legendFormat: __auto
               range: false
               refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
           - refId: condition
             relativeTimeRange:
               from: 600
@@ -300,7 +252,7 @@ groups:
                 uid: PBFA97CFB590B2093
               disableTextWrap: false
               editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
               fullMetaSearch: false
               includeNullMetadata: true
               instant: true
@@ -326,22 +278,6 @@ groups:
               legendFormat: __auto
               range: false
               refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
           - refId: condition
             relativeTimeRange:
               from: 600
@@ -392,7 +328,7 @@ groups:
                 uid: PBFA97CFB590B2093
               disableTextWrap: false
               editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
               fullMetaSearch: false
               includeNullMetadata: true
               instant: true
@@ -418,22 +354,6 @@ groups:
               legendFormat: __auto
               range: false
               refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
           - refId: condition
             relativeTimeRange:
               from: 600
@@ -484,7 +404,7 @@ groups:
                 uid: PBFA97CFB590B2093
               disableTextWrap: false
               editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
               fullMetaSearch: false
               includeNullMetadata: true
               instant: true
@@ -510,22 +430,6 @@ groups:
               legendFormat: __auto
               range: false
               refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
           - refId: condition
             relativeTimeRange:
               from: 600
@@ -576,7 +480,7 @@ groups:
                 uid: PBFA97CFB590B2093
               disableTextWrap: false
               editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="linear_star_release", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="linear_star_release", kind="latest_indexed"}
               fullMetaSearch: false
               includeNullMetadata: true
               instant: true
@@ -602,22 +506,6 @@ groups:
               legendFormat: __auto
               range: false
               refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
           - refId: condition
             relativeTimeRange:
               from: 600
@@ -668,7 +556,7 @@ groups:
                 uid: PBFA97CFB590B2093
               disableTextWrap: false
               editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
               fullMetaSearch: false
               includeNullMetadata: true
               instant: true
@@ -694,22 +582,6 @@ groups:
               legendFormat: __auto
               range: false
               refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
           - refId: condition
             relativeTimeRange:
               from: 600
@@ -762,7 +634,7 @@ groups:
                 uid: PBFA97CFB590B2093
               disableTextWrap: false
               editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
               fullMetaSearch: false
               includeNullMetadata: true
               instant: true
@@ -788,22 +660,6 @@ groups:
               legendFormat: __auto
               range: false
               refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
           - refId: condition
             relativeTimeRange:
               from: 600
@@ -854,7 +710,163 @@ groups:
                 uid: PBFA97CFB590B2093
               disableTextWrap: false
               editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+
+      # Ajna
+      - uid: ajna_diff_external
+        title: ajna_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="ajna", instance="ajna", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      
+      # Secured Finance
+      - uid: secured_finance_diff_external
+        title: secured_finance_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="secured-finance", instance="secured-finance", kind="latest_indexed"}
               fullMetaSearch: false
               includeNullMetadata: true
               instant: true
@@ -880,22 +892,6 @@ groups:
               legendFormat: __auto
               range: false
               refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
           - refId: condition
             relativeTimeRange:
               from: 600

stack_orchestrator/data/config/watcher-ajna/start-job-runner.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+set -u
+
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
+
+# Read in the config template TOML file and modify it
+WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
+WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
+
+# Write the modified content to a new file
+echo "$WATCHER_CONFIG" > environments/local.toml
+
+echo "Running job-runner..."
+DEBUG=vulcanize:* exec node --enable-source-maps dist/job-runner.js

stack_orchestrator/data/config/watcher-ajna/start-server.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+set -u
+
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
+
+# Read in the config template TOML file and modify it
+WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
+WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
+
+# Write the modified content to a new file
+echo "$WATCHER_CONFIG" > environments/local.toml
+
+echo "Running server..."
+DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js

stack_orchestrator/data/config/watcher-ajna/watcher-config-template.toml

@@ -0,0 +1,110 @@
+[server]
+  host = "0.0.0.0"
+  port = 3008
+  kind = "active"
+
+  # Checkpointing state.
+  checkpointing = true
+
+  # Checkpoint interval in number of blocks.
+  checkpointInterval = 2000
+
+  # Enable state creation
+  # CAUTION: Disable only if state creation is not desired or can be filled subsequently
+  enableState = false
+
+  subgraphPath = "./subgraph-build"
+
+  # Interval to restart wasm instance periodically
+  wasmRestartBlocksInterval = 20
+
+  # Interval in number of blocks at which to clear entities cache.
+  clearEntitiesCacheInterval = 1000
+
+  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
+  rpcSupportsBlockHashParam = false
+
+  # Server GQL config
+  [server.gql]
+    path = "/"
+
+    # Max block range for which to return events in eventsInRange GQL query.
+    # Use -1 for skipping check on block range.
+    maxEventsBlockRange = 1000
+
+    # Log directory for GQL requests
+    logDir = "./gql-logs"
+
+    # GQL cache settings
+    [server.gql.cache]
+      enabled = true
+
+      # Max in-memory cache size (in bytes) (default 8 MB)
+      # maxCacheSize
+
+      # GQL cache-control max-age settings (in seconds)
+      maxAge = 15
+      timeTravelMaxAge = 86400 # 1 day
+
+[metrics]
+  host = "0.0.0.0"
+  port = 9000
+  [metrics.gql]
+    port = 9001
+
+[database]
+  type = "postgres"
+  host = "ajna-watcher-db"
+  port = 5432
+  database = "ajna-watcher"
+  username = "vdbm"
+  password = "password"
+  synchronize = true
+  logging = false
+
+[upstream]
+  [upstream.ethServer]
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS
+
+    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
+    rpcClient = true
+
+    # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint
+    isFEVM = true
+
+    # Boolean flag to filter event logs by contracts
+    filterLogsByAddresses = true
+    # Boolean flag to filter event logs by topics
+    filterLogsByTopics = true
+
+  [upstream.cache]
+    name = "requests"
+    enabled = false
+    deleteOnStart = false
+
+[jobQueue]
+  dbConnectionString = "postgres://vdbm:password@ajna-watcher-db/ajna-watcher-job-queue"
+  maxCompletionLagInSecs = 300
+  jobDelayInMilliSecs = 100
+  eventsInBatch = 50
+  subgraphEventsOrder = true
+  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
+  blockDelayInMilliSecs = 30000
+
+  # Number of blocks by which block processing lags behind head
+  blockProcessingOffset = 0
+
+  # Boolean to switch between modes of processing events when starting the server.
+  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
+  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
+  useBlockRanges = true
+
+  # Block range in which logs are fetched during historical blocks processing
+  historicalLogsBlockRange = 2000
+
+  # Max block range of historical processing after which it waits for completion of events processing
+  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
+  historicalMaxFetchAhead = 10000
+
+  # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
+  maxNewBlockRetries = 3

stack_orchestrator/data/config/watcher-azimuth/start-job-runner.sh

@@ -4,16 +4,19 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
   set -x
 fi
 
-echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}"
-echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
+echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL_ENDPOINT}"
 echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}"
 
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 # Replace env variables in template TOML file
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \
-          s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}|g; \
+          s|REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT|${CERC_IPLD_ETH_GQL_ENDPOINT}|g; \
           s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ")
 
 # Write the modified content to a new file

stack_orchestrator/data/config/watcher-azimuth/start-server.sh

@@ -4,16 +4,19 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
   set -x
 fi
 
-echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}"
-echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
+echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL_ENDPOINT}"
 echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}"
 
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 # Replace env variables in template TOML file
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \
-          s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}|g; \
+          s|REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT|${CERC_IPLD_ETH_GQL_ENDPOINT}|g; \
           s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ")
 
 # Write the modified content to a new file

stack_orchestrator/data/config/watcher-azimuth/watcher-config-template.toml

@@ -1,6 +1,7 @@
 [server]
   host = "0.0.0.0"
-  maxSimultaneousRequests = -1
+  [server.gql]
+    maxSimultaneousRequests = -1
 
 [metrics]
   host = "0.0.0.0"
@@ -13,8 +14,8 @@
 
 [upstream]
   [upstream.ethServer]
-    gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL"
-    rpcProviderEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_RPC"
+    gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT"
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS
 
 [jobQueue]
   historicalLogsBlockRange = REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE

stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-job-runner.sh

@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u
 
-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
 
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
 
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml

stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/start-server.sh

@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u
 
-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
 
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
 
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml

stack_orchestrator/data/config/watcher-merkl-sushiswap-v3/watcher-config-template.toml

@@ -2,7 +2,6 @@
   host = "0.0.0.0"
   port = 3008
   kind = "active"
-  gqlPath = '/'
 
   # Checkpointing state.
   checkpointing = true
@@ -22,23 +21,30 @@
   # Interval in number of blocks at which to clear entities cache.
   clearEntitiesCacheInterval = 1000
 
-  # Max block range for which to return events in eventsInRange GQL query.
-  # Use -1 for skipping check on block range.
-  maxEventsBlockRange = 1000
-
   # Flag to specify whether RPC endpoint supports block hash as block tag parameter
   rpcSupportsBlockHashParam = false
 
-  # GQL cache settings
-  [server.gqlCache]
-    enabled = true
+  # Server GQL config
+  [server.gql]
+    path = "/"
 
-    # Max in-memory cache size (in bytes) (default 8 MB)
-    # maxCacheSize
+    # Max block range for which to return events in eventsInRange GQL query.
+    # Use -1 for skipping check on block range.
+    maxEventsBlockRange = 1000
 
-    # GQL cache-control max-age settings (in seconds)
-    maxAge = 15
-    timeTravelMaxAge = 86400 # 1 day
+    # Log directory for GQL requests
+    logDir = "./gql-logs"
+
+    # GQL cache settings
+    [server.gql.cache]
+      enabled = true
+
+      # Max in-memory cache size (in bytes) (default 8 MB)
+      # maxCacheSize
+
+      # GQL cache-control max-age settings (in seconds)
+      maxAge = 15
+      timeTravelMaxAge = 86400 # 1 day
 
 [metrics]
   host = "0.0.0.0"
@@ -58,7 +64,7 @@
 
 [upstream]
   [upstream.ethServer]
-    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS
 
     # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
     rpcClient = true
@@ -69,7 +75,7 @@
     # Boolean flag to filter event logs by contracts
     filterLogsByAddresses = true
     # Boolean flag to filter event logs by topics
-    filterLogsByTopics = false
+    filterLogsByTopics = true
 
   [upstream.cache]
     name = "requests"
@@ -85,6 +91,9 @@
   # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
   blockDelayInMilliSecs = 30000
 
+  # Number of blocks by which block processing lags behind head
+  blockProcessingOffset = 0
+
   # Boolean to switch between modes of processing events when starting the server.
   # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
   # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
@@ -96,3 +105,6 @@
   # Max block range of historical processing after which it waits for completion of events processing
   # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
   historicalMaxFetchAhead = 10000
+
+  # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
+  maxNewBlockRetries = 3

stack_orchestrator/data/config/watcher-mobymask-v3-demo/deploy-and-generate-invite.sh

@@ -1,80 +0,0 @@
-#!/bin/bash
-
-set -e
-if [ -n "$CERC_SCRIPT_DEBUG" ]; then
-  set -x
-fi
-
-# From env:
-# CERC_ETH_RPC_ENDPOINT
-# CERC_MOBYMASK_APP_BASE_URI
-# CERC_DEPLOYED_CONTRACT
-# CERC_PRIVATE_KEY_DEPLOYER
-
-# Check if CERC_DEPLOYED_CONTRACT environment variable set to skip contract deployment
-if [ -n "$CERC_DEPLOYED_CONTRACT" ]; then
-  echo "CERC_DEPLOYED_CONTRACT is set to '$CERC_DEPLOYED_CONTRACT'"
-  echo "Skipping contract deployment"
-  exit 0
-fi
-
-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
-
-# Wait till ETH RPC endpoint is available with block number > 1
-retry_interval=5
-while true; do
-  block_number_hex=$(curl -s -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' ${CERC_ETH_RPC_ENDPOINT} | jq -r '.result')
-
-  # Check if the request call was successful
-  if [ $? -ne 0 ]; then
-    echo "RPC endpoint not yet available, retrying in $retry_interval seconds..."
-    sleep $retry_interval
-    continue
-  fi
-
-  # Convert hex to decimal
-  block_number_dec=$(printf %u ${block_number_hex})
-
-  # Check if block number is > 1 to avoid failures in the deployment
-  if [ "$block_number_dec" -ge 1 ]; then
-    echo "RPC endpoint is up"
-    break
-  else
-    echo "RPC endpoint not yet available, retrying in $retry_interval seconds..."
-    sleep $retry_interval
-    continue
-  fi
-done
-
-echo "Using CERC_PRIVATE_KEY_DEPLOYER from env"
-
-# Create the required JSON and write it to a file
-secrets_file="secrets.json"
-secrets_json=$(jq -n \
-  --arg privateKey "$CERC_PRIVATE_KEY_DEPLOYER" \
-  --arg rpcUrl "$CERC_ETH_RPC_ENDPOINT" \
-  --arg baseURI "$CERC_MOBYMASK_APP_BASE_URI" \
-  '.privateKey = $privateKey | .rpcUrl = $rpcUrl | .baseURI = $baseURI')
-echo "$secrets_json" > "${secrets_file}"
-
-export RPC_URL="${CERC_ETH_RPC_ENDPOINT}"
-
-# Check and exit if a deployment already exists (on restarts)
-if [ -f ./config.json ]; then
-  echo "config.json already exists, checking the contract deployment"
-
-  # Read JSON file
-  deployment_details=$(cat config.json)
-  deployed_contract=$(echo "$deployment_details" | jq -r '.address')
-
-  cd ../hardhat
-  if yarn verifyDeployment --network optimism --contract "${deployed_contract}"; then
-    echo "Deployment verfication successful"
-    cd ../server
-  else
-    echo "Deployment verfication failed, please clear MobyMask deployment volume before starting"
-    exit 1
-  fi
-fi
-
-npm run deployAndGenerateInvite

stack_orchestrator/data/config/watcher-mobymask-v3-demo/local/config.env

@@ -1,16 +0,0 @@
-# Watchers
-
-CERC_ETH_RPC_ENDPOINT="http://op-geth:8545"
-
-# L1 pre-funded account #4, funded on L2 as well
-CERC_PRIVATE_KEY_DEPLOYER="0xbe4aa664815ea3bc3d63118649a733f6c96b243744310806ecb6d96359ab62cf"
-CERC_MOBYMASK_APP_BASE_URI="http://127.0.0.1:3004/#"
-
-CERC_ETH_RPC_QUERY_ENDPOINT="http://op-geth:8545"
-CERC_ETH_RPC_MUTATION_ENDPOINT="http://op-geth:8545"
-CERC_NITRO_CHAIN_URL="http://op-geth:8546"
-
-# App
-
-CERC_APP_WATCHER_URL="http://localhost:3001"
-CERC_SNAP_URL=http://localhost:8080

stack_orchestrator/data/config/watcher-mobymask-v3-demo/local/fund-accounts-on-l2.sh

@@ -1,72 +0,0 @@
-#!/bin/bash
-set -e
-if [ -n "$CERC_SCRIPT_DEBUG" ]; then
-  set -x
-fi
-
-echo "*************************************"
-
-CERC_L1_RPC="${CERC_L1_RPC:-${DEFAULT_CERC_L1_RPC}}"
-CERC_L1_CHAIN_ID="${CERC_L1_CHAIN_ID:-${DEFAULT_CERC_L1_CHAIN_ID}}"
-CERC_L1_ACCOUNTS_CSV_URL="${CERC_L1_ACCOUNTS_CSV_URL:-${DEFAULT_CERC_L1_ACCOUNTS_CSV_URL}}"
-
-# Fixture accounts
-WATCHER_1_ACCOUNT=0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266
-WATCHER_2_ACCOUNT=0x70997970C51812dc3A010C7d01b50e0d17dc79C8
-WATCHER_3_ACCOUNT=0x15d34AAf54267DB7D7c367839AAf71A00a2C6A65
-NITRO_APP_ACCOUNT=0xbDA5747bFD65F08deb54cb465eB87D40e51B197E
-
-WATCHER_1_ACCOUNT_KEY=0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80
-WATCHER_2_ACCOUNT_KEY=0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d
-WATCHER_3_ACCOUNT_KEY=0x47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a
-NITRO_APP_ACCOUNT_KEY=0x689af8efa8c651a91ad287602527f3af2fe9f6501a7ac4b061667b5a93e037fd
-
-# Check if watcher 1 account already funded
-WATCHER_1_ACCOUNT_BALANCE=$(cast balance ${WATCHER_1_ACCOUNT} --rpc-url $CERC_L1_RPC)
-echo "WATCHER_1_ACCOUNT_BALANCE ${WATCHER_1_ACCOUNT_BALANCE}"
-if [ "$WATCHER_1_ACCOUNT_BALANCE" != "0" ]; then
-  echo "Watcher account already funded, exiting."
-  exit 0
-fi
-
-echo "Funding accounts on L2"
-
-# Fetch the L1 funded accounts
-curl -o accounts.csv $CERC_L1_ACCOUNTS_CSV_URL
-
-# Use the accounts other than the ones used for Optimism deployment
-ACCOUNT_1=$(awk -F ',' 'NR == 5 {print $2}' accounts.csv)
-ACCOUNT_1_KEY=$(awk -F ',' 'NR == 5 {print $3}' accounts.csv)
-
-ACCOUNT_2=$(awk -F ',' 'NR == 6 {print $2}' accounts.csv)
-ACCOUNT_2_KEY=$(awk -F ',' 'NR == 6 {print $3}' accounts.csv)
-
-# Get the bridge contract address
-DEPLOYMENT_CONTEXT="$CERC_L1_CHAIN_ID"
-BRIDGE=$(cat /l1-deployment/$DEPLOYMENT_CONTEXT/L1StandardBridgeProxy.json | jq -r .address)
-
-# Send balance to bridge contract on L1
-cast send --rpc-url $CERC_L1_RPC --from $ACCOUNT_1 --value 10000ether $BRIDGE --private-key $ACCOUNT_1_KEY
-cast send --rpc-url $CERC_L1_RPC --from $ACCOUNT_2 --value 10000ether $BRIDGE --private-key $ACCOUNT_2_KEY
-
-echo "Following accounts have been funded; use them for transactions on L2:"
-echo "${ACCOUNT_1}"
-echo "${ACCOUNT_2}"
-
-echo "*************************************"
-echo "Funding the watcher and app Nitro accounts on L2"
-
-cast send --rpc-url $CERC_L1_RPC --from $ACCOUNT_1 --value 200ether $WATCHER_1_ACCOUNT --private-key $ACCOUNT_1_KEY
-cast send --rpc-url $CERC_L1_RPC --from $ACCOUNT_1 --value 200ether $WATCHER_2_ACCOUNT --private-key $ACCOUNT_1_KEY
-cast send --rpc-url $CERC_L1_RPC --from $ACCOUNT_1 --value 200ether $WATCHER_3_ACCOUNT --private-key $ACCOUNT_1_KEY
-cast send --rpc-url $CERC_L1_RPC --from $ACCOUNT_1 --value 200ether $NITRO_APP_ACCOUNT --private-key $ACCOUNT_1_KEY
-
-cast send --rpc-url $CERC_L1_RPC --from $WATCHER_1_ACCOUNT --value 100ether $BRIDGE --private-key $WATCHER_1_ACCOUNT_KEY
-sleep 5
-cast send --rpc-url $CERC_L1_RPC --from $WATCHER_2_ACCOUNT --value 100ether $BRIDGE --private-key $WATCHER_2_ACCOUNT_KEY
-sleep 5
-cast send --rpc-url $CERC_L1_RPC --from $WATCHER_3_ACCOUNT --value 100ether $BRIDGE --private-key $WATCHER_3_ACCOUNT_KEY
-sleep 5
-cast send --rpc-url $CERC_L1_RPC --from $NITRO_APP_ACCOUNT --value 100ether $BRIDGE --private-key $NITRO_APP_ACCOUNT_KEY
-
-echo "Done, exiting."

stack_orchestrator/data/config/watcher-mobymask-v3-demo/local/keys/watcher-1.json

@@ -1,21 +0,0 @@
-{
-  "peerId": {
-    "id": "12D3KooWAMjBkFCT9DtCnSDcxftxJzSuTBvzVojabv64cnEvX4AZ",
-    "privKey": "CAESQAKCrnY0QKTky1I18fqn+VPydXGUv1NYiV+nVKqBFkw/CAjE9sKKIDGnYAo8mivnI6dngFenERY+0Q8AJrPTaXY=",
-    "pubKey": "CAESIAgIxPbCiiAxp2AKPJor5yOnZ4BXpxEWPtEPACaz02l2"
-  },
-  "relayPeerId": {
-    "id": "12D3KooWNQJDGtH3YAZk2qG1ZbcoQvw6HaD814pfd9HHQthgX63K",
-    "privKey": "CAESQGc996T2OuyV3aY+hnna+GIexKBEcE6jDNX/RaofHr58uv2EYxXm4kcL2tO2tKgnPgtkegHFdjqy6h/xOA+YMZo=",
-    "pubKey": "CAESILr9hGMV5uJHC9rTtrSoJz4LZHoBxXY6suof8TgPmDGa"
-  },
-  "peer": {
-    "privateKey": "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80",
-    "nitroAddress": "0xAAA6628Ec44A8a742987EF3A114dDFE2D4F7aDCE",
-    "nitroPrivateKey": "2d999770f7b5d49b694080f987b82bbc9fc9ac2b4dcc10b0f8aba7d700f69c6d"
-  },
-  "consensus": {
-    "publicKey": "02cd17b05ca998955be5ca7bf4fd4531243d438f1aae7ce8a0ed5159f53cee5b40",
-    "privateKey": "67d80505614bdf61fca11cbad31d93acb2c7df1c653dc25975d77d05f05f154f"
-  }
-}

stack_orchestrator/data/config/watcher-mobymask-v3-demo/local/keys/watcher-2.json

@@ -1,21 +0,0 @@
-{
-  "peerId": {
-    "id": "12D3KooWBNEbY3QS4y23ngupDw9PDc4bvNvRJGVRejjV9EZLjux5",
-    "privKey": "CAESQGSTw0ymvn8+wX9Dbvyr4/Gib1q2voe0CC0VyeClMQP6FwW14x0fpRbBIx0XhLdxWHkRndphVg3gVAHyC+7ZI8o=",
-    "pubKey": "CAESIBcFteMdH6UWwSMdF4S3cVh5EZ3aYVYN4FQB8gvu2SPK"
-  },
-  "relayPeerId": {
-    "id": "12D3KooWPpBt2coC5BKEsewogHSkdgdyY9guUHJBgJFZoh66R4UL",
-    "privKey": "CAESQBdoGZzpiHsE8GlIhGGP+tl6A13sIPxOhgSwsWZR4cwmz/f7627X5GQRLD41ikH6oZ7ZeSbiiPsBtelL1+t3M80=",
-    "pubKey": "CAESIM/3++tu1+RkESw+NYpB+qGe2Xkm4oj7AbXpS9frdzPN"
-  },
-  "peer": {
-    "privateKey": "59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d",
-    "nitroAddress": "0xBBB676f9cFF8D242e9eaC39D063848807d3D1D94",
-    "nitroPrivateKey": "0279651921cd800ac560c21ceea27aab0107b67daf436cdd25ce84cad30159b4"
-  },
-  "consensus": {
-    "publicKey": "029c8035b3e9401b8f178f7c37285b5cb22501e017340e2058b3b842f2a1f0ae45",
-    "privateKey": "0261008e8e3ec808168e99333599da38ca59a056a2ae4510a6ad3d8b5cb0918c"
-  }
-}

stack_orchestrator/data/config/watcher-mobymask-v3-demo/local/keys/watcher-3.json

@@ -1,21 +0,0 @@
-{
-  "peerId": {
-    "id": "12D3KooWSRH6ftgkAZsKZK7UX1Zr6Hx6YAsEepHqzopFszqfTxxi",
-    "privKey": "CAESQHBjlHxfVhZ2gXsBItrIEEgSGKcjMkFiGs3PPz9E3ace9qyWEkvR4oit5ve9SAROVoh20hoa42IC91NIafMaqws=",
-    "pubKey": "CAESIPaslhJL0eKIreb3vUgETlaIdtIaGuNiAvdTSGnzGqsL"
-  },
-  "relayPeerId": {
-    "id": "12D3KooWLudDC3NnS8k9xLiqTZcS9xbFNrGSttu5GPynQgNgenPW",
-    "privKey": "CAESQHuEXPB1hTQATlZaNRQnAfSvnfYUeLF9PjSNKp9L33K2pMl37tTBYghnfQ0yluse0zHOIXRJM+9+ARn9+jIr0gU=",
-    "pubKey": "CAESIKTJd+7UwWIIZ30NMpbrHtMxziF0STPvfgEZ/foyK9IF"
-  },
-  "peer": {
-    "privateKey": "47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a",
-    "nitroAddress": "0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d",
-    "nitroPrivateKey": "58368d20ff12f17669c06158c21d885897aa56f9be430edc789614bf9851d53f"
-  },
-  "consensus": {
-    "publicKey": "039160c244a7ad8be16a64bdb69e6dbacdcfe20b37076792a0d06032a8097468ca",
-    "privateKey": "8894685fe81001d75662b079905472699373967451d1255ee5fc669d0a09a9ca"
-  }
-}

stack_orchestrator/data/config/watcher-mobymask-v3-demo/mobymask-app-start.sh

@@ -1,68 +0,0 @@
-#!/bin/bash
-
-set -e
-if [ -n "$CERC_SCRIPT_DEBUG" ]; then
-  set -x
-fi
-
-watcher_keys_dir="/server/keys"
-
-# Configure relay nodes to connect to
-# Note: Assuming host ports for watchers (1-3) relay nodes
-RELAY_NODES="[\"/ip4/127.0.0.1/tcp/9090/ws/p2p/$(jq -r '.relayPeerId.id' ${watcher_keys_dir}/watcher-1.json)\", \"/ip4/127.0.0.1/tcp/9091/ws/p2p/$(jq -r '.relayPeerId.id' ${watcher_keys_dir}/watcher-2.json)\", \"/ip4/127.0.0.1/tcp/9092/ws/p2p/$(jq -r '.relayPeerId.id' ${watcher_keys_dir}/watcher-3.json)\"]"
-
-# Configure watcher (1) Nitro account to make payments to
-PAYMENT_NITRO_ADDRESS=$(jq -r '.peer.nitroAddress' ${watcher_keys_dir}/watcher-1.json)
-
-echo "Using RELAY_NODES $RELAY_NODES"
-echo "Using PAYMENT_NITRO_ADDRESS $PAYMENT_NITRO_ADDRESS"
-
-# Use config from mounted volume
-echo "Taking config for deployed contracts from mounted volume"
-while [ ! -f /server/config.json ]; do
-  echo "Config not found, retrying in 5 seconds..."
-  sleep 5
-done
-
-# Get deployed contract address and chain id
-DEPLOYED_CONTRACT=$(jq -r '.address' /server/config.json | tr -d '"')
-CHAIN_ID=$(jq -r '.chainId' /server/config.json)
-
-echo "Using DEPLOYED_CONTRACT $DEPLOYED_CONTRACT"
-echo "Using CHAIN_ID $CHAIN_ID"
-
-nitro_addresses_file="/nitro/nitro-addresses.json"
-nitro_addresses_destination_file="/app/src/utils/nitro-addresses.json"
-if [ -f ${nitro_addresses_file} ]; then
-  echo "Using Nitro addresses from ${nitro_addresses_file}:"
-  cat "$nitro_addresses_file"
-  cat "$nitro_addresses_file" > "$nitro_addresses_destination_file"
-else
-  echo "Nitro addresses not available"
-  exit 1
-fi
-
-# Export config values in a json file
-app_config_file="/app/src/utils/config.json"
-app_config_json=$(jq -n \
-  --arg name "MobyMask" \
-  --argjson enableDebugInfo true \
-  --arg address "$DEPLOYED_CONTRACT" \
-  --argjson chainId "$CHAIN_ID" \
-  --argjson relayNodes "$RELAY_NODES" \
-  --argjson denyMultiaddrs "[]" \
-  --arg pubsub "" \
-  --argjson directPeers "[]" \
-  '.name = $name | .address = $address | .chainId = $chainId | .relayNodes = $relayNodes | .peer.enableDebugInfo = $enableDebugInfo | .peer.denyMultiaddrs = $denyMultiaddrs | .peer.pubsub = $pubsub | .peer.directPeers = $directPeers')
-echo "$app_config_json" > "${app_config_file}"
-
-echo "Using CERC_APP_WATCHER_URL $CERC_APP_WATCHER_URL"
-echo "Using CERC_SNAP_URL $CERC_SNAP_URL"
-
-REACT_APP_DEBUG_PEER=true \
-REACT_APP_WATCHER_URI="$CERC_APP_WATCHER_URL/graphql" \
-REACT_APP_PAY_TO_NITRO_ADDRESS="$PAYMENT_NITRO_ADDRESS" \
-REACT_APP_SNAP_ORIGIN="local:$CERC_SNAP_URL" \
-yarn build
-
-http-server -p 80 /app/build

stack_orchestrator/data/config/watcher-mobymask-v3-demo/start-server.sh

@@ -1,119 +0,0 @@
-#!/bin/bash
-
-set -e
-if [ -n "$CERC_SCRIPT_DEBUG" ]; then
-  set -x
-fi
-
-watcher_keys_dir="./keys"
-
-echo "Watcher index: ${CERC_WATCHER_INDEX}"
-echo "Using RPC query endpoint ${CERC_ETH_RPC_QUERY_ENDPOINT}"
-echo "Using RPC mutation endpoint ${CERC_ETH_RPC_MUTATION_ENDPOINT}"
-echo "Using Nitro chain URL ${CERC_NITRO_CHAIN_URL}"
-
-WATCHER_DB="mobymask-watcher-${CERC_WATCHER_INDEX}"
-
-# Use the docker container's host IP for relay multiaddr in peer config
-CERC_RELAY_MULTIADDR="/dns4/mobymask-watcher-${CERC_WATCHER_INDEX}-server/tcp/9090/ws/p2p/$(jq -r '.relayPeerId.id' ${watcher_keys_dir}/watcher-${CERC_WATCHER_INDEX}.json)"
-
-# Assign deployed contract address from server config (created by mobymask container after deploying contract)
-CONTRACT_ADDRESS=$(jq -r '.address' /server/config.json | tr -d '"')
-
-nitro_addresses_file="/nitro/nitro-addresses.json"
-nitro_addresses_destination_file="./src/nitro-addresses.json"
-if [ -f ${nitro_addresses_file} ]; then
-  echo "Using Nitro addresses from ${nitro_addresses_file}:"
-  cat "$nitro_addresses_file"
-  cat "$nitro_addresses_file" > "$nitro_addresses_destination_file"
-else
-  echo "Nitro addresses not available"
-  exit 1
-fi
-
-# Build after setting the Nitro addresses
-yarn build
-
-PEER_ID_FILE='./peer-id.json'
-RELAY_PEER_ID_FILE='./relay-id.json'
-
-# Create watcher party array
-watcher_parties=()
-
-# Iterate over each fixture JSON file
-for watcher_keys_file in "$watcher_keys_dir"/*.json; do
-  # Extract the filename without the path and extension
-  filename=$(basename "$watcher_keys_file" .json)
-
-  # Read the consensus keys
-  peer_id=$(jq -r '.peerId.id' "$watcher_keys_file")
-
-  # Read the consensus keys
-  consensus_public_key=$(jq -r '.consensus.publicKey' "$watcher_keys_file")
-  consensus_private_key=$(jq -r '.consensus.privateKey' "$watcher_keys_file")
-
-  # Append watcher party
-  watcher_party=$(jq -n \
-    --arg peerId "$peer_id" \
-    --arg publicKey "$consensus_public_key" \
-    '.peerId = $peerId | .publicKey = $publicKey')
-  watcher_parties+=("$watcher_party")
-
-  if [ "$filename" = "watcher-${CERC_WATCHER_INDEX}" ]; then
-    # Export watcher peer and relay ids
-    peer_id_data=$(jq '.peerId' "$watcher_keys_file")
-    relay_peer_id_data=$(jq '.relayPeerId' "$watcher_keys_file")
-    echo "$peer_id_data" > "${PEER_ID_FILE}"
-    echo "$relay_peer_id_data" > "${RELAY_PEER_ID_FILE}"
-
-    # Set peer and nitro account keys
-    CERC_PRIVATE_KEY_PEER=$(jq -r '.peer.privateKey' "$watcher_keys_file")
-    CERC_WATCHER_NITRO_PK=$(jq -r '.peer.nitroPrivateKey' "$watcher_keys_file")
-
-    # Set consensus keys for this peer
-    CONSENSUS_PUBLIC_KEY=${consensus_public_key}
-    CONSENSUS_PRIVATE_KEY=${consensus_private_key}
-  fi
-done
-
-echo "Consensus module enabled"
-CONSENSUS_ENABLED=true
-WATCHER_PARTY_PEERS_FILE='./watcher-party-peers.json'
-
-# Export watcher party file
-watcher_parties_json=$(printf '%s\n' "${watcher_parties[@]}" | jq -s .)
-echo "$watcher_parties_json" > "${WATCHER_PARTY_PEERS_FILE}"
-echo "Watcher party peers exported to ${WATCHER_PARTY_PEERS_FILE}"
-
-# Configure relay peers for watcher 2 and 3
-CERC_RELAY_PEERS=[]
-if [ "${CERC_WATCHER_INDEX}" = "2" ]; then
-  # 2 -> 1
-  CERC_RELAY_PEERS="['/dns4/mobymask-watcher-1-server/tcp/9090/ws/p2p/$(jq -r '.relayPeerId.id' ${watcher_keys_dir}/watcher-1.json)']"
-elif [ "${CERC_WATCHER_INDEX}" = "3" ]; then
-  # 3 -> 1, 2
-  CERC_RELAY_PEERS="['/dns4/mobymask-watcher-1-server/tcp/9090/ws/p2p/$(jq -r '.relayPeerId.id' ${watcher_keys_dir}/watcher-1.json)', '/dns4/mobymask-watcher-2-server/tcp/9090/ws/p2p/$(jq -r '.relayPeerId.id' ${watcher_keys_dir}/watcher-2.json)']"
-fi
-
-# Read in the config template TOML file and modify it
-WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
-WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_RELAY_PEERS|${CERC_RELAY_PEERS}|g; \
-    s|REPLACE_WITH_CERC_RELAY_MULTIADDR|${CERC_RELAY_MULTIADDR}|g; \
-    s/REPLACE_WITH_CERC_PRIVATE_KEY_PEER/${CERC_PRIVATE_KEY_PEER}/g; \
-    s/REPLACE_WITH_CERC_WATCHER_NITRO_PK/${CERC_WATCHER_NITRO_PK}/g; \
-    s/REPLACE_WITH_CONTRACT_ADDRESS/${CONTRACT_ADDRESS}/g; \
-    s|REPLACE_WITH_CERC_NITRO_CHAIN_URL|${CERC_NITRO_CHAIN_URL}|g; \
-    s/REPLACE_WITH_CONSENSUS_ENABLED/${CONSENSUS_ENABLED}/g; \
-    s/REPLACE_WITH_CONSENSUS_PUBLIC_KEY/${CONSENSUS_PUBLIC_KEY}/g; \
-    s/REPLACE_WITH_CONSENSUS_PRIVATE_KEY/${CONSENSUS_PRIVATE_KEY}/g; \
-    s|REPLACE_WITH_WATCHER_PARTY_PEERS_FILE|${WATCHER_PARTY_PEERS_FILE}|g; \
-    s|REPLACE_WITH_CERC_ETH_RPC_QUERY_ENDPOINT|${CERC_ETH_RPC_QUERY_ENDPOINT}|g; \
-    s|REPLACE_WITH_CERC_ETH_RPC_MUTATION_ENDPOINT|${CERC_ETH_RPC_MUTATION_ENDPOINT}|g; \
-    s|REPLACE_WITH_WATCHER_DB|${WATCHER_DB}| ")
-
-# Write the modified content to a new file
-echo "$WATCHER_CONFIG" > environments/local.toml
-
-echo 'Running watcher server'
-yarn server

stack_orchestrator/data/config/watcher-mobymask-v3-demo/watcher-config-rates.toml

@@ -1,14 +0,0 @@
-freeQueriesLimit = 10
-
-freeQueriesList = []
-
-[queries]
-  multiNonce = '50'
-  _owner = '50'
-  isRevoked = '50'
-  isPhisher = '50'
-  isMember = '50'
-
-[mutations]
-  invoke = '100'
-  revoke = '100'

stack_orchestrator/data/config/watcher-mobymask-v3-demo/watcher-config-template.toml

@@ -1,111 +0,0 @@
-[server]
-  host = "0.0.0.0"
-  port = 3001
-  kind = "lazy"
-
-  # Checkpointing state.
-  checkpointing = false
-
-  # Checkpoint interval in number of blocks.
-  checkpointInterval = 2000
-
-  # Enable state creation
-  enableState = false
-
-  # Boolean to filter logs by contract.
-  filterLogs = true
-
-  # Max block range for which to return events in eventsInRange GQL query.
-  # Use -1 for skipping check on block range.
-  maxEventsBlockRange = -1
-
-  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
-  rpcSupportsBlockHashParam = true
-
-  [server.p2p]
-    enableRelay = true
-    enablePeer = true
-
-    [server.p2p.relay]
-      host = "0.0.0.0"
-      port = 9090
-      relayPeers = REPLACE_WITH_CERC_RELAY_PEERS
-      denyMultiaddrs = []
-      peerIdFile = './relay-id.json'
-      announce = ''
-      pubsub = ''
-      enableDebugInfo = true
-
-    [server.p2p.peer]
-      relayMultiaddr = 'REPLACE_WITH_CERC_RELAY_MULTIADDR'
-      pubSubTopic = 'mobymask'
-      denyMultiaddrs = []
-      peerIdFile = './peer-id.json'
-      pubsub = ''
-      enableDebugInfo = true
-      enableL2Txs = true
-      pingInterval = 4000
-      pingTimeout = 1500
-      maxRelayConnections = 10
-
-      [server.p2p.peer.l2TxsConfig]
-        privateKey = 'REPLACE_WITH_CERC_PRIVATE_KEY_PEER'
-        contractAddress = 'REPLACE_WITH_CONTRACT_ADDRESS'
-
-    [server.p2p.nitro]
-      chainUrl = 'REPLACE_WITH_CERC_NITRO_CHAIN_URL'
-      store = './out/nitro-db'
-      privateKey = 'REPLACE_WITH_CERC_WATCHER_NITRO_PK'
-      chainPrivateKey = 'REPLACE_WITH_CERC_PRIVATE_KEY_PEER'
-
-      [server.p2p.nitro.payments]
-        ratesFile = './environments/rates.toml'
-        requestTimeoutInSecs = 10
-
-        [server.p2p.nitro.payments.cache]
-          maxAccounts = 1000
-          accountTTLInSecs = 1800
-          maxVouchersPerAccount = 1000
-          voucherTTLInSecs = 300
-          maxPaymentChannels = 10000
-          paymentChannelTTLInSecs = 1800
-
-    [server.p2p.consensus]
-      enabled = REPLACE_WITH_CONSENSUS_ENABLED
-      publicKey = 'REPLACE_WITH_CONSENSUS_PUBLIC_KEY'
-      privateKey = 'REPLACE_WITH_CONSENSUS_PRIVATE_KEY'
-      watcherPartyPeersFile = 'REPLACE_WITH_WATCHER_PARTY_PEERS_FILE'
-
-[metrics]
-  host = "0.0.0.0"
-  port = 9000
-  [metrics.gql]
-    port = 9001
-
-[database]
-  type = "postgres"
-  host = "mobymask-watcher-db"
-  port = 5432
-  database = "REPLACE_WITH_WATCHER_DB"
-  username = "vdbm"
-  password = "password"
-  synchronize = true
-  logging = false
-
-[upstream]
-  [upstream.ethServer]
-    gqlApiEndpoint = "http://ipld-eth-server:8083/graphql"
-    rpcProviderEndpoint = 'REPLACE_WITH_CERC_ETH_RPC_QUERY_ENDPOINT'
-    rpcProviderMutationEndpoint = 'REPLACE_WITH_CERC_ETH_RPC_MUTATION_ENDPOINT'
-
-  [upstream.cache]
-    name = "requests"
-    enabled = false
-    deleteOnStart = false
-
-[jobQueue]
-  dbConnectionString = "postgres://vdbm:password@mobymask-watcher-db/REPLACE_WITH_WATCHER_DB-job-queue"
-  maxCompletionLagInSecs = 300
-  jobDelayInMilliSecs = 100
-  eventsInBatch = 50
-  blockDelayInMilliSecs = 60000

stack_orchestrator/data/config/watcher-sushiswap-v3/start-job-runner.sh

@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u
 
-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
 
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
 
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml

stack_orchestrator/data/config/watcher-sushiswap-v3/start-server.sh

@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u
 
-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
 
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
 
 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml

stack_orchestrator/data/config/watcher-sushiswap-v3/watcher-config-template.toml

@@ -2,7 +2,6 @@
   host = "0.0.0.0"
   port = 3008
   kind = "active"
-  gqlPath = "/"
 
   # Checkpointing state.
   checkpointing = true
@@ -22,23 +21,30 @@
   # Interval in number of blocks at which to clear entities cache.
   clearEntitiesCacheInterval = 1000
 
-  # Max block range for which to return events in eventsInRange GQL query.
-  # Use -1 for skipping check on block range.
-  maxEventsBlockRange = 1000
-
   # Flag to specify whether RPC endpoint supports block hash as block tag parameter
   rpcSupportsBlockHashParam = false
 
-  # GQL cache settings
-  [server.gqlCache]
-    enabled = true
+  # Server GQL config
+  [server.gql]
+    path = "/"
 
-    # Max in-memory cache size (in bytes) (default 8 MB)
-    # maxCacheSize
+    # Max block range for which to return events in eventsInRange GQL query.
+    # Use -1 for skipping check on block range.
+    maxEventsBlockRange = 1000
 
-    # GQL cache-control max-age settings (in seconds)
-    maxAge = 15
-    timeTravelMaxAge = 86400 # 1 day
+    # Log directory for GQL requests
+    logDir = "./gql-logs"
+
+    # GQL cache settings
+    [server.gql.cache]
+      enabled = true
+
+      # Max in-memory cache size (in bytes) (default 8 MB)
+      # maxCacheSize
+
+      # GQL cache-control max-age settings (in seconds)
+      maxAge = 15
+      timeTravelMaxAge = 86400 # 1 day
 
 [metrics]
   host = "0.0.0.0"
@@ -58,7 +64,7 @@
 
 [upstream]
   [upstream.ethServer]
-    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS
 
     # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
     rpcClient = true
@@ -85,6 +91,9 @@
   # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
   blockDelayInMilliSecs = 30000
 
+  # Number of blocks by which block processing lags behind head
+  blockProcessingOffset = 0
+
   # Boolean to switch between modes of processing events when starting the server.
   # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
   # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
@@ -96,3 +105,6 @@
   # Max block range of historical processing after which it waits for completion of events processing
   # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
   historicalMaxFetchAhead = 10000
+
+  # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
+  maxNewBlockRetries = 3

stack_orchestrator/data/container-build/cerc-builder-js/Dockerfile

@@ -68,5 +68,5 @@ ENV PATH="${PATH}:/scripts"
 COPY entrypoint.sh .
 ENTRYPOINT ["./entrypoint.sh"]
 # Placeholder CMD : generally this will be overridden at run time like :
-# docker run -it -v /home/builder/cerc/laconic-sdk:/workspace cerc/builder-js sh -c 'cd /workspace && yarn && yarn build'
+# docker run -it -v /home/builder/cerc/registry-sdk:/workspace cerc/builder-js sh -c 'cd /workspace && yarn && yarn build'
 CMD node --version

stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/Dockerfile

@@ -1,26 +1,23 @@
-FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen
+FROM ethpandaops/ethereum-genesis-generator:3.0.0 AS ethgen
 
 FROM golang:1.20-alpine as builder
 
-RUN apk add --no-cache python3 py3-pip
-
-COPY genesis /opt/genesis
+RUN apk add --no-cache python3 py3-pip make bash envsubst jq
 
 # Install ethereum-genesis-generator tools
-COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/
-COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/
 COPY --from=ethgen /apps /apps
 RUN cd /apps/el-gen && pip3 install --break-system-packages -r requirements.txt
-# web3==5.24.0 used by el-gen is broken on python 3.11
-RUN pip3 install --break-system-packages --upgrade "web3==6.5.0"
+RUN pip3 install --break-system-packages --upgrade "web3==v6.15.1"
 RUN pip3 install --break-system-packages --upgrade "typing-extensions"
 
+# Install tool to generate initial block
+RUN go install github.com/cerc-io/eth-dump-genblock@b29516740fc01cf1d1d623acbfd0e9a2b6440a96
+
 # Build genesis config
-RUN apk add --no-cache make bash envsubst jq
+COPY genesis /opt/genesis
 RUN cd /opt/genesis && make genesis-el
 
 # Snag the genesis block info.
-RUN go install github.com/cerc-io/eth-dump-genblock@latest
 RUN eth-dump-genblock /opt/genesis/build/el/geth.json > /opt/genesis/build/el/genesis_block.json
 
 FROM alpine:latest

stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/genesis/el/build_el.sh

@@ -9,32 +9,8 @@ mkdir -p ../build/el
 tmp_dir=$(mktemp -d -t ci-XXXXXXXXXX)
 envsubst < el-config.yaml > $tmp_dir/genesis-config.yaml
 
-ttd=`cat $tmp_dir/genesis-config.yaml | grep terminal_total_difficulty | awk '{ print $2 }'`
-homestead_block=`cat $tmp_dir/genesis-config.yaml | grep homestead_block | awk '{ print $2 }'`
-eip150_block=`cat $tmp_dir/genesis-config.yaml | grep eip150_block | awk '{ print $2 }'`
-eip155_block=`cat $tmp_dir/genesis-config.yaml | grep eip155_block | awk '{ print $2 }'`
-eip158_block=`cat $tmp_dir/genesis-config.yaml | grep eip158_block | awk '{ print $2 }'`
-byzantium_block=`cat $tmp_dir/genesis-config.yaml | grep byzantium_block | awk '{ print $2 }'`
-constantinople_block=`cat $tmp_dir/genesis-config.yaml | grep constantinople_block | awk '{ print $2 }'`
-petersburg_block=`cat $tmp_dir/genesis-config.yaml | grep petersburg_block | awk '{ print $2 }'`
-istanbul_block=`cat $tmp_dir/genesis-config.yaml | grep istanbul_block | awk '{ print $2 }'`
-berlin_block=`cat $tmp_dir/genesis-config.yaml | grep berlin_block | awk '{ print $2 }'`
-london_block=`cat $tmp_dir/genesis-config.yaml | grep london_block | awk '{ print $2 }'`
-merge_fork_block=`cat $tmp_dir/genesis-config.yaml | grep merge_fork_block | awk '{ print $2 }'`
-
 python3 /apps/el-gen/genesis_geth.py $tmp_dir/genesis-config.yaml | \
-  jq ".config.terminalTotalDifficulty=$ttd" | \
-  jq ".config.homesteadBlock=$homestead_block" | \
-  jq ".config.eip150Block=$eip150_block" | \
-  jq ".config.eip155Block=$eip155_block" | \
-  jq ".config.eip158Block=$eip158_block" | \
-  jq ".config.byzantiumBlock=$byzantium_block" | \
-  jq ".config.constantinopleBlock=$constantinople_block" | \
-  jq ".config.petersburgBlock=$petersburg_block" | \
-  jq ".config.istanbulBlock=$istanbul_block" | \
-  jq ".config.berlinBlock=$berlin_block" | \
-  jq ".config.londonBlock=$london_block" | \
-  jq ".config.mergeForkBlock=$merge_fork_block" | \
-  jq ".config.mergeNetsplitBlock=$merge_fork_block" \
+  jq 'del(.config.pragueTime)' \
   > ../build/el/geth.json
+
 python3 ../accounts/mnemonic_to_csv.py $tmp_dir/genesis-config.yaml > ../build/el/accounts.csv

stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/genesis/el/el-config.yaml

@@ -10,22 +10,8 @@ el_premine_addrs: {}
 chain_id: 1212
 deposit_contract_address: "0x1212121212121212121212121212121212121212"
 genesis_timestamp: 0
-terminal_total_difficulty: 1000
-homestead_block: 1
-eip150_block: 1
-eip155_block: 1
-eip158_block: 1
-byzantium_block: 1
-constantinople_block: 1
-petersburg_block: 1
-istanbul_block: 1
-berlin_block: 1
-london_block: 1
-merge_fork_block: 1
-
-clique:
-  enabled: false
-  signers:
-    - 36d56343bc308d4ffaac2f793d121aba905fa6cc
-    - 5e762d4a3847cadaf40a4b0c39574b0ff6698c78
-    - 15d7acc1019fdf8ab4f0f7bd31ec1487ecb5a2bd
+genesis_delay: 0
+deneb_fork_epoch: 0
+# note: only needed as workaround https://github.com/ethpandaops/ethereum-genesis-generator/pull/105
+electra_fork_epoch: 0
+slot_duration_in_seconds: 3

stack_orchestrator/data/container-build/cerc-fixturenet-eth-geth/run-el.sh

@@ -6,7 +6,7 @@ fi
 
 ETHERBASE=`cat /opt/testnet/build/el/accounts.csv | head -1 | cut -d',' -f2`
 NETWORK_ID=`cat /opt/testnet/el/el-config.yaml | grep 'chain_id' | awk '{ print $2 }'`
-NETRESTRICT=`ip addr | grep inet | grep -v '127.0' | awk '{print $2}'`
+NETRESTRICT=`ip addr | grep -w inet | grep -v '127.0' | awk '{print $2}'`
 CERC_ETH_DATADIR="${CERC_ETH_DATADIR:-$HOME/ethdata}"
 CERC_PLUGINS_DIR="${CERC_PLUGINS_DIR:-/usr/local/lib/plugeth}"
 
@@ -102,6 +102,13 @@ else
       fi
     fi
 
+    OTHER_OPTS=""
+    # miner options were removed in v1.12
+    GETH_VERSION=$(geth --version | grep -io '[0-9][0-9a-z.-]*')
+    if echo -e "$GETH_VERSION\n1.12" | sort -Vc; then
+      OTHER_OPTS="--miner.threads=1"
+    fi
+
     $START_CMD \
       --datadir="${CERC_ETH_DATADIR}" \
       --bootnodes="${ENODE}" \
@@ -126,12 +133,12 @@ else
       --cache.preimages \
       --syncmode=full \
       --mine \
-      --miner.threads=1 \
       --metrics \
       --metrics.addr="0.0.0.0" \
       --verbosity=${CERC_GETH_VERBOSITY:-3} \
       --log.vmodule="${CERC_GETH_VMODULE:-statediff/*=5}" \
       --miner.etherbase="${ETHERBASE}" \
+      ${OTHER_OPTS} \
       ${STATEDIFF_OPTS} \
       &
 

stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/Dockerfile

@@ -1,5 +1,4 @@
 FROM cerc/lighthouse-cli:local AS lcli
-FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen
 FROM cerc/fixturenet-eth-genesis:local AS fnetgen
 
 FROM cerc/lighthouse:local
@@ -12,16 +11,13 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-reco
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 
-COPY genesis /opt/testnet
-COPY run-cl.sh /opt/testnet/run.sh
-
 COPY --from=lcli /usr/local/bin/lcli /usr/local/bin/lcli
-COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/eth2-testnet-genesis
-COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/eth2-val-tools
-COPY --from=ethgen /apps /apps
 COPY --from=fnetgen /opt/genesis/el /opt/testnet/el
 COPY --from=fnetgen /opt/genesis/build/el /opt/testnet/build/el
 
+COPY genesis /opt/testnet
+COPY run-cl.sh /opt/testnet/run.sh
+
 RUN cd /opt/testnet && make genesis-cl
 
 # Work around some bugs in lcli where the default path is always used.