Add git to webapp base container.

Merge pull request 'Fix mainnet laconic deploy setup' (#850 ) from dboreham/fix-mainnet-laconic-init into main
Reviewed-on: cerc-io/stack-orchestrator#850
2024-06-24 23:57:15 -05:00 · 2024-06-22 01:25:07 +00:00 · 2024-06-21 19:24:33 -06:00 · 2024-06-20 09:27:23 +00:00 · 2024-06-19 10:40:54 +00:00 · 2024-06-19 06:41:58 +00:00
117 changed files with 38367 additions and 890 deletions
@@ -9,10 +9,6 @@ on:
  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
    - cron: '2 14 * * *'

-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-

 jobs:
  test:
@@ -43,11 +39,19 @@ jobs:
        run: ./scripts/build_shiv_package.sh
      - name: "Run fixturenet-eth tests"
        run: ./tests/fixturenet-eth-plugeth/run-test.sh
-      - name: Notify Slack on CI failure
-          if: always()
-          uses: ravsamhq/notify-slack-action@v2
-          with:
-            status: ${{ job.status }}
-            notify_when: 'failure'
-          env:
-            SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -9,10 +9,6 @@ on:
  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
    - cron: '2 14 * * *'

-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-

 jobs:
  test:
@@ -41,17 +37,21 @@ jobs:
        run: ./scripts/create_build_tag_file.sh
      - name: "Build local shiv package"
        run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
      - name: "Run fixturenet-eth tests"
        run: ./tests/fixturenet-eth-plugeth/run-test.sh
-      - name: Notify Slack on CI failure
-          if: always()
-          uses: ravsamhq/notify-slack-action@v2
-          with:
-            status: ${{ job.status }}
-            notify_when: 'failure'
-          env:
-            SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -7,10 +7,6 @@ on:
      - '!**'
      - '.gitea/workflows/triggers/fixturenet-eth-test'

-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-

 jobs:
  test:
@@ -39,17 +35,21 @@ jobs:
        run: ./scripts/create_build_tag_file.sh
      - name: "Build local shiv package"
        run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
      - name: "Run fixturenet-eth tests"
        run: ./tests/fixturenet-eth/run-test.sh
-      - name: Notify Slack on CI failure
-          if: always()
-          uses: ravsamhq/notify-slack-action@v2
-          with:
-            status: ${{ job.status }}
-            notify_when: 'failure'
-          env:
-            SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -48,11 +48,19 @@ jobs:
        run: ./tests/fixturenet-laconicd/run-test.sh
      - name: "Run laconic CLI tests"
        run: ./tests/fixturenet-laconicd/run-cli-test.sh
-      - name: Notify Slack on CI failure
-          if: always()
-          uses: ravsamhq/notify-slack-action@v2
-          with:
-            status: ${{ job.status }}
-            notify_when: 'failure'
-          env:
-            SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -19,11 +19,19 @@ jobs:
          python-version: '3.8'
      - name : "Run flake8"
        uses: py-actions/flake8@v2
-      - name: Notify Slack on CI failure
-          if: always()
-          uses: ravsamhq/notify-slack-action@v2
-          with:
-            status: ${{ job.status }}
-            notify_when: 'failure'
-          env:
-            SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -54,11 +54,19 @@ jobs:
          # Hack using endsWith to workaround Gitea sometimes sending "publish-test" vs "refs/heads/publish-test"
          draft: ${{ endsWith('publish-test', github.ref ) }}
          files: ./laconic-so
-      - name: Notify Slack on CI failure
-          if: always()
-          uses: ravsamhq/notify-slack-action@v2
-          with:
-            status: ${{ job.status }}
-            notify_when: 'failure'
-          env:
-            SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -51,11 +51,19 @@ jobs:
          source /opt/bash-utils/cgroup-helper.sh
          join_cgroup
          ./tests/container-registry/run-test.sh
-      - name: Notify Slack on CI failure
-          if: always()
-          uses: ravsamhq/notify-slack-action@v2
-          with:
-            status: ${{ job.status }}
-            notify_when: 'failure'
-          env:
-            SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -49,11 +49,19 @@ jobs:
          source /opt/bash-utils/cgroup-helper.sh
          join_cgroup
          ./tests/database/run-test.sh
-      - name: Notify Slack on CI failure
-          if: always()
-          uses: ravsamhq/notify-slack-action@v2
-          with:
-            status: ${{ job.status }}
-            notify_when: 'failure'
-          env:
-            SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -10,9 +10,6 @@ on:
    paths-ignore:
      - '.gitea/workflows/triggers/*'

-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock

 jobs:
  test:
@@ -41,17 +38,21 @@ jobs:
        run: ./scripts/create_build_tag_file.sh
      - name: "Build local shiv package"
        run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
      - name: "Run deploy tests"
        run: ./tests/deploy/run-deploy-test.sh
-      - name: Notify Slack on CI failure
-          if: always()
-          uses: ravsamhq/notify-slack-action@v2
-          with:
-            status: ${{ job.status }}
-            notify_when: 'failure'
-          env:
-            SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -0,0 +1,58 @@
+name: External Stack Test
+
+on:
+  push:
+    branches: '*'
+    paths:
+      - '!**'
+      - '.gitea/workflows/triggers/test-external-stack'
+      - '.gitea/workflows/test-external-stack.yml'
+      - 'tests/external-stack/run-test.sh'
+  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
+    - cron: '8 19 * * *'
+
+jobs:
+  test:
+    name: "Run external stack test suite"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      # At present the stock setup-python action fails on Linux/aarch64
+      # Conditional steps below workaroud this by using deadsnakes for that case only
+      - name: "Install Python for ARM on Linux"
+        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
+        uses: deadsnakes/action@v3.0.1
+        with:
+          python-version: '3.8'
+      - name: "Install Python cases other than ARM on Linux"
+        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: "Print Python version"
+        run: python3 --version
+      - name: "Install shiv"
+        run: pip install shiv
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: "Run external stack tests"
+        run: ./tests/external-stack/run-test.sh
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -51,12 +51,19 @@ jobs:
          source /opt/bash-utils/cgroup-helper.sh
          join_cgroup
          ./tests/k8s-deploy/run-deploy-test.sh
-      - name: Notify Slack on CI failure
-          if: always()
-          uses: ravsamhq/notify-slack-action@v2
-          with:
-            status: ${{ job.status }}
-            notify_when: 'failure'
-          env:
-            SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
-
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -10,10 +10,6 @@ on:
    paths-ignore:
      - '.gitea/workflows/triggers/*'

-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
 jobs:
  test:
    name: "Run webapp test suite"
@@ -43,17 +39,21 @@ jobs:
        run: ./scripts/build_shiv_package.sh
      - name: "Install wget" # 20240109 - Only needed until the executors are updated.
        run: apt update && apt install -y wget
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
      - name: "Run webapp tests"
        run: ./tests/webapp-test/run-webapp-test.sh
-      - name: Notify Slack on CI failure
-          if: always()
-          uses: ravsamhq/notify-slack-action@v2
-          with:
-            status: ${{ job.status }}
-            notify_when: 'failure'
-          env:
-            SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -10,9 +10,6 @@ on:
    paths-ignore:
      - '.gitea/workflows/triggers/*'

-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock

 jobs:
  test:
@@ -41,17 +38,21 @@ jobs:
        run: ./scripts/create_build_tag_file.sh
      - name: "Build local shiv package"
        run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
      - name: "Run smoke tests"
        run: ./tests/smoke-test/run-smoke-test.sh
-      - name: Notify Slack on CI failure
-          if: always()
-          uses: ravsamhq/notify-slack-action@v2
-          with:
-            status: ${{ job.status }}
-            notify_when: 'failure'
-          env:
-            SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -3,3 +3,5 @@ Trigger
 Trigger
 Trigger
 Trigger
+Trigger
+Trigger
@@ -0,0 +1,2 @@
+Change this file to trigger running the external-stack CI job
+trigger
@@ -26,7 +26,8 @@ curl -SL https://github.com/docker/compose/releases/download/v2.11.2/docker-comp
 chmod +x ~/.docker/cli-plugins/docker-compose
 ```

-Next decide on a directory where you would like to put the stack-orchestrator program. Typically this would be a "user" binary directory such as `~/bin` or perhaps `/usr/local/laconic` or possibly just the current working directory.
+Next decide on a directory where you would like to put the stack-orchestrator program. Typically this would be 
+a "user" binary directory such as `~/bin` or perhaps `/usr/local/laconic` or possibly just the current working directory.

 Now, having selected that directory, download the latest release from [this page](https://git.vdb.to/cerc-io/stack-orchestrator/tags) into it (we're using `~/bin` below for concreteness but edit to suit if you selected a different directory). Also be sure that the destination directory exists and is writable:

@@ -55,7 +56,6 @@ echo "distribution-url: https://git.vdb.to/cerc-io/stack-orchestrator/releases/d
 ```

 ### Update
-
 If Stack Orchestrator was installed using the process described above, it is able to subsequently self-update to the current latest version by running:

 ```bash
@@ -78,3 +78,5 @@ See the [CONTRIBUTING.md](/docs/CONTRIBUTING.md) for developer mode install.
 ## Platform Support

 Native aarm64 is _not_ currently supported. x64 emulation on ARM64 macos should work (not yet tested).
+
+
@@ -71,7 +71,7 @@ def process_container(build_context: BuildContext) -> bool:

    # Check if this is in an external stack
    if stack_is_external(build_context.stack):
-        container_parent_dir = Path(build_context.stack).joinpath("container-build")
+        container_parent_dir = Path(build_context.stack).parent.parent.joinpath("container-build")
        temp_build_dir = container_parent_dir.joinpath(build_context.container.replace("/", "-"))
        temp_build_script_filename = temp_build_dir.joinpath("build.sh")
        # Now check if the container exists in the external stack.
@@ -0,0 +1,80 @@
+
+# From: https://raw.githubusercontent.com/blast-io/deployment/master/docker-compose.yml
+services:
+  # generate jwt.txt if it's absent
+  generate-jwt:
+    image: blastio/openssl
+    volumes:
+      - blast-data:/blast:rw
+    command: >
+      sh -c "[ ! -f /blast/jwt.txt ] && openssl rand -hex 32 | tr -d '\n' > /blast/jwt.txt || exit 0"
+  # initialise geth db
+  geth-init:
+    image: blastio/blast-geth:${NETWORK:-testnet-sepolia}
+    volumes:
+      - blast-data:/blast:rw
+      - ../config/fixturenet-blast/genesis.json:/blast/genesis.json
+    entrypoint: /bin/sh
+    command: >
+      -c "[ ! -d /blast/${GETH_DATA_DIR:-blast-geth-data}/geth ] && /usr/local/bin/geth init --datadir=/blast/${GETH_DATA_DIR:-blast-geth-data} /blast/genesis.json || exit 0"
+    depends_on:
+      generate-jwt:
+        condition: service_completed_successfully
+    env_file: 
+      - ../config/fixturenet-blast/${NETWORK:-fixturenet}.config
+  blast-geth:
+    image: blastio/blast-geth:${NETWORK:-testnet-sepolia}
+    volumes:
+      - blast-data:/blast
+    ports:
+      - "9545"
+      - "9546"
+    command: >
+      --datadir=/blast/${GETH_DATA_DIR:-blast-geth-data}
+      --http
+      --http.corsdomain="*"
+      --http.vhosts="*"
+      --http.addr=0.0.0.0
+      --http.port=9545
+      --http.api=web3,debug,eth,txpool,net,engine
+      --ws
+      --ws.addr=0.0.0.0
+      --ws.port=9546
+      --ws.origins="*"
+      --ws.api=debug,eth,txpool,net,engine
+      --authrpc.addr="0.0.0.0"
+      --authrpc.port="8551"
+      --authrpc.vhosts="*"
+      --authrpc.jwtsecret=/blast/jwt.txt
+      --syncmode=full
+      --gcmode=archive
+      --nodiscover
+      --maxpeers=0
+      --rollup.disabletxpoolgossip=true
+    env_file: 
+      - ../config/fixturenet-blast/${NETWORK:-fixturenet}.config
+    depends_on:
+      geth-init:
+        condition: service_completed_successfully
+  op-node:
+    image: blastio/blast-optimism:${NETWORK:-testnet-sepolia}
+    volumes:
+      - blast-data:/blast
+      - ../config/fixturenet-blast/rollup.json:/blast/rollup.json
+    ports:
+      - "9003"
+    command: >
+      op-node
+      --l1="${CERC_L1_RPC}"
+      --l1.rpckind="any"
+      --l1.trustrpc=true
+      --l2="http://blast-geth:8551"
+      --l2.jwt-secret=/blast/jwt.txt
+      --rollup.config="/blast/rollup.json"
+    depends_on:
+      - blast-geth
+    env_file: 
+      - ../config/fixturenet-blast/${NETWORK:-fixturenet}.config
+
+volumes:
+  blast-data:
@@ -6,6 +6,7 @@ services:
      - ../config/fixturenet-eth/fixturenet-eth.env
    environment:
      RUN_BOOTNODE: "true"
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
    image: cerc/fixturenet-plugeth-plugeth:local
    volumes:
      - fixturenet_plugeth_bootnode_geth_data:/root/ethdata
@@ -2,16 +2,24 @@ version: "3.7"

 services:
  grafana:
-    image: grafana/grafana:10.2.2
+    image: grafana/grafana:10.2.3
    restart: always
    environment:
      GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL}
+      CERC_GRAFANA_ALERTS_SUBGRAPH_IDS: ${CERC_GRAFANA_ALERTS_SUBGRAPH_IDS}
    volumes:
      - ../config/monitoring/grafana/provisioning:/etc/grafana/provisioning
      - ../config/monitoring/grafana/dashboards:/etc/grafana/dashboards
+      - ../config/monitoring/update-grafana-alerts-config.sh:/update-grafana-alerts-config.sh
      - grafana_storage:/var/lib/grafana
+    user: root
+    entrypoint: ["bash", "-c"]
+    command: |
+      "/update-grafana-alerts-config.sh && /run.sh"
    ports:
      - "3000"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
    healthcheck:
      test: ["CMD", "nc", "-vz", "localhost", "3000"]
      interval: 30s
@@ -16,8 +16,13 @@ services:
      postgres_pass: password
      postgres_db: graph-node
      ethereum: ${ETH_NETWORKS:-lotus-fixturenet:http://lotus-node-1:1234/rpc/v1}
+      # Env varaibles reference: https://git.vdb.to/cerc-io/graph-node/src/branch/master/docs/environment-variables.md
      GRAPH_LOG: debug
      ETHEREUM_REORG_THRESHOLD: 3
+      GRAPH_ETHEREUM_JSON_RPC_TIMEOUT: ${GRAPH_ETHEREUM_JSON_RPC_TIMEOUT:-180}
+      GRAPH_ETHEREUM_REQUEST_RETRIES: ${GRAPH_ETHEREUM_REQUEST_RETRIES:-10}
+      GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE: ${GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE:-2000}
+      GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS: ${GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS:-1000}
    entrypoint: ["bash", "-c"]
    # Wait for ETH RPC endpoint to be up when running with fixturenet-lotus
    command: |
@@ -27,6 +32,7 @@ services:
      - "8001"
      - "8020"
      - "8030"
+      - "8040"
    healthcheck:
      test: ["CMD", "nc", "-vz", "localhost", "8020"]
      interval: 30s
@@ -0,0 +1,83 @@
+
+# From: https://raw.githubusercontent.com/blast-io/deployment/master/docker-compose.yml
+services:
+  # generate jwt.txt if it's absent
+  generate-jwt:
+    image: blastio/openssl
+    volumes:
+      - blast-data:/blast:rw
+    command: >
+      sh -c "[ ! -f /blast/jwt.txt ] && openssl rand -hex 32 | tr -d '\n' > /blast/jwt.txt || exit 0"
+  # initialise geth db
+  geth-init:
+    image: blastio/blast-geth:${NETWORK:-mainnet}
+    volumes:
+      - blast-data:/blast:rw
+    entrypoint: /bin/sh
+    command: >
+      -c "[ ! -d /blast/${GETH_DATA_DIR:-blast-geth-data}/geth ] && /usr/local/bin/geth init --datadir=/blast/${GETH_DATA_DIR:-blast-geth-data} /blast/genesis.json || exit 0"
+    depends_on:
+      generate-jwt:
+        condition: service_completed_successfully
+    env_file: 
+      - ../config/mainnet-blast/${NETWORK:-mainnet}.config
+  blast-geth:
+    image: blastio/blast-geth:${NETWORK:-mainnet}
+    volumes:
+      - blast-data:/blast
+    ports:
+      - "9545"
+      - "9546"
+      - "6060"
+    command: >
+      --datadir=/blast/${GETH_DATA_DIR:-blast-geth-data}
+      --http
+      --http.corsdomain="*"
+      --http.vhosts="*"
+      --http.addr=0.0.0.0
+      --http.port=9545
+      --http.api=web3,debug,eth,txpool,net,engine
+      --ws
+      --ws.addr=0.0.0.0
+      --ws.port=9546
+      --ws.origins="*"
+      --ws.api=debug,eth,txpool,net,engine
+      --authrpc.addr="0.0.0.0"
+      --authrpc.port="8551"
+      --authrpc.vhosts="*"
+      --authrpc.jwtsecret=/blast/jwt.txt
+      --syncmode=full
+      --metrics
+      --metrics.addr=0.0.0.0
+      --gcmode=archive
+      --nodiscover
+      --maxpeers=0
+      --rollup.disabletxpoolgossip=true
+    env_file: 
+      - ../config/mainnet-blast/${NETWORK:-mainnet}.config
+    depends_on:
+      geth-init:
+        condition: service_completed_successfully
+  op-node:
+    image: blastio/blast-optimism:${NETWORK:-mainnet}
+    volumes:
+      - blast-data:/blast
+    ports:
+      - "9003"
+      - "7300"
+    command: >
+      op-node
+      --l1="https://eth-mainnet-1.vdb.to/"
+      --metrics.enabled
+      --l1.rpckind="any"
+      --l1.trustrpc=true
+      --l2="http://blast-geth:8551"
+      --l2.jwt-secret=/blast/jwt.txt
+      --rollup.config="/blast/rollup.json"
+    depends_on:
+      - blast-geth
+    env_file: 
+      - ../config/mainnet-blast/${NETWORK:-mainnet}.config
+
+volumes:
+  blast-data:
@@ -28,15 +28,37 @@ services:
    extra_hosts:
      - "host.docker.internal:host-gateway"

-  chain-head-exporter:
+  ethereum-chain-head-exporter:
    image: cerc/watcher-ts:local
    restart: always
    working_dir: /app/packages/cli
    environment:
-      ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
-      FIL_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT}
+      ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT:-https://mainnet.infura.io/v3}
      ETH_RPC_API_KEY: ${CERC_INFURA_KEY}
-      PORT: ${CERC_METRICS_PORT}
+    command: ["sh", "-c", "yarn export-metrics:chain-heads"]
+    ports:
+      - '5000'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  filecoin-chain-head-exporter:
+    image: cerc/watcher-ts:local
+    restart: always
+    working_dir: /app/packages/cli
+    environment:
+      ETH_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT:-https://api.node.glif.io/rpc/v1}
+    command: ["sh", "-c", "yarn export-metrics:chain-heads"]
+    ports:
+      - '5000'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  graph-node-upstream-head-exporter:
+    image: cerc/watcher-ts:local
+    restart: always
+    working_dir: /app/packages/cli
+    environment:
+      ETH_RPC_ENDPOINT: ${GRAPH_NODE_RPC_ENDPOINT}
    command: ["sh", "-c", "yarn export-metrics:chain-heads"]
    ports:
      - '5000'
@@ -1,13 +0,0 @@
-services:
-  snowballtools-base-backend:
-    image: cerc/snowballtools-base-backend:local
-    restart: always
-    volumes:
-      - data:/data
-      - config:/config:ro
-    ports:
-      - 8000
-
-volumes:
-  data:
-  config:
@@ -29,7 +29,7 @@ services:
    image: cerc/watcher-ajna:local
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
    command: ["bash", "./start-job-runner.sh"]
    volumes:
      - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -37,7 +37,7 @@ services:
    ports:
      - "9000"
    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -55,16 +55,17 @@ services:
    image: cerc/watcher-ajna:local
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
    command: ["bash", "./start-server.sh"]
    volumes:
      - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
      - ../config/watcher-ajna/start-server.sh:/app/start-server.sh
+      - ajna_watcher_gql_logs_data:/app/gql-logs
    ports:
      - "3008"
      - "9001"
    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -74,3 +75,4 @@ services:

 volumes:
  ajna_watcher_db_data:
+  ajna_watcher_gql_logs_data:
@@ -32,8 +32,8 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
      CERC_HISTORICAL_BLOCK_RANGE: 500
      CONTRACT_ADDRESS: 0x223c067F8CF28ae173EE5CafEa60cA44C335fecB
      CONTRACT_NAME: Azimuth
@@ -47,7 +47,7 @@ services:
    ports:
      - "9000"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -66,18 +66,20 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
    working_dir: /app/packages/azimuth-watcher
    command: "./start-server.sh"
    volumes:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/azimuth-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/azimuth-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/azimuth-watcher/start-server.sh
+      - azimuth_watcher_gql_logs_data:/app/packages/azimuth-watcher/gql-logs
    ports:
      - "3001"
+      - "9001"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3001"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3001"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -94,8 +96,8 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
      CONTRACT_ADDRESS: 0x325f68d32BdEe6Ed86E7235ff2480e2A433D6189
      CONTRACT_NAME: Censures
      STARTING_BLOCK: 6784954
@@ -108,7 +110,7 @@ services:
    ports:
      - "9002"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9002"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9002"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -127,18 +129,20 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
    working_dir: /app/packages/censures-watcher
    command: "./start-server.sh"
    volumes:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/censures-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/censures-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/censures-watcher/start-server.sh
+      - censures_watcher_gql_logs_data:/app/packages/censures-watcher/gql-logs
    ports:
      - "3002"
+      - "9003"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3002"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3002"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -155,8 +159,8 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
      CONTRACT_ADDRESS: 0xe7e7f69b34D7d9Bd8d61Fb22C33b22708947971A
      CONTRACT_NAME: Claims
      STARTING_BLOCK: 6784941
@@ -169,7 +173,7 @@ services:
    ports:
      - "9004"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9004"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9004"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -188,18 +192,20 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
    working_dir: /app/packages/claims-watcher
    command: "./start-server.sh"
    volumes:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/claims-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/claims-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/claims-watcher/start-server.sh
+      - claims_watcher_gql_logs_data:/app/packages/claims-watcher/gql-logs
    ports:
      - "3003"
+      - "9005"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3003"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3003"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -216,8 +222,8 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
      CONTRACT_ADDRESS: 0x8C241098C3D3498Fe1261421633FD57986D74AeA
      CONTRACT_NAME: ConditionalStarRelease
      STARTING_BLOCK: 6828004
@@ -230,7 +236,7 @@ services:
    ports:
      - "9006"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9006"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9006"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -249,18 +255,20 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
    working_dir: /app/packages/conditional-star-release-watcher
    command: "./start-server.sh"
    volumes:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/conditional-star-release-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/conditional-star-release-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/conditional-star-release-watcher/start-server.sh
+      - conditional_star_release_watcher_gql_logs_data:/app/packages/conditional-star-release-watcher/gql-logs
    ports:
      - "3004"
+      - "9007"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3004"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3004"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -277,8 +285,8 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
      CONTRACT_ADDRESS: 0xf6b461fE1aD4bd2ce25B23Fe0aff2ac19B3dFA76
      CONTRACT_NAME: DelegatedSending
      STARTING_BLOCK: 6784956
@@ -291,7 +299,7 @@ services:
    ports:
      - "9008"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9008"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -310,18 +318,20 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
    working_dir: /app/packages/delegated-sending-watcher
    command: "./start-server.sh"
    volumes:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/delegated-sending-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/delegated-sending-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/delegated-sending-watcher/start-server.sh
+      - delegated_sending_watcher_gql_logs_data:/app/packages/delegated-sending-watcher/gql-logs
    ports:
      - "3005"
+      - "9009"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3005"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3005"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -338,8 +348,8 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
      CONTRACT_ADDRESS: 0x33EeCbf908478C10614626A9D304bfe18B78DD73
      CONTRACT_NAME: Ecliptic
      STARTING_BLOCK: 13692129
@@ -352,7 +362,7 @@ services:
    ports:
      - "9010"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9010"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9010"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -371,18 +381,20 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
    working_dir: /app/packages/ecliptic-watcher
    command: "./start-server.sh"
    volumes:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/ecliptic-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/ecliptic-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/ecliptic-watcher/start-server.sh
+      - ecliptic_watcher_gql_logs_data:/app/packages/ecliptic-watcher/gql-logs
    ports:
      - "3006"
+      - "9011"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3006"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3006"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -399,8 +411,8 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
      CONTRACT_ADDRESS: 0x86cd9cd0992F04231751E3761De45cEceA5d1801
      CONTRACT_NAME: LinearStarRelease
      STARTING_BLOCK: 6784943
@@ -413,7 +425,7 @@ services:
    ports:
      - "9012"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9012"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9012"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -432,18 +444,20 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
    working_dir: /app/packages/linear-star-release-watcher
    command: "./start-server.sh"
    volumes:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/linear-star-release-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/linear-star-release-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/linear-star-release-watcher/start-server.sh
+      - linear_star_release_watcher_gql_logs_data:/app/packages/linear-star-release-watcher/gql-logs
    ports:
      - "3007"
+      - "9013"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3007"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3007"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -460,8 +474,8 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
      CONTRACT_ADDRESS: 0x7fEcaB617c868Bb5996d99D95200D2Fa708218e4
      CONTRACT_NAME: Polls
      STARTING_BLOCK: 6784912
@@ -474,7 +488,7 @@ services:
    ports:
      - "9014"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9014"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9014"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -493,18 +507,20 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
    working_dir: /app/packages/polls-watcher
    command: "./start-server.sh"
    volumes:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/polls-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/polls-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/polls-watcher/start-server.sh
+      - polls_watcher_gql_logs_data:/app/packages/polls-watcher/gql-logs
    ports:
      - "3008"
+      - "9015"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -542,7 +558,7 @@ services:
    ports:
      - "0.0.0.0:4000:4000"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "4000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "4000"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -552,3 +568,11 @@ services:

 volumes:
  watcher_db_data:
+  azimuth_watcher_gql_logs_data:
+  censures_watcher_gql_logs_data:
+  claims_watcher_gql_logs_data:
+  conditional_star_release_watcher_gql_logs_data:
+  delegated_sending_watcher_gql_logs_data:
+  ecliptic_watcher_gql_logs_data:
+  linear_star_release_watcher_gql_logs_data:
+  polls_watcher_gql_logs_data:
@@ -29,7 +29,7 @@ services:
    image: cerc/watcher-merkl-sushiswap-v3:local
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
    command: ["bash", "./start-job-runner.sh"]
    volumes:
      - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -37,7 +37,7 @@ services:
    ports:
      - "9002:9000"
    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -55,16 +55,17 @@ services:
    image: cerc/watcher-merkl-sushiswap-v3:local
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
    command: ["bash", "./start-server.sh"]
    volumes:
      - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
      - ../config/watcher-merkl-sushiswap-v3/start-server.sh:/app/start-server.sh
+      - merkl_sushiswap_v3_watcher_gql_logs_data:/app/gql-logs
    ports:
      - "127.0.0.1:3007:3008"
      - "9003:9001"
    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -74,3 +75,4 @@ services:

 volumes:
  merkl_sushiswap_v3_watcher_db_data:
+  merkl_sushiswap_v3_watcher_gql_logs_data:
@@ -29,7 +29,7 @@ services:
    image: cerc/watcher-sushiswap-v3:local
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
    command: ["bash", "./start-job-runner.sh"]
    volumes:
      - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -37,7 +37,7 @@ services:
    ports:
      - "9000:9000"
    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -55,16 +55,17 @@ services:
    image: cerc/watcher-sushiswap-v3:local
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
    command: ["bash", "./start-server.sh"]
    volumes:
      - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
      - ../config/watcher-sushiswap-v3/start-server.sh:/app/start-server.sh
+      - sushiswap_v3_watcher_gql_logs_data:/app/gql-logs
    ports:
      - "127.0.0.1:3008:3008"
      - "9001:9001"
    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -74,3 +75,4 @@ services:

 volumes:
  sushiswap_v3_watcher_db_data:
+  sushiswap_v3_watcher_gql_logs_data:
@@ -0,0 +1,2 @@
+GETH_ROLLUP_SEQUENCERHTTP=https://sequencer.s2.testblast.io
+OP_NODE_P2P_BOOTNODES=enr:-J-4QM3GLUFfKMSJQuP1UvuKQe8DyovE7Eaiit0l6By4zjTodkR4V8NWXJxNmlg8t8rP-Q-wp3jVmeAOml8cjMj__ROGAYznzb_HgmlkgnY0gmlwhA-cZ_eHb3BzdGFja4X947FQAIlzZWNwMjU2azGhAiuDqvB-AsVSRmnnWr6OHfjgY8YfNclFy9p02flKzXnOg3RjcIJ2YYN1ZHCCdmE,enr:-J-4QDCVpByqQ8nFqCS9aHicqwUfXgzFDslvpEyYz19lvkHLIdtcIGp2d4q5dxHdjRNTO6HXCsnIKxUeuZSPcEbyVQCGAYznzz0RgmlkgnY0gmlwhANiQfuHb3BzdGFja4X947FQAIlzZWNwMjU2azGhAy3AtF2Jh_aPdOohg506Hjmtx-fQ1AKmu71C7PfkWAw9g3RjcIJ2YYN1ZHCCdmE
@@ -0,0 +1,57 @@
+{
+  "config": {
+    "chainId": 608943043,
+    "homesteadBlock": 0,
+    "eip150Block": 0,
+    "eip155Block": 0,
+    "eip158Block": 0,
+    "byzantiumBlock": 0,
+    "constantinopleBlock": 0,
+    "petersburgBlock": 0,
+    "istanbulBlock": 0,
+    "muirGlacierBlock": 0,
+    "berlinBlock": 0,
+    "londonBlock": 0,
+    "arrowGlacierBlock": 0,
+    "grayGlacierBlock": 0,
+    "mergeNetsplitBlock": 0,
+    "shanghaiTime": 0,
+    "bedrockBlock": 0,
+    "regolithTime": 0,
+    "canyonTime": 0,
+    "terminalTotalDifficulty": 0,
+    "terminalTotalDifficultyPassed": true,
+    "optimism": {
+      "eip1559Elasticity": 6,
+      "eip1559Denominator": 50,
+      "eip1559DenominatorCanyon": 250
+    }
+  },
+  "alloc": {
+    "0000000000000000000000000000000000000000": {
+      "balance": "0x1"
+    },
+    "4200000000000000000000000000000000000000": {
+      "code": "0x60806040526004361061004e5760003560e01c80633659cfe6146100655780634f1ef286146100855780635c60da1b146100ae5780638f283970146100db578063f851a440146100fb5761005d565b3661005d5761005b610110565b005b61005b610110565b34801561007157600080fd5b5061005b610080366004610521565b6101c8565b61009861009336600461053c565b61020e565b6040516100a591906105bf565b60405180910390f35b3480156100ba57600080fd5b506100c361033e565b6040516001600160a01b0390911681526020016100a5565b3480156100e757600080fd5b5061005b6100f6366004610521565b6103a9565b34801561010757600080fd5b506100c36103e4565b600061013a7f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc5490565b90506001600160a01b0381166101a55760405162461bcd60e51b815260206004820152602560248201527f50726f78793a20696d706c656d656e746174696f6e206e6f7420696e697469616044820152641b1a5e995960da1b60648201526084015b60405180910390fd5b3660008037600080366000845af43d6000803e806101c2573d6000fd5b503d6000f35b600080516020610625833981519152546001600160a01b0316336001600160a01b031614806101f5575033155b156102065761020381610432565b50565b610203610110565b60606102266000805160206106258339815191525490565b6001600160a01b0316336001600160a01b03161480610243575033155b1561032f5761025184610432565b600080856001600160a01b0316858560405161026e929190610614565b600060405180830381855af49150503d80600081146102a9576040519150601f19603f3d011682016040523d82523d6000602084013e6102ae565b606091505b5091509150816103265760405162461bcd60e51b815260206004820152603960248201527f50726f78793a2064656c656761746563616c6c20746f206e657720696d706c6560448201527f6d656e746174696f6e20636f6e7472616374206661696c656400000000000000606482015260840161019c565b91506103379050565b610337610110565b9392505050565b60006103566000805160206106258339815191525490565b6001600160a01b0316336001600160a01b03161480610373575033155b1561039e57507f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc5490565b6103a6610110565b90565b600080516020610625833981519152546001600160a01b0316336001600160a01b031614806103d6575033155b15610206576102038161048e565b60006103fc6000805160206106258339815191525490565b6001600160a01b0316336001600160a01b03161480610419575033155b1561039e57506000805160206106258339815191525490565b7f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc8181556040516001600160a01b038316907fbc7cd75a20ee27fd9adebab32041f755214dbc6bffa90cc0225b39da2e5c2d3b90600090a25050565b60006104a66000805160206106258339815191525490565b600080516020610625833981519152838155604080516001600160a01b0380851682528616602082015292935090917f7e644d79422f17c01e4894b5f4f588d331ebfa28653d42ae832dc59e38c9798f910160405180910390a1505050565b80356001600160a01b038116811461051c57600080fd5b919050565b60006020828403121561053357600080fd5b61033782610505565b60008060006040848603121561055157600080fd5b61055a84610505565b9250602084013567ffffffffffffffff8082111561057757600080fd5b818601915086601f83011261058b57600080fd5b81358181111561059a57600080fd5b8760208285010111156105ac57600080fd5b6020830194508093505050509250925092565b600060208083528351808285015260005b818110156105ec578581018301518582016040015282016105d0565b818111156105fe576000604083870101525b50601f01601f1916929092016040019392505050565b818382376000910190815291905056feb53127684a568b3173ae13b9f8a6016e243e63b6e8ee1178d6a717850b5d6103a164736f6c634300080f000a",
+      "storage": {
+        "0x360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc": "0x000000000000000000000000c0d3c0d3c0d3c0d3c0d3c0d3c0d3c0d3c0d30000",
+        "0xb53127684a568b3173ae13b9f8a6016e243e63b6e8ee1178d6a717850b5d6103": "0x0000000000000000000000004200000000000000000000000000000000000018"
+      },
+      "balance": "0x0",
+      "flags": 1
+    }
+  },
+  "nonce": "0x0",
+  "timestamp": "0x659b7460",
+  "extraData": "0x424544524f434b",
+  "gasLimit": "0x1c9c380",
+  "difficulty": "0x0",
+  "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+  "coinbase": "0x4200000000000000000000000000000000000011",
+  "number": "0x0",
+  "gasUsed": "0x0",
+  "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+  "baseFeePerGas": "0x3b9aca00",
+  "excessBlobGas": null,
+  "blobGasUsed": null
+}
@@ -0,0 +1,31 @@
+{
+  "genesis": {
+    "l1": {
+      "hash": "0x17728cf4d8e0b4f292d2390a869fd7c632d39e72efb00ca3462b4387c6aa2437",
+      "number": 5044255
+    },
+    "l2": {
+      "hash": "0x26a1c0faad7b041f34569a1bb383f00ab74b335883a44bed53e9f41ced5fd906",
+      "number": 0
+    },
+    "l2_time": 1704686688,
+    "system_config": {
+      "batcherAddr": "0xba26fee2fa917443e05e65de8d4350bcd2f59222",
+      "overhead": "0x00000000000000000000000000000000000000000000000000000000000000bc",
+      "scalar": "0x00000000000000000000000000000000000000000000000000000000000a6fe0",
+      "gasLimit": 30000000
+    }
+  },
+  "block_time": 2,
+  "max_sequencer_drift": 600,
+  "seq_window_size": 3600,
+  "channel_timeout": 300,
+  "l1_chain_id": 11155111,
+  "l2_chain_id": 608943043,
+  "regolith_time": 0,
+  "canyon_time": 0,
+  "batch_inbox_address": "0x1c3b85a2108784eab6a4bf56cdd6f722e415b331",
+  "deposit_contract_address": "0x2757e4430e694f27b73ec9c02257cab3a498c8c5",
+  "l1_system_config_address": "0x329faf078c364a316e08bf6a17b7eee6ae75a613",
+  "protocol_versions_address": "0x0000000000000000000000000000000000000000"
+}
@@ -22,4 +22,7 @@ CERC_STATEDIFF_DB_LOG_STATEMENTS="${CERC_STATEDIFF_DB_LOG_STATEMENTS:-false}"
 CERC_STATEDIFF_WORKERS=2

 CERC_GETH_VMODULE="statediff/*=5,rpc/*=5"
-CERC_GETH_VERBOSITY=${CERC_GETH_VERBOSITY:-3}
+CERC_GETH_VERBOSITY=${CERC_GETH_VERBOSITY:-3}
+
+# Used by Lighthouse
+SECONDS_PER_ETH1_BLOCK=${SECONDS_PER_ETH1_BLOCK:-3}
@@ -102,6 +102,17 @@ if [ "$1" == "clean" ] || [ ! -d "$HOME/.laconicd/data/blockstore.db" ]; then
    fi
  fi

+  # Enable telemetry (prometheus metrics: http://localhost:1317/metrics?format=prometheus)
+  if [[ "$OSTYPE" == "darwin"* ]]; then
+    sed -i '' 's/enabled = false/enabled = true/g' $HOME/.laconicd/config/app.toml
+    sed -i '' 's/prometheus-retention-time = 0/prometheus-retention-time = 60/g' $HOME/.laconicd/config/app.toml
+    sed -i '' 's/prometheus = false/prometheus = true/g' $HOME/.laconicd/config/config.toml
+  else
+    sed -i 's/enabled = false/enabled = true/g' $HOME/.laconicd/config/app.toml
+    sed -i 's/prometheus-retention-time = 0/prometheus-retention-time = 60/g' $HOME/.laconicd/config/app.toml
+    sed -i 's/prometheus = false/prometheus = true/g' $HOME/.laconicd/config/config.toml
+  fi
+
  # Allocate genesis accounts (cosmos formatted addresses)
  laconicd add-genesis-account $KEY 100000000000000000000000000aphoton --keyring-backend $KEYRING

@@ -0,0 +1,32 @@
+POSTGRES_DB=keycloak
+POSTGRES_USER=keycloak
+POSTGRES_PASSWORD=keycloak
+# Don't change this unless you also change the healthcheck in docker-compose-mainnet-eth-keycloak.yml
+PGPORT=35432
+KC_DB=postgres
+KC_DB_URL_HOST=keycloak-db
+KC_DB_URL_PORT=${PGPORT}
+KC_DB_URL_DATABASE=${POSTGRES_DB}
+KC_DB_USERNAME=${POSTGRES_USER}
+KC_DB_PASSWORD=${POSTGRES_PASSWORD}
+KC_DB_SCHEMA=public
+KC_HOSTNAME=localhost
+KC_HTTP_ENABLED="true"
+KC_HTTP_RELATIVE_PATH="/auth"
+KC_HOSTNAME_STRICT_HTTPS="false"
+KEYCLOAK_ADMIN=admin
+KEYCLOAK_ADMIN_PASSWORD=admin
+X_API_CHECK_REALM=cerc
+X_API_CHECK_CLIENT_ID="%user_id%"
+
+
+# keycloak-reg-api
+CERC_KCUSERREG_LISTEN_PORT=9292
+CERC_KCUSERREG_LISTEN_ADDR='0.0.0.0'
+CERC_KCUSERREG_API_URL='http://keycloak:8080/auth'
+CERC_KCUSERREG_REG_USER="${KEYCLOAK_ADMIN}"
+CERC_KCUSERREG_REG_PW="${KEYCLOAK_ADMIN_PASSWORD}"
+CERC_KCUSERREG_REG_CLIENT_ID='admin-cli'
+CERC_KCUSERREG_TARGET_REALM=cerc
+CERC_KCUSERREG_TARGET_GROUPS=eth
+CERC_KCUSERREG_CREATE_ENABLED=true
@@ -0,0 +1,33 @@
+# Enable startup script debug output.
+CERC_SCRIPT_DEBUG=false
+
+# Specify any other lighthouse CLI options.
+LIGHTHOUSE_OPTS=""
+
+# Override the advertised public IP (optional)
+# --enr-address
+#LIGHTHOUSE_ENR_ADDRESS=""
+
+# --checkpoint-sync-url
+LIGHTHOUSE_CHECKPOINT_SYNC_URL="https://beaconstate.ethstaker.cc"
+
+# --checkpoint-sync-url-timeout
+LIGHTHOUSE_CHECKPOINT_SYNC_URL_TIMEOUT=300
+
+# --datadir
+LIGHTHOUSE_DATADIR=/data
+
+# --debug-level
+LIGHTHOUSE_DEBUG_LEVEL=info
+
+# --http-port
+LIGHTHOUSE_HTTP_PORT=5052
+
+# --execution-jwt
+LIGHTHOUSE_JWTSECRET=/etc/mainnet-eth/jwtsecret
+
+# --metrics-port
+LIGHTHOUSE_METRICS_PORT=5054
+
+# --port --enr-udp-port --enr-tcp-port
+LIGHTHOUSE_NETWORK_PORT=9000
@@ -0,0 +1,2 @@
+GETH_ROLLUP_SEQUENCERHTTP=https://sequencer.blast.io
+OP_NODE_P2P_BOOTNODES=enr:-J64QGwHl9uYLfC_cnmxSA6wQH811nkOWJDWjzxqkEUlJoZHWvI66u-BXgVcPCeMUmg0dBpFQAPotFchG67FHJMZ9OSGAY3d6wevgmlkgnY0gmlwhANizeSHb3BzdGFja4Sx_AQAiXNlY3AyNTZrMaECg4pk0cskPAyJ7pOmo9E6RqGBwV-Lex4VS9a3MQvu7PWDdGNwgnZhg3VkcIJ2YQ,enr:-J64QDge2jYBQtcNEpRqmKfci5E5BHAhNBjgv4WSdwH1_wPqbueq2bDj38-TSW8asjy5lJj1Xftui6Or8lnaYFCqCI-GAY3d6wf3gmlkgnY0gmlwhCO2D9yHb3BzdGFja4Sx_AQAiXNlY3AyNTZrMaEDo4aCTq7pCEN8om9U5n_VyWdambGnQhwHNwKc8o-OicaDdGNwgnZhg3VkcIJ2YQ
@@ -0,0 +1,32 @@
+{
+    "genesis": {
+      "l1": {
+        "hash": "0xfcfb8d586bdae763f1189988789211c69eb893a895e7ba48be3ca6289f0941b7",
+        "number": 19300102
+      },
+      "l2": {
+        "hash": "0xb689b35ef29d0bec5816938e0e52683c7257d2e325420ea69b739a2be4754b89",
+        "number": 0
+      },
+      "l2_time": 1708809815,
+      "system_config": {
+        "batcherAddr": "0x415c8893d514f9bc5211d36eeda4183226b84aa7",
+        "overhead": "0x00000000000000000000000000000000000000000000000000000000000000bc",
+        "scalar": "0x00000000000000000000000000000000000000000000000000000000000a6fe0",
+        "gasLimit": 30000000
+      }
+    },
+    "block_time": 2,
+    "max_sequencer_drift": 600,
+    "seq_window_size": 3600,
+    "channel_timeout": 300,
+    "l1_chain_id": 1,
+    "l2_chain_id": 81457,
+    "regolith_time": 0,
+    "canyon_time": 0,
+    "batch_inbox_address": "0xff00000000000000000000000000000000081457",
+    "deposit_contract_address": "0x0ec68c5b10f21effb74f2a5c61dfe6b08c0db6cb",
+    "l1_system_config_address": "0x5531dcff39ec1ec727c4c5d2fc49835368f805a9",
+    "protocol_versions_address": "0x0000000000000000000000000000000000000000"
+  }
+  
@@ -0,0 +1,30 @@
+#!/bin/bash
+if [[ "true" == "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi
+
+ENR_OPTS=""
+if [[ -n "$LIGHTHOUSE_ENR_ADDRESS" ]]; then
+  ENR_OPTS="--enr-address $LIGHTHOUSE_ENR_ADDRESS"
+fi
+
+exec lighthouse bn \
+  --checkpoint-sync-url "$LIGHTHOUSE_CHECKPOINT_SYNC_URL" \
+  --checkpoint-sync-url-timeout ${LIGHTHOUSE_CHECKPOINT_SYNC_URL_TIMEOUT} \
+  --datadir "$LIGHTHOUSE_DATADIR" \
+  --debug-level $LIGHTHOUSE_DEBUG_LEVEL \
+  --disable-deposit-contract-sync \
+  --disable-upnp \
+  --enr-tcp-port $LIGHTHOUSE_NETWORK_PORT \
+  --enr-udp-port $LIGHTHOUSE_NETWORK_PORT \
+  --execution-endpoint "$LIGHTHOUSE_EXECUTION_ENDPOINT" \
+  --execution-jwt /etc/mainnet-eth/jwtsecret \
+  --http \
+  --http-address 0.0.0.0 \
+  --http-port $LIGHTHOUSE_HTTP_PORT \
+  --metrics \
+  --metrics-address=0.0.0.0 \
+  --metrics-port $LIGHTHOUSE_METRICS_PORT \
+  --network mainnet \
+  --port $LIGHTHOUSE_NETWORK_PORT \
+  $ENR_OPTS $LIGHTHOUSE_OPTS
@@ -0,0 +1,20 @@
+apiVersion: 1
+
+datasources:
+  - name: Graph Node Postgres
+    type: postgres
+    jsonData:
+      database: graph-node
+      sslmode: 'disable'
+      maxOpenConns: 100
+      maxIdleConns: 100
+      maxIdleConnsAuto: true
+      connMaxLifetime: 14400
+      postgresVersion: 1411 # 903=9.3, 1000=10, 1411=14.11
+      timescaledb: false
+    user: graph-node
+    # # Add URL for graph-node database
+    # url: graph-node-db:5432
+    # # Set password for graph-node database
+    # secureJsonData:
+    #   password: 'password'
@@ -45,7 +45,18 @@ scrape_configs:
    metrics_path: /metrics
    scheme: http
    static_configs:
-      - targets: ['chain-head-exporter:5000']
+      - targets: ['ethereum-chain-head-exporter:5000']
+        labels:
+          instance: 'external'
+          chain: 'ethereum'
+      - targets: ['filecoin-chain-head-exporter:5000']
+        labels:
+          instance: 'external'
+          chain: 'filecoin'
+      - targets: ['graph-node-upstream-head-exporter:5000']
+        labels:
+          instance: 'graph-node'
+          chain: 'filecoin'

  - job_name: 'postgres'
    scrape_interval: 30s
@@ -65,3 +76,20 @@ scrape_configs:
        target_label: instance
      - target_label: __address__
        replacement: postgres-exporter:9187
+
+  - job_name: laconicd
+    metrics_path: /metrics
+    scrape_interval: 30s
+    static_configs:
+      # Add laconicd REST endpoint target with host and port (1317)
+      # - targets: ['example-host:1317']
+    params:
+      format: ['prometheus']
+
+  - job_name: graph-node
+    metrics_path: /metrics
+    scrape_interval: 30s
+    scheme: http
+    static_configs:
+      # Add graph-node targets to be monitored below
+      # - targets: ['graph-node:8040']
@@ -0,0 +1,64 @@
+apiVersion: 1
+groups:
+  - orgId: 1
+    name: subgraph
+    folder: SubgraphAlerts
+    interval: 30s
+    rules:
+      - uid: b2a9144b-6104-46fc-92b5-352f4e643c4c
+        title: subgraph_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                  type: prometheus
+                  uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: ethereum_chain_head_number - on(network) group_right deployment_head{deployment=~"REPLACE_WITH_SUBGRAPH_IDS"}
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                  - evaluator:
+                      params:
+                          - 15
+                          - 0
+                      type: gt
+                    operator:
+                      type: and
+                    query:
+                      params: []
+                    reducer:
+                      params: []
+                      type: avg
+                    type: query
+              datasource:
+                  name: Expression
+                  type: __expr__
+                  uid: __expr__
+              expression: diff
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: threshold
+        noDataState: OK
+        execErrState: Alerting
+        for: 5m
+        annotations:
+          summary: Subgraph deployment {{ index $labels "deployment" }} is falling behind head by {{ index $values "diff" }}
+        labels: {}
+        isPaused: false
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+echo Using CERC_GRAFANA_ALERTS_SUBGRAPH_IDS ${CERC_GRAFANA_ALERTS_SUBGRAPH_IDS}
+
+# Replace subgraph ids in subgraph alerting config
+# Note: Requires the grafana container to be run with user root
+if [ -n "$CERC_GRAFANA_ALERTS_SUBGRAPH_IDS" ]; then
+  sed -i "s/REPLACE_WITH_SUBGRAPH_IDS/$CERC_GRAFANA_ALERTS_SUBGRAPH_IDS/g" /etc/grafana/provisioning/alerting/subgraph-alert-rules.yml
+fi
@@ -24,7 +24,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -100,7 +100,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -176,7 +176,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -252,7 +252,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -328,7 +328,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -404,7 +404,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -480,7 +480,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="linear_star_release", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="linear_star_release", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -556,7 +556,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -634,7 +634,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -710,7 +710,163 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+
+      # Ajna
+      - uid: ajna_diff_external
+        title: ajna_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="ajna", instance="ajna", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      
+      # Secured Finance
+      - uid: secured_finance_diff_external
+        title: secured_finance_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="secured-finance", instance="secured-finance", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u

-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"

 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")

 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml
@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u

-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"

 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")

 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml
@@ -2,7 +2,6 @@
  host = "0.0.0.0"
  port = 3008
  kind = "active"
-  gqlPath = "/"

  # Checkpointing state.
  checkpointing = true
@@ -22,23 +21,30 @@
  # Interval in number of blocks at which to clear entities cache.
  clearEntitiesCacheInterval = 1000

-  # Max block range for which to return events in eventsInRange GQL query.
-  # Use -1 for skipping check on block range.
-  maxEventsBlockRange = 1000
-
  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
  rpcSupportsBlockHashParam = false

-  # GQL cache settings
-  [server.gqlCache]
-    enabled = true
+  # Server GQL config
+  [server.gql]
+    path = "/"

-    # Max in-memory cache size (in bytes) (default 8 MB)
-    # maxCacheSize
+    # Max block range for which to return events in eventsInRange GQL query.
+    # Use -1 for skipping check on block range.
+    maxEventsBlockRange = 1000

-    # GQL cache-control max-age settings (in seconds)
-    maxAge = 15
-    timeTravelMaxAge = 86400 # 1 day
+    # Log directory for GQL requests
+    logDir = "./gql-logs"
+
+    # GQL cache settings
+    [server.gql.cache]
+      enabled = true
+
+      # Max in-memory cache size (in bytes) (default 8 MB)
+      # maxCacheSize
+
+      # GQL cache-control max-age settings (in seconds)
+      maxAge = 15
+      timeTravelMaxAge = 86400 # 1 day

 [metrics]
  host = "0.0.0.0"
@@ -58,7 +64,7 @@

 [upstream]
  [upstream.ethServer]
-    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS

    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
    rpcClient = true
@@ -85,6 +91,9 @@
  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
  blockDelayInMilliSecs = 30000

+  # Number of blocks by which block processing lags behind head
+  blockProcessingOffset = 0
+
  # Boolean to switch between modes of processing events when starting the server.
  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
@@ -96,3 +105,6 @@
  # Max block range of historical processing after which it waits for completion of events processing
  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
  historicalMaxFetchAhead = 10000
+
+  # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
+  maxNewBlockRetries = 3
@@ -4,16 +4,19 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
  set -x
 fi

-echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}"
-echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
+echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL_ENDPOINT}"
 echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}"

+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 # Replace env variables in template TOML file
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \
-          s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}|g; \
+          s|REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT|${CERC_IPLD_ETH_GQL_ENDPOINT}|g; \
          s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ")

 # Write the modified content to a new file
@@ -4,16 +4,19 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
  set -x
 fi

-echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}"
-echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
+echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL_ENDPOINT}"
 echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}"

+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 # Replace env variables in template TOML file
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \
-          s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}|g; \
+          s|REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT|${CERC_IPLD_ETH_GQL_ENDPOINT}|g; \
          s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ")

 # Write the modified content to a new file
@@ -1,6 +1,7 @@
 [server]
  host = "0.0.0.0"
-  maxSimultaneousRequests = -1
+  [server.gql]
+    maxSimultaneousRequests = -1

 [metrics]
  host = "0.0.0.0"
@@ -13,8 +14,8 @@

 [upstream]
  [upstream.ethServer]
-    gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL"
-    rpcProviderEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_RPC"
+    gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT"
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS

 [jobQueue]
  historicalLogsBlockRange = REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE
@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u

-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"

 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")

 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml
@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u

-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"

 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")

 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml
@@ -2,7 +2,6 @@
  host = "0.0.0.0"
  port = 3008
  kind = "active"
-  gqlPath = '/'

  # Checkpointing state.
  checkpointing = true
@@ -22,23 +21,30 @@
  # Interval in number of blocks at which to clear entities cache.
  clearEntitiesCacheInterval = 1000

-  # Max block range for which to return events in eventsInRange GQL query.
-  # Use -1 for skipping check on block range.
-  maxEventsBlockRange = 1000
-
  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
  rpcSupportsBlockHashParam = false

-  # GQL cache settings
-  [server.gqlCache]
-    enabled = true
+  # Server GQL config
+  [server.gql]
+    path = "/"

-    # Max in-memory cache size (in bytes) (default 8 MB)
-    # maxCacheSize
+    # Max block range for which to return events in eventsInRange GQL query.
+    # Use -1 for skipping check on block range.
+    maxEventsBlockRange = 1000

-    # GQL cache-control max-age settings (in seconds)
-    maxAge = 15
-    timeTravelMaxAge = 86400 # 1 day
+    # Log directory for GQL requests
+    logDir = "./gql-logs"
+
+    # GQL cache settings
+    [server.gql.cache]
+      enabled = true
+
+      # Max in-memory cache size (in bytes) (default 8 MB)
+      # maxCacheSize
+
+      # GQL cache-control max-age settings (in seconds)
+      maxAge = 15
+      timeTravelMaxAge = 86400 # 1 day

 [metrics]
  host = "0.0.0.0"
@@ -58,7 +64,7 @@

 [upstream]
  [upstream.ethServer]
-    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS

    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
    rpcClient = true
@@ -69,7 +75,7 @@
    # Boolean flag to filter event logs by contracts
    filterLogsByAddresses = true
    # Boolean flag to filter event logs by topics
-    filterLogsByTopics = false
+    filterLogsByTopics = true

  [upstream.cache]
    name = "requests"
@@ -85,6 +91,9 @@
  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
  blockDelayInMilliSecs = 30000

+  # Number of blocks by which block processing lags behind head
+  blockProcessingOffset = 0
+
  # Boolean to switch between modes of processing events when starting the server.
  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
@@ -96,3 +105,6 @@
  # Max block range of historical processing after which it waits for completion of events processing
  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
  historicalMaxFetchAhead = 10000
+
+  # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
+  maxNewBlockRetries = 3
@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u

-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"

 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")

 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml
@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u

-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"

 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")

 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml
@@ -2,7 +2,6 @@
  host = "0.0.0.0"
  port = 3008
  kind = "active"
-  gqlPath = "/"

  # Checkpointing state.
  checkpointing = true
@@ -22,23 +21,30 @@
  # Interval in number of blocks at which to clear entities cache.
  clearEntitiesCacheInterval = 1000

-  # Max block range for which to return events in eventsInRange GQL query.
-  # Use -1 for skipping check on block range.
-  maxEventsBlockRange = 1000
-
  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
  rpcSupportsBlockHashParam = false

-  # GQL cache settings
-  [server.gqlCache]
-    enabled = true
+  # Server GQL config
+  [server.gql]
+    path = "/"

-    # Max in-memory cache size (in bytes) (default 8 MB)
-    # maxCacheSize
+    # Max block range for which to return events in eventsInRange GQL query.
+    # Use -1 for skipping check on block range.
+    maxEventsBlockRange = 1000

-    # GQL cache-control max-age settings (in seconds)
-    maxAge = 15
-    timeTravelMaxAge = 86400 # 1 day
+    # Log directory for GQL requests
+    logDir = "./gql-logs"
+
+    # GQL cache settings
+    [server.gql.cache]
+      enabled = true
+
+      # Max in-memory cache size (in bytes) (default 8 MB)
+      # maxCacheSize
+
+      # GQL cache-control max-age settings (in seconds)
+      maxAge = 15
+      timeTravelMaxAge = 86400 # 1 day

 [metrics]
  host = "0.0.0.0"
@@ -58,7 +64,7 @@

 [upstream]
  [upstream.ethServer]
-    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS

    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
    rpcClient = true
@@ -85,6 +91,9 @@
  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
  blockDelayInMilliSecs = 30000

+  # Number of blocks by which block processing lags behind head
+  blockProcessingOffset = 0
+
  # Boolean to switch between modes of processing events when starting the server.
  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
@@ -96,3 +105,6 @@
  # Max block range of historical processing after which it waits for completion of events processing
  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
  historicalMaxFetchAhead = 10000
+
+  # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
+  maxNewBlockRetries = 3
@@ -1,26 +1,23 @@
-FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen
+FROM ethpandaops/ethereum-genesis-generator:3.0.0 AS ethgen

 FROM golang:1.20-alpine as builder

-RUN apk add --no-cache python3 py3-pip
-
-COPY genesis /opt/genesis
+RUN apk add --no-cache python3 py3-pip make bash envsubst jq

 # Install ethereum-genesis-generator tools
-COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/
-COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/
 COPY --from=ethgen /apps /apps
 RUN cd /apps/el-gen && pip3 install --break-system-packages -r requirements.txt
-# web3==5.24.0 used by el-gen is broken on python 3.11
-RUN pip3 install --break-system-packages --upgrade "web3==6.5.0"
+RUN pip3 install --break-system-packages --upgrade "web3==v6.15.1"
 RUN pip3 install --break-system-packages --upgrade "typing-extensions"

+# Install tool to generate initial block
+RUN go install github.com/cerc-io/eth-dump-genblock@b29516740fc01cf1d1d623acbfd0e9a2b6440a96
+
 # Build genesis config
-RUN apk add --no-cache make bash envsubst jq
+COPY genesis /opt/genesis
 RUN cd /opt/genesis && make genesis-el

 # Snag the genesis block info.
-RUN go install github.com/cerc-io/eth-dump-genblock@latest
 RUN eth-dump-genblock /opt/genesis/build/el/geth.json > /opt/genesis/build/el/genesis_block.json

 FROM alpine:latest
@@ -9,32 +9,8 @@ mkdir -p ../build/el
 tmp_dir=$(mktemp -d -t ci-XXXXXXXXXX)
 envsubst < el-config.yaml > $tmp_dir/genesis-config.yaml

-ttd=`cat $tmp_dir/genesis-config.yaml | grep terminal_total_difficulty | awk '{ print $2 }'`
-homestead_block=`cat $tmp_dir/genesis-config.yaml | grep homestead_block | awk '{ print $2 }'`
-eip150_block=`cat $tmp_dir/genesis-config.yaml | grep eip150_block | awk '{ print $2 }'`
-eip155_block=`cat $tmp_dir/genesis-config.yaml | grep eip155_block | awk '{ print $2 }'`
-eip158_block=`cat $tmp_dir/genesis-config.yaml | grep eip158_block | awk '{ print $2 }'`
-byzantium_block=`cat $tmp_dir/genesis-config.yaml | grep byzantium_block | awk '{ print $2 }'`
-constantinople_block=`cat $tmp_dir/genesis-config.yaml | grep constantinople_block | awk '{ print $2 }'`
-petersburg_block=`cat $tmp_dir/genesis-config.yaml | grep petersburg_block | awk '{ print $2 }'`
-istanbul_block=`cat $tmp_dir/genesis-config.yaml | grep istanbul_block | awk '{ print $2 }'`
-berlin_block=`cat $tmp_dir/genesis-config.yaml | grep berlin_block | awk '{ print $2 }'`
-london_block=`cat $tmp_dir/genesis-config.yaml | grep london_block | awk '{ print $2 }'`
-merge_fork_block=`cat $tmp_dir/genesis-config.yaml | grep merge_fork_block | awk '{ print $2 }'`
-
 python3 /apps/el-gen/genesis_geth.py $tmp_dir/genesis-config.yaml | \
-  jq ".config.terminalTotalDifficulty=$ttd" | \
-  jq ".config.homesteadBlock=$homestead_block" | \
-  jq ".config.eip150Block=$eip150_block" | \
-  jq ".config.eip155Block=$eip155_block" | \
-  jq ".config.eip158Block=$eip158_block" | \
-  jq ".config.byzantiumBlock=$byzantium_block" | \
-  jq ".config.constantinopleBlock=$constantinople_block" | \
-  jq ".config.petersburgBlock=$petersburg_block" | \
-  jq ".config.istanbulBlock=$istanbul_block" | \
-  jq ".config.berlinBlock=$berlin_block" | \
-  jq ".config.londonBlock=$london_block" | \
-  jq ".config.mergeForkBlock=$merge_fork_block" | \
-  jq ".config.mergeNetsplitBlock=$merge_fork_block" \
+  jq 'del(.config.pragueTime)' \
  > ../build/el/geth.json
+
 python3 ../accounts/mnemonic_to_csv.py $tmp_dir/genesis-config.yaml > ../build/el/accounts.csv
@@ -10,22 +10,8 @@ el_premine_addrs: {}
 chain_id: 1212
 deposit_contract_address: "0x1212121212121212121212121212121212121212"
 genesis_timestamp: 0
-terminal_total_difficulty: 1000
-homestead_block: 1
-eip150_block: 1
-eip155_block: 1
-eip158_block: 1
-byzantium_block: 1
-constantinople_block: 1
-petersburg_block: 1
-istanbul_block: 1
-berlin_block: 1
-london_block: 1
-merge_fork_block: 1
-
-clique:
-  enabled: false
-  signers:
-    - 36d56343bc308d4ffaac2f793d121aba905fa6cc
-    - 5e762d4a3847cadaf40a4b0c39574b0ff6698c78
-    - 15d7acc1019fdf8ab4f0f7bd31ec1487ecb5a2bd
+genesis_delay: 0
+deneb_fork_epoch: 0
+# note: only needed as workaround https://github.com/ethpandaops/ethereum-genesis-generator/pull/105
+electra_fork_epoch: 0
+slot_duration_in_seconds: 3
@@ -6,7 +6,7 @@ fi

 ETHERBASE=`cat /opt/testnet/build/el/accounts.csv | head -1 | cut -d',' -f2`
 NETWORK_ID=`cat /opt/testnet/el/el-config.yaml | grep 'chain_id' | awk '{ print $2 }'`
-NETRESTRICT=`ip addr | grep inet | grep -v '127.0' | awk '{print $2}'`
+NETRESTRICT=`ip addr | grep -w inet | grep -v '127.0' | awk '{print $2}'`
 CERC_ETH_DATADIR="${CERC_ETH_DATADIR:-$HOME/ethdata}"
 CERC_PLUGINS_DIR="${CERC_PLUGINS_DIR:-/usr/local/lib/plugeth}"

@@ -102,6 +102,13 @@ else
      fi
    fi

+    OTHER_OPTS=""
+    # miner options were removed in v1.12
+    GETH_VERSION=$(geth --version | grep -io '[0-9][0-9a-z.-]*')
+    if echo -e "$GETH_VERSION\n1.12" | sort -Vc; then
+      OTHER_OPTS="--miner.threads=1"
+    fi
+
    $START_CMD \
      --datadir="${CERC_ETH_DATADIR}" \
      --bootnodes="${ENODE}" \
@@ -126,12 +133,12 @@ else
      --cache.preimages \
      --syncmode=full \
      --mine \
-      --miner.threads=1 \
      --metrics \
      --metrics.addr="0.0.0.0" \
      --verbosity=${CERC_GETH_VERBOSITY:-3} \
      --log.vmodule="${CERC_GETH_VMODULE:-statediff/*=5}" \
      --miner.etherbase="${ETHERBASE}" \
+      ${OTHER_OPTS} \
      ${STATEDIFF_OPTS} \
      &

@@ -1,5 +1,4 @@
 FROM cerc/lighthouse-cli:local AS lcli
-FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen
 FROM cerc/fixturenet-eth-genesis:local AS fnetgen

 FROM cerc/lighthouse:local
@@ -12,16 +11,13 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-reco
  && apt-get clean \
  && rm -rf /var/lib/apt/lists/*

-COPY genesis /opt/testnet
-COPY run-cl.sh /opt/testnet/run.sh
-
 COPY --from=lcli /usr/local/bin/lcli /usr/local/bin/lcli
-COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/eth2-testnet-genesis
-COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/eth2-val-tools
-COPY --from=ethgen /apps /apps
 COPY --from=fnetgen /opt/genesis/el /opt/testnet/el
 COPY --from=fnetgen /opt/genesis/build/el /opt/testnet/build/el

+COPY genesis /opt/testnet
+COPY run-cl.sh /opt/testnet/run.sh
+
 RUN cd /opt/testnet && make genesis-cl

 # Work around some bugs in lcli where the default path is always used.
@@ -10,7 +10,6 @@ set -Eeuo pipefail
 source ./vars.env

 SUBSCRIBE_ALL_SUBNETS=
-DEBUG_LEVEL=${DEBUG_LEVEL:-debug}

 # Get positional arguments
 data_dir=$DATADIR/node_${NODE_NUMBER}
@@ -9,8 +9,6 @@ set -Eeuo pipefail

 source ./vars.env

-DEBUG_LEVEL=${1:-info}
-
 echo "Starting bootnode"

 # Clean up existing ENR dir to avoid node connectivity issues on a restart
@@ -1,5 +1,6 @@
 #!/usr/bin/env bash

+# See https://github.com/sigp/lighthouse/scripts/local_testnet/setup.sh
 #
 # Deploys the deposit contract and makes deposits for $VALIDATOR_COUNT insecure deterministic validators.
 # Produces a testnet specification and a genesis state where the genesis time
@@ -24,23 +25,26 @@ echo "(Note: errors of the form 'WARN: Scrypt parameters are too weak...' below
 lcli \
  new-testnet \
  --spec $SPEC_PRESET \
-  --deposit-contract-address $ETH1_DEPOSIT_CONTRACT_ADDRESS \
  --testnet-dir $TESTNET_DIR \
+  --deposit-contract-address $ETH1_DEPOSIT_CONTRACT_ADDRESS \
  --min-genesis-active-validator-count $GENESIS_VALIDATOR_COUNT \
  --validator-count $VALIDATOR_COUNT \
  --min-genesis-time $GENESIS_TIME \
  --genesis-delay $GENESIS_DELAY \
  --genesis-fork-version $GENESIS_FORK_VERSION \
  --altair-fork-epoch $ALTAIR_FORK_EPOCH \
-  --bellatrix-fork-epoch $MERGE_FORK_EPOCH \
+  --bellatrix-fork-epoch $BELLATRIX_FORK_EPOCH \
+  --capella-fork-epoch $CAPELLA_FORK_EPOCH \
+  --deneb-fork-epoch $DENEB_FORK_EPOCH \
  --eth1-id $ETH1_CHAIN_ID \
  --eth1-block-hash $ETH1_BLOCK_HASH \
  --eth1-follow-distance 1 \
  --seconds-per-slot $SECONDS_PER_SLOT \
  --seconds-per-eth1-block $SECONDS_PER_ETH1_BLOCK \
+  --interop-genesis-state \
  --force

-echo Specification generated at $TESTNET_DIR.
+echo Specification and genesis.ssz generated at $TESTNET_DIR.
 echo "Generating $VALIDATOR_COUNT validators concurrently... (this may take a while)"

 lcli \
@@ -50,13 +54,3 @@ lcli \
  --node-count $BN_COUNT

 echo Validators generated with keystore passwords at $DATADIR.
-echo "Building genesis state... (this might take a while)"
-
-lcli \
-  interop-genesis \
-  --spec $SPEC_PRESET \
-  --genesis-time $GENESIS_TIME \
-  --testnet-dir $TESTNET_DIR \
-  $GENESIS_VALIDATOR_COUNT
-
-echo Created genesis state in $TESTNET_DIR
@@ -8,8 +8,6 @@ set -Eeuo pipefail

 source ./vars.env

-DEBUG_LEVEL=info
-
 BUILDER_PROPOSALS=

 # Get options
@@ -25,7 +25,9 @@ BOOTNODE_PORT=${BOOTNODE_PORT:-4242}

 # Hard fork configuration
 ALTAIR_FORK_EPOCH=${ALTAIR_FORK_EPOCH:-0}
-MERGE_FORK_EPOCH=${MERGE_FORK_EPOCH:-0}
+BELLATRIX_FORK_EPOCH=${BELLATRIX_FORK_EPOCH:-0}
+CAPELLA_FORK_EPOCH=${CAPELLA_FORK_EPOCH:-0}
+DENEB_FORK_EPOCH=${DENEB_FORK_EPOCH:-0}

 # Spec version (mainnet or minimal)
 SPEC_PRESET=${SPEC_PRESET:-mainnet}
@@ -51,3 +53,6 @@ ETH1_TTD=${ETH1_TTD:-`cat $ETH1_GENESIS_JSON | jq -r '.config.terminalTotalDiffi
 ETH1_DEPOSIT_CONTRACT_ADDRESS=${ETH1_DEPOSIT_CONTRACT_ADDRESS:-`cat $ETH1_CONFIG_YAML | grep 'deposit_contract_address' | awk '{ print $2 }' | sed 's/"//g'`}
 ETH1_DEPOSIT_CONTRACT_BLOCK=${ETH1_DEPOSIT_CONTRACT_BLOCK:-0x0}
 SUGGESTED_FEE_RECIPIENT=`cat ../build/el/accounts.csv | head -1 | cut -d',' -f2`
+
+# --debug-level
+DEBUG_LEVEL=${LIGHTHOUSE_DEBUG_LEVEL:-debug}
@@ -1,9 +1,10 @@
-ARG TAG_SUFFIX="-modern"
-FROM sigp/lighthouse:v4.3.0${TAG_SUFFIX}
+FROM sigp/lighthouse:v5.1.2

-RUN apt-get update; apt-get install bash netcat curl less jq wget -y;
+RUN apt-get update && apt-get -y upgrade \
+    && apt-get -y install bash netcat curl less jq wget \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*

-WORKDIR /root/
+WORKDIR /root
 ADD start-lighthouse.sh .

 ENTRYPOINT [ "./start-lighthouse.sh" ]
@@ -6,4 +6,4 @@ source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 # See: https://stackoverflow.com/a/246128/1701505
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )

-docker build -t cerc/lighthouse:local ${build_command_args} --build-arg TAG_SUFFIX="" ${SCRIPT_DIR}
+docker build -t cerc/lighthouse:local ${build_command_args} ${SCRIPT_DIR}
@@ -1,6 +0,0 @@
-FROM cerc/snowballtools-base-backend-base:local
-
-WORKDIR /app/packages/backend
-COPY run.sh .
-
-ENTRYPOINT ["./run.sh"]
@@ -1,26 +0,0 @@
-FROM ubuntu:22.04 as builder
-
-RUN apt update && \
-      apt install -y --no-install-recommends --no-install-suggests \
-      ca-certificates curl gnupg
-
-# Node
-ARG NODE_MAJOR=20
-RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
-      echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \
-      apt update && apt install -y nodejs
-
-# npm setup
-RUN npm config set @cerc-io:registry https://git.vdb.to/api/packages/cerc-io/npm/ && npm install -g yarn
-
-COPY . /app/
-WORKDIR /app/
-
-RUN find . -name 'node_modules' | xargs -n1 rm -rf
-RUN yarn && yarn build --ignore frontend
-
-FROM cerc/webapp-base:local
-
-COPY --from=builder /app /app
-
-WORKDIR /app/packages/backend
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-# Build cerc/webapp-deployer-backend
-
-source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
-
-# See: https://stackoverflow.com/a/246128/1701505
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-
-docker build -t cerc/snowballtools-base-backend-base:local ${build_command_args} -f ${SCRIPT_DIR}/Dockerfile-base ${CERC_REPO_BASE_DIR}/snowballtools-base
-docker build -t cerc/snowballtools-base-backend:local ${build_command_args} ${SCRIPT_DIR} 
@@ -1,19 +0,0 @@
-#!/bin/bash
-
-
-LACONIC_HOSTED_CONFIG_FILE=${LACONIC_HOSTED_CONFIG_FILE}
-if [ -z "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
-  if [ -f "/config/laconic-hosted-config.yml" ]; then
-    LACONIC_HOSTED_CONFIG_FILE="/config/laconic-hosted-config.yml"
-  elif [ -f "/config/config.yml" ]; then
-    LACONIC_HOSTED_CONFIG_FILE="/config/config.yml"
-  fi
-fi
-
-if [ -f "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
-  /scripts/apply-webapp-config.sh $LACONIC_HOSTED_CONFIG_FILE "`pwd`/dist"
-fi
-
-/scripts/apply-runtime-env.sh "`pwd`/dist"
-
-yarn start
@@ -2,4 +2,4 @@
 # Build cerc/test-container
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-docker build -t cerc/test-container:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR
+docker build -t cerc/test-container:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR
@@ -6,5 +6,10 @@ WORKDIR /app

 COPY . .

+# Get the latest Git commit hash and set in package.json
+RUN COMMIT_HASH=$(git rev-parse HEAD) && \
+  jq --arg commitHash "$COMMIT_HASH" '.commitHash = $commitHash' package.json > tmp.json && \
+  mv tmp.json package.json
+
 RUN echo "Installing dependencies and building ajna-watcher-ts" && \
  yarn && yarn build
@@ -1,11 +1,20 @@
 FROM node:18.16.0-alpine3.16

-RUN apk --update --no-cache add git python3 alpine-sdk
+RUN apk --update --no-cache add git python3 alpine-sdk jq

 WORKDIR /app

 COPY . .

+# Get the latest Git commit hash and set it in package.json of all watchers
+RUN COMMIT_HASH=$(git rev-parse HEAD) && \
+    find . -name 'package.json' -exec sh -c ' \
+    for packageFile; do \
+        jq --arg commitHash "$0" ".commitHash = \$commitHash" "$packageFile" > "$packageFile.tmp" && \
+        mv "$packageFile.tmp" "$packageFile"; \
+    done \
+    ' "$COMMIT_HASH" {} \;
+
 RUN echo "Building azimuth-watcher-ts" && \
    yarn && yarn build

@@ -6,5 +6,10 @@ WORKDIR /app

 COPY . .

+# Get the latest Git commit hash and set in package.json
+RUN COMMIT_HASH=$(git rev-parse HEAD) && \
+  jq --arg commitHash "$COMMIT_HASH" '.commitHash = $commitHash' package.json > tmp.json && \
+  mv tmp.json package.json
+
 RUN echo "Installing dependencies and building merkl-sushiswap-v3-watcher-ts" && \
  yarn && yarn build
@@ -6,5 +6,10 @@ WORKDIR /app

 COPY . .

+# Get the latest Git commit hash and set in package.json
+RUN COMMIT_HASH=$(git rev-parse HEAD) && \
+  jq --arg commitHash "$COMMIT_HASH" '.commitHash = $commitHash' package.json > tmp.json && \
+  mv tmp.json package.json
+
 RUN echo "Installing dependencies and building sushiswap-v3-watcher-ts" && \
  yarn && yarn build
@@ -32,7 +32,7 @@ RUN \

 # [Optional] Uncomment this section to install additional OS packages.
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends jq gettext-base
+    && apt-get -y install --no-install-recommends jq gettext-base git

 # [Optional] Uncomment if you want to install an additional version of node using nvm
 # ARG EXTRA_NODE_VERSION=10
@@ -8,21 +8,27 @@ CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/data}"
 CERC_ENABLE_CORS="${CERC_ENABLE_CORS:-false}"
 CERC_SINGLE_PAGE_APP="${CERC_SINGLE_PAGE_APP}"

-if [ -z "${CERC_SINGLE_PAGE_APP}" ]; then
-  if [ 1 -eq $(find "${CERC_WEBAPP_FILES_DIR}" -name '*.html' | wc -l) ] && [ -d "${CERC_WEBAPP_FILES_DIR}/static" ]; then
+if [ -z "${CERC_SINGLE_PAGE_APP}" ]; then 
+  # If there is only one HTML file, assume an SPA.
+  if [ 1 -eq $(find "${CERC_WEBAPP_FILES_DIR}" -name '*.html' | wc -l) ]; then
    CERC_SINGLE_PAGE_APP=true
  else
    CERC_SINGLE_PAGE_APP=false
  fi
 fi

-if [ "true" == "$CERC_ENABLE_CORS" ]; then
+# ${var,,} is a lower-case comparison
+if [ "true" == "${CERC_ENABLE_CORS,,}" ]; then
  CERC_HTTP_EXTRA_ARGS="$CERC_HTTP_EXTRA_ARGS --cors"
 fi

-if [ "true" == "$CERC_SINGLE_PAGE_APP" ]; then
+# ${var,,} is a lower-case comparison
+if [ "true" == "${CERC_SINGLE_PAGE_APP,,}" ]; then
+  echo "Serving content as single-page app.  If this is wrong, set 'CERC_SINGLE_PAGE_APP=false'"
  # Create a catchall redirect back to /
  CERC_HTTP_EXTRA_ARGS="$CERC_HTTP_EXTRA_ARGS --proxy http://localhost:${CERC_LISTEN_PORT}?"
+else
+  echo "Serving content normally.  If this is a single-page app, set 'CERC_SINGLE_PAGE_APP=true'"
 fi

 LACONIC_HOSTED_CONFIG_FILE=${LACONIC_HOSTED_CONFIG_FILE}
@@ -39,4 +45,4 @@ if [ -f "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
 fi

 /scripts/apply-runtime-env.sh ${CERC_WEBAPP_FILES_DIR}
-http-server $CERC_HTTP_EXTRA_ARGS -p ${CERC_LISTEN_PORT} "${CERC_WEBAPP_FILES_DIR}"
+http-server $CERC_HTTP_EXTRA_ARGS -p ${CERC_LISTEN_PORT} "${CERC_WEBAPP_FILES_DIR}"
@@ -53,7 +53,7 @@ Inside deployment directory, open the `config.env` file  and set following env v

 ```bash
 # External Filecoin (ETH RPC) endpoint to point the watcher to
-CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
+CERC_ETH_RPC_ENDPOINTS=https://example-lotus-endpoint-1/rpc/v1,https://example-lotus-endpoint-2/rpc/v1
 ```

 ### Start the deployment
@@ -2,7 +2,7 @@ version: "1.0"
 name: ajna
 description: "Ajna watcher stack"
 repos:
-  - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.1
+  - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.13
 containers:
  - cerc/watcher-ajna
 pods:
@@ -4,7 +4,7 @@ Instructions to setup and deploy Azimuth Watcher stack

 ## Setup

-Prerequisite: `ipld-eth-server` RPC and GQL endpoints
+Prerequisite: External RPC endpoints

 Clone required repositories:

@@ -44,34 +44,42 @@ network:
     - 0.0.0.0:9000:9000
    azimuth-watcher-server:
     - 0.0.0.0:3001:3001
+     - 0.0.0.0:9001:9001
    censures-watcher-job-runner:
     - 0.0.0.0:9002:9002
    censures-watcher-server:
     - 0.0.0.0:3002:3002
+     - 0.0.0.0:9003:9003
    claims-watcher-job-runner:
     - 0.0.0.0:9004:9004
    claims-watcher-server:
     - 0.0.0.0:3003:3003
+     - 0.0.0.0:9005:9005
    conditional-star-release-watcher-job-runner:
     - 0.0.0.0:9006:9006
    conditional-star-release-watcher-server:
     - 0.0.0.0:3004:3004
+     - 0.0.0.0:9007:9007
    delegated-sending-watcher-job-runner:
     - 0.0.0.0:9008:9008
    delegated-sending-watcher-server:
     - 0.0.0.0:3005:3005
+     - 0.0.0.0:9009:9009
    ecliptic-watcher-job-runner:
     - 0.0.0.0:9010:9010
    ecliptic-watcher-server:
     - 0.0.0.0:3006:3006
+     - 0.0.0.0:9011:9011
    linear-star-release-watcher-job-runner:
     - 0.0.0.0:9012:9012
    linear-star-release-watcher-server:
     - 0.0.0.0:3007:3007
+     - 0.0.0.0:9013:9013
    polls-watcher-job-runner:
     - 0.0.0.0:9014:9014
    polls-watcher-server:
     - 0.0.0.0:3008:3008
+     - 0.0.0.0:9015:9015
    gateway-server:
     - 0.0.0.0:4000:4000
 ...
@@ -94,7 +102,7 @@ Inside the deployment directory, open the file `config.env` and add variable to

  ```bash
  # External RPC endpoints
-  CERC_IPLD_ETH_RPC=
+  CERC_ETH_RPC_ENDPOINTS=https://example-rpc-endpoint-1,https://example-rpc-endpoint-2
  ```

 * NOTE: If RPC endpoint is on the host machine, use `host.docker.internal` as the hostname to access the host port, or use the `ip a` command to find the IP address of the `docker0` interface (this will usually be something like `172.17.0.1` or `172.18.0.1`)
@@ -120,4 +128,7 @@ To stop all azimuth services and also delete data:

 ```bash
 laconic-so deployment --dir azimuth-deployment stop --delete-volumes
+
+# Remove deployment directory (deployment will have to be recreated for a re-run)
+rm -r azimuth-deployment
 ```
@@ -1,7 +1,7 @@
 version: "1.0"
 name: azimuth
 repos:
-  - github.com/cerc-io/azimuth-watcher-ts@v0.1.3
+  - github.com/cerc-io/azimuth-watcher-ts@0.1.6
 containers:
  - cerc/watcher-azimuth
 pods:
@@ -0,0 +1,26 @@
+# Blast stack
+
+## Clone required repositories
+```
+$ laconic-so --stack fixturenet-blast setup-repositories
+```
+## Build the stack's containers
+```
+$ laconic-so --stack fixturenet-blast build-containers
+```
+## Create a deployment of the stack
+```
+$ laconic-so --stack fixturenet-blast deploy init --map-ports-to-host any-same --output blast-spec.yml
+```
+[Insert details on how to configure the stack]
+```
+$ laconic-so --stack fixturenet-blast deploy create --deployment-dir blast-deployment --spec-file blast-spec.yml
+```
+## Start the stack
+```
+$ laconic-so deployment --dir blast-deployment start
+```
+Check logs:
+```
+$ laconic-so deployment --dir blast-deployment logs
+```
@@ -0,0 +1,17 @@
+version: "1.0"
+name: fixturenet-blast
+description: "A blast devnet stack"
+repos:
+  - github.com/blast-io/blast
+  - git.vdb.to/cerc-io/lighthouse
+containers:
+  - cerc/webapp-base
+  - cerc/lighthouse
+  - cerc/lighthouse-cli
+  - cerc/foundry
+  - cerc/fixturenet-eth-lighthouse
+
+pods:
+  - fixturenet-blast
+  - foundry
+  
@@ -43,16 +43,18 @@ customized by editing the "spec" file generated by `laconic-so deploy init`.
 ```
 $ cat graph-node-spec.yml
 stack: graph-node
-ports:
-  graph-node:
-    - '8000:8000'
-    - '8001'
-    - '8020:8020'
-    - '8030'
-  ipfs:
-    - '8080'
-    - '4001'
-    - '5001:5001'
+network:
+  ports:
+    graph-node:
+      - '8000:8000'
+      - '8001'
+      - '8020:8020'
+      - '8030'
+      - '8040'
+    ipfs:
+      - '8080'
+      - '4001'
+      - '5001:5001'
 ...
 ```

@@ -64,7 +66,7 @@ laconic-so --stack graph-node deploy create --spec-file graph-node-spec.yml --de

 ## Start the stack

-Create an env file with the following values to be set before starting the stack:
+Update `config.env` file inside deployment directory with the following values before starting the stack:

 ```bash
 # Set ETH RPC endpoint the graph node will use
@@ -76,21 +78,35 @@ export ETH_RPC_PORT=
 # The etherum network(s) graph-node will connect to
 # Set this to a space-separated list of the networks where each entry has the form NAME:URL
 export ETH_NETWORKS=
+
+# Optional:
+
+# Timeout for ETH RPC requests in seconds (default: 180s)
+export GRAPH_ETHEREUM_JSON_RPC_TIMEOUT=
+
+# Number of times to retry ETH RPC requests (default: 10)
+export GRAPH_ETHEREUM_REQUEST_RETRIES=
+
+# Maximum number of blocks to scan for triggers in each request (default: 2000)
+export GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE=
+
+# Maximum number of concurrent requests made against Ethereum for requesting transaction receipts during block ingestion (default: 1000)
+export GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS=
+
+# Ref: https://git.vdb.to/cerc-io/graph-node/src/branch/master/docs/environment-variables.md
 ```

-Example env file:
+Example `config.env` file:

 ```bash
 export ETH_RPC_HOST=filecoin.chainup.net
 export ETH_RPC_PORT=443

 export ETH_NETWORKS=filecoin:https://filecoin.chainup.net/rpc/v1
-```

-Set the environment variables:
-
-```bash
-source <PATH_TO_ENV_FILE>
+export GRAPH_ETHEREUM_JSON_RPC_TIMEOUT=360
+export GRAPH_ETHEREUM_REQUEST_RETRIES=5
+export GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE=50
 ```

 Deploy the stack:
@@ -3,7 +3,9 @@ name: graph-node
 description: "Stack for running graph-node"
 repos:
  - github.com/graphprotocol/graph-node
+  - github.com/cerc-io/watcher-ts@v0.2.92
 containers:
  - cerc/graph-node
+  - cerc/watcher-ts
 pods:
  - graph-node
@@ -0,0 +1,26 @@
+# Blast stack
+
+## Clone required repositories
+```
+$ laconic-so --stack mainnet-blast setup-repositories
+```
+## Build the stack's containers
+```
+$ laconic-so --stack mainnet-blast build-containers
+```
+## Create a deployment of the stack
+```
+$ laconic-so --stack mainnet-blast deploy init --map-ports-to-host any-same --output blast-spec.yml
+```
+[Insert details on how to configure the stack]
+```
+$ laconic-so --stack mainnet-blast deploy create --deployment-dir blast-deployment --spec-file blast-spec.yml
+```
+## Start the stack
+```
+$ laconic-so deployment --dir blast-deployment start
+```
+Check logs:
+```
+$ laconic-so deployment --dir blast-deployment logs
+```
@@ -0,0 +1,39 @@
+# Copyright © 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+
+from pathlib import Path
+from shutil import copy
+import yaml
+
+
+def create(context, extra_args):
+    # Our goal here is just to copy the json files for blast
+    yml_path = context.deployment_dir.joinpath("spec.yml")
+    with open(yml_path, 'r') as file:
+        data = yaml.safe_load(file)
+
+    mount_point = data['volumes']['blast-data']
+    if mount_point[0] == "/":
+        deploy_dir = Path(mount_point)
+    else:
+        deploy_dir = context.deployment_dir.joinpath(mount_point)
+
+    command_context = extra_args[2]
+    compose_file = [f for f in command_context.cluster_context.compose_files if "mainnet-blast" in f][0]
+    source_config_file = Path(compose_file).parent.parent.joinpath("config", "mainnet-blast", "genesis.json")
+    copy(source_config_file, deploy_dir)
+    source_config_file = Path(compose_file).parent.parent.joinpath("config", "mainnet-blast", "rollup.json")
+    copy(source_config_file, deploy_dir)
@@ -0,0 +1,12 @@
+version: "1.0"
+name: mainnet-blast
+description: "A blast stack"
+repos:
+  - github.com/blast-io/blast
+  - git.vdb.to/cerc-io/lighthouse
+containers:
+  - cerc/webapp-base
+  - cerc/lighthouse
+  - cerc/lighthouse-cli
+pods:
+  - mainnet-blast
@@ -14,7 +14,8 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.

 from stack_orchestrator.util import get_yaml
-from stack_orchestrator.deploy.deploy_types import DeployCommandContext, LaconicStackSetupCommand, DeploymentContext
+from stack_orchestrator.deploy.deploy_types import DeployCommandContext, LaconicStackSetupCommand
+from stack_orchestrator.deploy.deployment_context import DeploymentContext
 from stack_orchestrator.deploy.stack_state import State
 from stack_orchestrator.deploy.deploy_util import VolumeMapping, run_container_command
 from stack_orchestrator.command_types import CommandOptions
@@ -53,7 +53,7 @@ Inside deployment directory, open the `config.env` file  and set following env v

 ```bash
 # External Filecoin (ETH RPC) endpoint to point the watcher to
-CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
+CERC_ETH_RPC_ENDPOINTS=https://example-lotus-endpoint-1/rpc/v1,https://example-lotus-endpoint-2/rpc/v1
 ```

 ### Start the deployment
@@ -2,7 +2,7 @@ version: "1.0"
 name: merkl-sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.7
+  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.14
 containers:
  - cerc/watcher-merkl-sushiswap-v3
 pods:
@@ -4,6 +4,7 @@
 * Comes with the following built-in exporters / dashboards:
  * Chain Head Exporter - for tracking chain heads given external ETH RPC endpoints
  * Watchers dashboard
+  * laconicd dashboard
  * [Prometheus Blackbox](https://grafana.com/grafana/dashboards/7587-prometheus-blackbox-exporter/) - for tracking HTTP endpoints
  * [NodeJS Application Dashboard](https://grafana.com/grafana/dashboards/11159-nodejs-application-dashboard/) - for default NodeJS metrics
  * [PostgreSQL Database](https://grafana.com/grafana/dashboards/9628-postgresql-database/) - for monitoring Postgres dbs
@@ -99,6 +100,7 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --de
      - targets:
        - <HTTP_ENDPOINT_1>
        - <HTTP_ENDPOINT_2>
+        - <LACONICD_GQL_ENDPOINT>
  ```

 * Postgres (in-stack exporter):
@@ -116,12 +118,45 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --de
    ```
  * Add database credentials to be used in `auth_modules` in the postgres-exporter config file (`monitoring-deployment/config/monitoring/postgres-exporter.yml`)

+* laconicd: update the `laconicd` job with a laconicd node's REST endpoint host and port:
+
+  ```yml
+  ...
+  - job_name: laconicd
+  static_configs:
+    - targets: ['example-host:1317']
+  ...
+  ```
+
 Note: Use `host.docker.internal` as host to access ports on the host machine

 ### Grafana Config

 Place the dashboard json files in grafana dashboards config directory (`monitoring-deployment/config/monitoring/grafana/dashboards`) in the deployment folder

+#### Graph Node Config
+
+For graph-node dashboard postgres datasource needs to be setup in `monitoring-deployment/config/monitoring/grafana/provisioning/datasources/graph-node-postgres.yml` (in deployment folder)
+
+```yml
+# graph-node-postgres.yml
+...
+datasources:
+  - name: Graph Node Postgres
+    type: postgres
+    jsonData:
+      # Set name to remote graph-node database name
+      database: graph-node
+      ...
+    # Set user to remote graph-node database username
+    user: graph-node
+    # Add URL for remote graph-node database
+    url: graph-node-db:5432
+    # Set password for graph-node database
+    secureJsonData:
+      password: 'password'
+```
+
 ### Env

 Set the following env variables in the deployment env config file (`monitoring-deployment/config.env`):
@@ -144,6 +179,11 @@ Set the following env variables in the deployment env config file (`monitoring-d
  # Grafana server host URL (used in various links in alerts, etc.)
  # (Optional, default: http://localhost:3000)
  GF_SERVER_ROOT_URL=
+
+  
+  # RPC endpoint used by graph-node for upstream head metric
+  # (Optional, default: https://mainnet.infura.io/v3)
+  GRAPH_NODE_RPC_ENDPOINT=
  ```

 ## Start the stack
@@ -46,41 +46,46 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
    static_configs:
      - targets:
        - <AZIMUTH_GATEWAY_GQL_ENDPOINT>
+        - <LACONICD_GQL_ENDPOINT>
+  ...
+  - job_name: laconicd
+  static_configs:
+    - targets: ['LACONICD_REST_HOST:LACONICD_REST_PORT']
  ...
  - job_name: azimuth
    scrape_interval: 10s
    metrics_path: /metrics
    scheme: http
    static_configs:
-      - targets: ['AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_PORT']
+      - targets: ['AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_METRICS_PORT', 'AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'azimuth'
          chain: 'ethereum'
-      - targets: ['CENSURES_WATCHER_HOST:CENSURES_WATCHER_PORT']
+      - targets: ['CENSURES_WATCHER_HOST:CENSURES_WATCHER_METRICS_PORT', 'CENSURES_WATCHER_HOST:CENSURES_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'censures'
          chain: 'ethereum'
-      - targets: ['CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_PORT']
+      - targets: ['CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_METRICS_PORT', 'CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'claims'
          chain: 'ethereum'
-      - targets: ['CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_PORT']
+      - targets: ['CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_METRICS_PORT', 'CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'conditional_star_release'
          chain: 'ethereum'
-      - targets: ['DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_PORT']
+      - targets: ['DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_METRICS_PORT', 'DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'delegated_sending'
          chain: 'ethereum'
-      - targets: ['ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_PORT']
+      - targets: ['ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_METRICS_PORT', 'ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'ecliptic'
          chain: 'ethereum'
-      - targets: ['LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_PORT']
+      - targets: ['LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_METRICS_PORT', 'LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'linear_star_release'
          chain: 'ethereum'
-      - targets: ['POLLS_WATCHER_HOST:POLLS_WATCHER_PORT']
+      - targets: ['POLLS_WATCHER_HOST:POLLS_WATCHER_METRICS_PORT', 'POLLS_WATCHER_HOST:POLLS_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'polls'
          chain: 'ethereum'
@@ -90,29 +95,49 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
    metrics_path: /metrics
    scheme: http
    static_configs:
-      - targets: ['SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_PORT']
+      - targets: ['SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_METRICS_PORT', 'SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'sushiswap'
          chain: 'filecoin'
-      - targets: ['MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_PORT']
+      - targets: ['MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_METRICS_PORT', 'MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'merkl_sushiswap'
          chain: 'filecoin'
+
+  - job_name: ajna
+    scrape_interval: 20s
+    metrics_path: /metrics
+    scheme: http
+    static_configs:
+      - targets: ['AJNA_WATCHER_HOST:AJNA_WATCHER_METRICS_PORT', 'AJNA_WATCHER_HOST:AJNA_WATCHER_GQL_METRICS_PORT']
+        labels:
+          instance: 'ajna'
+          chain: 'filecoin'
+
+  - job_name: graph-node
+    metrics_path: /metrics
+    scrape_interval: 30s
+    static_configs:
+      - targets: ['GRAPH_NODE_HOST:GRAPH_NODE_HOST_METRICS_PORT']
  ```

 Add scrape config as done above for any additional watcher to add it to the Watchers dashboard.

 ### Grafana alerts config

-Place the pre-configured watcher alerts rules in Grafana provisioning directory:
+Place the pre-configured alerts rules in Grafana provisioning directory:

  ```bash
+  # watcher alert rules
  cp monitoring-watchers-deployment/config/monitoring/watcher-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
+
+  # subgraph alert rules
+  cp monitoring-watchers-deployment/config/monitoring/subgraph-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
  ```

 Update the alerting contact points config (`monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/contactpoints.yml`) with desired contact points

-Add corresponding routes to the notification policies config (`monitoring-watchers-deployment/monitoring/grafana/provisioning/alerting/policies.yaml`) with appropriate object-matchers:
+Add corresponding routes to the notification policies config (`monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/policies.yml`) with appropriate object-matchers:

  ```yml
  ...
@@ -120,7 +145,7 @@ Add corresponding routes to the notification policies config (`monitoring-watche
      - receiver: SlackNotifier
      object_matchers:
        # Add matchers below
-        - ['grafana_folder', '=', 'WatcherAlerts']
+        - ['grafana_folder', '=~', 'WatcherAlerts|SubgraphAlerts']
  ```

 ### Env
@@ -134,6 +159,9 @@ Set the following env variables in the deployment env config file (`monitoring-w
  # Grafana server host URL to be used
  # (Optional, default: http://localhost:3000)
  GF_SERVER_ROOT_URL=
+
+  # List of subgraph ids to configure alerts for (separated by |)
+  CERC_GRAFANA_ALERTS_SUBGRAPH_IDS=
  ```

 ## Start the stack
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
telackey	3ba9436863	Add git to webapp base container.	2024-06-24 23:57:15 -05:00
dboreham	b4e82ebc19	Merge pull request 'Fix mainnet laconic deploy setup' (#850 ) from dboreham/fix-mainnet-laconic-init into main Reviewed-on: cerc-io/stack-orchestrator#850	2024-06-22 01:25:07 +00:00
dboreham	2364924a59	Fix mainnet laconic deploy setup	2024-06-21 19:24:33 -06:00
nabarun	a223797b4a	Update graph-node dashboard to show individual subgraph increase in query count (#846 ) Part of [Metrics and logging for GQL queries in watcher](https://www.notion.so/Metrics-and-logging-for-GQL-queries-in-watcher-928c692292b140a2a4f52cda9795df5e) Reviewed-on: cerc-io/stack-orchestrator#846 Co-authored-by: Nabarun <nabarun@deepstacksoft.com> Co-committed-by: Nabarun <nabarun@deepstacksoft.com>	2024-06-20 09:27:23 +00:00
nabarun	b8004e9870	Add Grafana panels for graph-node subgraph GQL queries (#845 ) Part of [Deploy v2 and updated v3 sushiswap subgraphs](https://www.notion.so/Deploy-v2-and-updated-v3-sushiswap-subgraphs-e331945fdeea487c890706fc22c6cc94) Reviewed-on: cerc-io/stack-orchestrator#845 Co-authored-by: Nabarun <nabarun@deepstacksoft.com> Co-committed-by: Nabarun <nabarun@deepstacksoft.com>	2024-06-19 10:40:54 +00:00
prathamesh	3fd99a1522	Handle race condition in laconic registry CLI tests (#843 ) Part of cerc-io/laconic-registry-cli#63 Reviewed-on: cerc-io/stack-orchestrator#843 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-06-19 06:41:58 +00:00
nabarun	842d999792	Add alert rules for secured secured-finance subgraph watcher (#842 ) Part of [Generate secured-finance subgraph watcher with codegen](https://www.notion.so/Generate-secured-finance-subgraph-watcher-with-codegen-2923413e0af54ea787c5435d6966f3bb) - Update watcher dashboard labels Reviewed-on: cerc-io/stack-orchestrator#842 Co-authored-by: Nabarun <nabarun@deepstacksoft.com> Co-committed-by: Nabarun <nabarun@deepstacksoft.com>	2024-06-18 12:28:02 +00:00
dboreham	d6a1fb3279	Merge pull request 'Fix image tag name' (#841 ) from dboreham/fix-remote-image-tags into main Reviewed-on: cerc-io/stack-orchestrator#841	2024-06-13 14:32:26 +00:00
dboreham	bf1eccd486	Fix image tag name	2024-06-13 08:31:45 -06:00
dboreham	3fb025b5c9	Make remote image tags unique to the deployment (#838 ) Reviewed-on: cerc-io/stack-orchestrator#838 Reviewed-by: Thomas E Lackey <telackey@noreply.git.vdb.to> Co-authored-by: David Boreham <david@bozemanpass.com> Co-committed-by: David Boreham <david@bozemanpass.com>	2024-06-13 03:26:58 +00:00
prathameshandnabarun	4acb06325b	Update watcher dashboard and config templates (#835 ) Part of [Metrics and logging for GQL queries in watcher](https://www.notion.so/Metrics-and-logging-for-GQL-queries-in-watcher-928c692292b140a2a4f52cda9795df5e) - Update watcher config templates after config refactoring - Mount watcher GQL query log files on volumes - Update watcher dashboard to - add a panel to show latest processed block number - use latest processed block from sync status for diff values Co-authored-by: Nabarun <nabarun@deepstacksoft.com> Reviewed-on: cerc-io/stack-orchestrator#835 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-06-12 11:52:51 +00:00
dboreham	b80b647fa4	Merge pull request 'Remove files migrated to external repo' (#836 ) from dboreham/remove-snowball-stack into main Reviewed-on: cerc-io/stack-orchestrator#836	2024-06-07 03:17:48 +00:00
dboreham	9a1d3bb0f1	Remove files migrated to external repo	2024-06-06 20:44:51 -06:00
prathamesh	abc0c2423f	Add panels for GQL metrics to watcher dashboard (#834 ) Part of [Metrics and logging for GQL queries in watcher](https://www.notion.so/Metrics-and-logging-for-GQL-queries-in-watcher-928c692292b140a2a4f52cda9795df5e) Reviewed-on: cerc-io/stack-orchestrator#834 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-06-06 11:47:18 +00:00
nabarun	a322d6eed4	Add dashboard for graph-node subgraphs (#832 ) Part of [Deploy v2 and updated v3 sushiswap subgraphs](https://www.notion.so/Deploy-v2-and-updated-v3-sushiswap-subgraphs-e331945fdeea487c890706fc22c6cc94) - Add param `GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS` in graph-node stack - <https://github.com/graphprotocol/graph-node/blob/v0.31.0/docs/environment-variables.md#json-rpc-configuration-for-evm-chains> - Add dashboard for subgraphs deployment in graph-node - Show subgraph names in dashboard - Add watcher dashboard panel for showing watcher release version and commit hash Reviewed-on: cerc-io/stack-orchestrator#832 Co-authored-by: Nabarun <nabarun@deepstacksoft.com> Co-committed-by: Nabarun <nabarun@deepstacksoft.com>	2024-06-04 07:21:27 +00:00
prathameshandnabarun	ed8914b8d3	Upgrade watchers and their config (#827 ) Part of [Investigate subgraph watchers lagging behind head](https://www.notion.so/Investigate-subgraph-watchers-lagging-behind-head-01b72294ca8e4f658e4c0e86b36d19e2) Co-authored-by: Nabarun <nabarun@deepstacksoft.com> Reviewed-on: cerc-io/stack-orchestrator#827 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-05-23 04:12:31 +00:00
telackey	fef7649683	Refactor SPA check. (#831 ) Reviewed-on: cerc-io/stack-orchestrator#831 Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com> Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>	2024-05-22 19:00:42 +00:00
telackey	579b402f2f	Make the SPA detection even simpler. (#830 ) Reviewed-on: cerc-io/stack-orchestrator#830 Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com> Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>	2024-05-22 18:31:57 +00:00
telackey	25d0bc8a98	Case insensitive comparison (#829 ) Reviewed-on: cerc-io/stack-orchestrator#829 Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com> Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>	2024-05-22 18:18:27 +00:00
telackey	855288368c	Add messages around the SPA auto-detect. (#828 ) Tweak the auto-detection logic slightly for single-page apps, and also print the results. Reviewed-on: cerc-io/stack-orchestrator#828 Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com> Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>	2024-05-22 17:42:28 +00:00
prathamesh	8f2da38183	Add alerts for graph-node subgraphs (#821 ) Part of [Deploy v2 and updated v3 sushiswap subgraphs](https://www.notion.so/Deploy-v2-and-updated-v3-sushiswap-subgraphs-e331945fdeea487c890706fc22c6cc94) Reviewed-on: cerc-io/stack-orchestrator#821 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-05-17 04:01:41 +00:00
dboreham	254f95e59f	Merge pull request 'Remove legacy commands for docker startup' (#826 ) from dboreham/remove-old-docker-ci into main Reviewed-on: cerc-io/stack-orchestrator#826	2024-05-15 16:09:09 +00:00
dboreham	0acb6ea6bc	Remove legacy commands for docker startup	2024-05-15 09:04:19 -06:00
prathamesh	b9369a13e6	Update watcher dashboard with panels for ETH RPC request failures and durations (#825 ) Part of [Regenerate ajna watcher with updated subgraph config ](https://www.notion.so/Regenerate-ajna-watcher-with-updated-subgraph-config-c9bbecb033024c13a7515c7f1efc3363) Requires [Add metrics to monitor errors and duration for ETH RPC requests](https://github.com/cerc-io/watcher-ts/pull/507) Reviewed-on: cerc-io/stack-orchestrator#825 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-05-15 13:52:18 +00:00
prathamesh	0b1eb8eb0f	Update ajna-watcher version in ajna stack (#824 ) Part of [Regenerate ajna watcher with updated subgraph config](https://www.notion.so/Regenerate-ajna-watcher-with-updated-subgraph-config-c9bbecb033024c13a7515c7f1efc3363) Reviewed-on: cerc-io/stack-orchestrator#824 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-05-14 08:27:59 +00:00
nabarun	78092f5793	Update subgraph watcher stacks to configure multiple RPC endpoints (#822 ) Part of [Ability to configure watchers with multiple RPC endpoints](https://www.notion.so/Ability-to-configure-watchers-with-multiple-RPC-endpoints-dc8d3ff4d647404ab718dfd5a4c9035c) Reviewed-on: cerc-io/stack-orchestrator#822 Co-authored-by: Nabarun <nabarun@deepstacksoft.com> Co-committed-by: Nabarun <nabarun@deepstacksoft.com>	2024-05-10 04:58:30 +00:00
prathameshandnabarun	247dbdd2f0	Update subgraph watcher versions for improved `eth_getLogs` calls (#820 ) Part of [Investigate subgraph watchers lagging behind head](https://www.notion.so/Investigate-subgraph-watchers-lagging-behind-head-01b72294ca8e4f658e4c0e86b36d19e2) Co-authored-by: Nabarun <nabarun@deepstacksoft.com> Reviewed-on: cerc-io/stack-orchestrator#820 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-05-07 04:18:45 +00:00
dboreham	30db1f58d0	Refactor for new external stack directory layout under common parent (#815 ) Reviewed-on: cerc-io/stack-orchestrator#815 Co-authored-by: David Boreham <david@bozemanpass.com> Co-committed-by: David Boreham <david@bozemanpass.com>	2024-04-29 23:03:20 +00:00
prathamesh	09a1cbb966	Additional env configuration in graph-node stack (#812 ) Part of [Deploy v2 and updated v3 sushiswap subgraphs](https://www.notion.so/Deploy-v2-and-updated-v3-sushiswap-subgraphs-e331945fdeea487c890706fc22c6cc94) Reviewed-on: cerc-io/stack-orchestrator#812 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-04-26 05:45:00 +00:00
dboreham	13ce521d84	Fix config dir processing for external stacks (#810 ) Reviewed-on: cerc-io/stack-orchestrator#810 Co-authored-by: David Boreham <david@bozemanpass.com> Co-committed-by: David Boreham <david@bozemanpass.com>	2024-04-23 21:47:20 +00:00
dboreham	6e4dae9777	Add external stack support (#806 ) Reviewed-on: cerc-io/stack-orchestrator#806 Co-authored-by: David Boreham <david@bozemanpass.com> Co-committed-by: David Boreham <david@bozemanpass.com>	2024-04-18 21:22:47 +00:00
telackey	9043a67c7c	Skip checks on requests we've already seen (#805 ) Reviewed-on: cerc-io/stack-orchestrator#805 Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com> Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>	2024-04-15 20:27:35 +00:00
telackey	7f84a45cfd	Switch repo to cerc-io org. (#804 ) Update stack to track moved repo. Reviewed-on: cerc-io/stack-orchestrator#804 Reviewed-by: David Boreham <dboreham@noreply.git.vdb.to>	2024-04-15 18:59:08 +00:00
telackey	4126f2fc43	Add --fqdn-policy option to deploy-webapp-from-registry. (#802 ) This add a new option `--fqdn-policy` to the `deploy-webapp-from-registry`. The default policy, `prohibit` means that `ApplicationDeploymentRequests` which specify a FQDN will be rejected. The `allow` policy will cause them to be processed. The `preexisting` policy will only process them if an existing `DnsRecord` exists in the registry with the correct ownership. The latter would be useful in conjunction with a pre-checking scheme in the UI (eg, that the DNS entry is properly configured, the domain is under the control of the requestor, etc.) Only after all the checks were successful would the `DnsRecord` be created, allowing for `ApplicationDeploymentRequests` to use it. Reviewed-on: cerc-io/stack-orchestrator#802 Reviewed-by: David Boreham <dboreham@noreply.git.vdb.to> Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com> Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>	2024-04-15 12:20:35 +00:00
prathamesh	345d200873	Add a laconicd Grafana dashboard to monitoring stack (#799 ) Part of https://www.notion.so/Monitoring-and-alerting-for-laconicd-86727c3b4dde4dc993d87d6e29f935fe - Add a laconicd Grafana dashboard - Update fixturenet-laconicd script to expose metrics - Upgrade Grafana version to avoid errors while saving changes made to a dashboard (see [thread](https://community.grafana.com/t/error-cannot-add-property-ishandled-object-is-not-extensible/109268)) - Add an alert rule for Ajna watcher Reviewed-on: cerc-io/stack-orchestrator#799 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-04-11 07:59:36 +00:00
roysc	87fffca358	fixturenet-plugeth Deneb/Cancun upgrade (#789 ) Updates fixturenet-plugeth stack for the Deneb fork based on Geth v1.13.x: - updates genesis generator tool, and simplifies the config: the default from `ethereum-genesis-generator` can be used for a from-genesis Merged chain. Reviewed-on: cerc-io/stack-orchestrator#789 Reviewed-by: jonathanface <jonathanface@noreply.git.vdb.to> Reviewed-by: Thomas E Lackey <telackey@noreply.git.vdb.to>	2024-04-11 03:21:36 +00:00
jonathanface	66b92df498	Merge pull request 'Blast stack' (#777 ) from blast-stack into main Reviewed-on: cerc-io/stack-orchestrator#777	2024-04-08 15:51:10 +00:00
jonathanface	108a5a3440	Merge branch 'main' into blast-stack	2024-04-08 15:21:03 +00:00
jonathanface	4b04a39faf	linted	2024-04-08 15:17:44 +00:00
prathamesh	40f362511b	Run CI alert steps only on main (#797 ) Part of https://www.notion.so/Alerting-for-failing-CI-jobs-d0183b65453947aeab11dbddf989d9c0 - Run CI alert steps only on main to avoid alerts for in-progress PRs - The Slack alerts will be sent on a CI job failure if - A commit is pushed directly to main - A PR gets merged into main - A scheduled job runs on main Reviewed-on: cerc-io/stack-orchestrator#797 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-04-05 09:26:08 +00:00
prathamesh	9cd34ffebb	Add Slack alerts for failures on CI workflows (#793 ) Part of https://www.notion.so/Alerting-for-failing-CI-jobs-d0183b65453947aeab11dbddf989d9c0 Reviewed-on: cerc-io/stack-orchestrator#793 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-04-05 08:27:46 +00:00
jonathan@vulcanize.io	62f1962546	metrics on op-node	2024-04-02 17:35:13 +00:00
jonathan@vulcanize.io	2a24e71c92	added metrics addr flag	2024-04-02 16:20:25 +00:00
jonathan@vulcanize.io	c789b82782	metrics	2024-04-01 19:38:00 +00:00
jonathan@vulcanize.io	b3bc5a19ae	blast testnet, initial commit	2024-03-27 15:03:30 +00:00
jonathan@vulcanize.io	7f9e1da8ba	removed keycloak	2024-03-15 15:15:15 +00:00
jonathan@vulcanize.io	0149346927	adding trustrpc flag to op-node	2024-03-13 18:18:57 +00:00
jonathan@vulcanize.io	06de4fe485	follow established naming convention	2024-03-13 16:21:29 +00:00
jonathan@vulcanize.io	821d401575	fixed missing rollup	2024-03-13 04:28:16 +00:00
jonathan@vulcanize.io	5123111db0	copy whether absolute path or local	2024-03-13 03:56:17 +00:00
jonathan@vulcanize.io	02c33cb229	working state	2024-03-13 02:49:21 +00:00
jonathan@vulcanize.io	b4eda902ea	fixed optimum deployment	2024-03-08 19:27:36 +00:00
jonathan@vulcanize.io	b4df8104c8	tweaking yml	2024-03-08 17:20:44 +00:00
jonathan@vulcanize.io	07282cdd6e	minimal build	2024-03-08 02:56:23 +00:00
jonathan@vulcanize.io	e7c935fb78	integration testing, I think	2024-03-07 22:24:46 +00:00
jonathan@vulcanize.io	1a636799a6	filepath	2024-03-06 19:34:25 +00:00
jonathan@vulcanize.io	0aa4b350bd	keycloak implementation	2024-03-06 19:21:29 +00:00
jonathan@vulcanize.io	2252252072	comment format	2024-03-05 20:42:53 +00:00
jonathan@vulcanize.io	c92f15f47c	comment format	2024-03-05 20:38:09 +00:00
jonathan@vulcanize.io	fee32ec703	copying genesis.json to /data/blast-data for blast	2024-03-05 20:32:56 +00:00
dboreham	fb55c1425e	Beginnings of blast stack	2024-03-04 15:05:31 -07:00