testing blast update ecotone - add overrides and l2 beacon to compose, add RPC variables

Merge branch 'main' into blast-stack
linted
2024-05-24 22:57:53 +00:00 · 2024-04-08 15:21:03 +00:00 · 2024-04-08 15:17:44 +00:00 · 2024-04-05 09:26:08 +00:00 · 2024-04-05 08:27:46 +00:00 · 2024-04-04 10:50:02 +00:00
62 changed files with 33528 additions and 298 deletions
@@ -43,3 +43,19 @@ jobs:
        run: ./scripts/build_shiv_package.sh
      - name: "Run fixturenet-eth tests"
        run: ./tests/fixturenet-eth-plugeth/run-test.sh
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -47,3 +47,19 @@ jobs:
          sleep 5
      - name: "Run fixturenet-eth tests"
        run: ./tests/fixturenet-eth-plugeth/run-test.sh
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -45,4 +45,19 @@ jobs:
          sleep 5
      - name: "Run fixturenet-eth tests"
        run: ./tests/fixturenet-eth/run-test.sh
-
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -11,7 +11,7 @@ on:

 jobs:
  test:
-    name: "Run an Laconicd fixturenet test"
+    name: "Run Laconicd fixturenet and Laconic CLI tests"
    runs-on: ubuntu-latest
    steps:
      - name: 'Update'
@@ -46,3 +46,21 @@ jobs:
        run: ./scripts/build_shiv_package.sh
      - name: "Run fixturenet-laconicd tests"
        run: ./tests/fixturenet-laconicd/run-test.sh
+      - name: "Run laconic CLI tests"
+        run: ./tests/fixturenet-laconicd/run-cli-test.sh
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -19,3 +19,19 @@ jobs:
          python-version: '3.8'
      - name : "Run flake8"
        uses: py-actions/flake8@v2
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -54,3 +54,19 @@ jobs:
          # Hack using endsWith to workaround Gitea sometimes sending "publish-test" vs "refs/heads/publish-test"
          draft: ${{ endsWith('publish-test', github.ref ) }}
          files: ./laconic-so
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -51,4 +51,19 @@ jobs:
          source /opt/bash-utils/cgroup-helper.sh
          join_cgroup
          ./tests/container-registry/run-test.sh
-
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -49,4 +49,19 @@ jobs:
          source /opt/bash-utils/cgroup-helper.sh
          join_cgroup
          ./tests/database/run-test.sh
-
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -47,3 +47,19 @@ jobs:
          sleep 5
      - name: "Run deploy tests"
        run: ./tests/deploy/run-deploy-test.sh
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -51,4 +51,19 @@ jobs:
          source /opt/bash-utils/cgroup-helper.sh
          join_cgroup
          ./tests/k8s-deploy/run-deploy-test.sh
-
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -49,3 +49,19 @@ jobs:
          sleep 5
      - name: "Run webapp tests"
        run: ./tests/webapp-test/run-webapp-test.sh
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -47,5 +47,19 @@ jobs:
          sleep 5
      - name: "Run smoke tests"
        run: ./tests/smoke-test/run-smoke-test.sh
-
-
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}
@@ -1,3 +1,6 @@
 Change this file to trigger running the fixturenet-laconicd-test CI job
 Trigger
 Trigger
+Trigger
+Trigger
+Trigger
@@ -0,0 +1,9 @@
+# Fetching pre-built container images
+When Stack Orchestrator deploys a stack containing a suite of one or more containers it expects images for those containers to be on the local machine with a tag of the form `<image-name>:local` Images for these containers can be built from source (and optionally base container images from public registries) with the `build-containers` subcommand. 
+
+However, the task of building a large number of containers from source may consume considerable time and machine resources. This is where the `fetch-containers` subcommand steps in. It is designed to work exactly like `build-containers` but instead the images, pre-built, are fetched from an image registry then re-tagged for deployment. It can be used in place of `build-containers` for any stack provided the necessary containers, built for the local machine architecture (e.g. arm64 or x86-64) have already been published in an image registry.
+## Usage
+To use `fetch-containers`, provide an image registry path, a username and token/password with read access to the registry, and optionally specify `--force-local-overwrite`. If this argument is not specified, if there is already a locally built or previously fetched image for a stack container on the machine, it will not be overwritten and a warning issued.
+```
+$ laconic-so --stack mobymask-v3-demo fetch-containers --image-registry git.vdb.to/cerc-io --registry-username <registry-user> --registry-token <registry-token> --force-local-overwrite
+```
@@ -27,7 +27,7 @@ from decouple import config
 import click
 from pathlib import Path
 from stack_orchestrator.build import build_containers
-from stack_orchestrator.deploy.webapp.util import determine_base_container
+from stack_orchestrator.deploy.webapp.util import determine_base_container, TimedLogger
 from stack_orchestrator.build.build_types import BuildContext


@@ -40,9 +40,11 @@ from stack_orchestrator.build.build_types import BuildContext
@click.pass_context
 def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, tag):
    '''build the specified webapp container'''
+    logger = TimedLogger()

    quiet = ctx.obj.quiet
    debug = ctx.obj.debug
+    verbose = ctx.obj.verbose
    local_stack = ctx.obj.local_stack
    stack = ctx.obj.stack

@@ -51,12 +53,12 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t

    if local_stack:
        dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
-        print(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
+        logger.log(f'Local stack dev_root_path (CERC_REPO_BASE_DIR) overridden to: {dev_root_path}')
    else:
        dev_root_path = os.path.expanduser(config("CERC_REPO_BASE_DIR", default="~/cerc"))

-    if not quiet:
-        print(f'Dev Root is: {dev_root_path}')
+    if verbose:
+        logger.log(f'Dev Root is: {dev_root_path}')

    if not base_container:
        base_container = determine_base_container(source_repo)
@@ -65,6 +67,9 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
    container_build_env = build_containers.make_container_build_env(dev_root_path, container_build_dir, debug,
                                                                    force_rebuild, extra_build_args)

+    if verbose:
+        logger.log(f"Building base container: {base_container}")
+
    build_context_1 = BuildContext(
        stack,
        base_container,
@@ -74,9 +79,12 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
    )
    ok = build_containers.process_container(build_context_1)
    if not ok:
-        print("ERROR: Build failed.", file=sys.stderr)
+        logger.log("ERROR: Build failed.")
        sys.exit(1)

+    if verbose:
+        logger.log(f"Base container {base_container} build finished.")
+
    # Now build the target webapp.  We use the same build script, but with a different Dockerfile and work dir.
    container_build_env["CERC_WEBAPP_BUILD_RUNNING"] = "true"
    container_build_env["CERC_CONTAINER_BUILD_WORK_DIR"] = os.path.abspath(source_repo)
@@ -85,9 +93,12 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
                                                                          "Dockerfile.webapp")
    if not tag:
        webapp_name = os.path.abspath(source_repo).split(os.path.sep)[-1]
-        container_build_env["CERC_CONTAINER_BUILD_TAG"] = f"cerc/{webapp_name}:local"
-    else:
-        container_build_env["CERC_CONTAINER_BUILD_TAG"] = tag
+        tag = f"cerc/{webapp_name}:local"
+
+    container_build_env["CERC_CONTAINER_BUILD_TAG"] = tag
+
+    if verbose:
+        logger.log(f"Building app container: {tag}")

    build_context_2 = BuildContext(
        stack,
@@ -98,5 +109,9 @@ def command(ctx, base_container, source_repo, force_rebuild, extra_build_args, t
    )
    ok = build_containers.process_container(build_context_2)
    if not ok:
-        print("ERROR: Build failed.", file=sys.stderr)
-        sys.exit(1)
+        logger.log("ERROR: Build failed.")
+        sys.exit(1)
+
+    if verbose:
+        logger.log(f"App container {base_container} build finished.")
+        logger.log("build-webapp complete", show_step_time=False, show_total_time=True)
@@ -0,0 +1,80 @@
+
+# From: https://raw.githubusercontent.com/blast-io/deployment/master/docker-compose.yml
+services:
+  # generate jwt.txt if it's absent
+  generate-jwt:
+    image: blastio/openssl
+    volumes:
+      - blast-data:/blast:rw
+    command: >
+      sh -c "[ ! -f /blast/jwt.txt ] && openssl rand -hex 32 | tr -d '\n' > /blast/jwt.txt || exit 0"
+  # initialise geth db
+  geth-init:
+    image: blastio/blast-geth:${NETWORK:-testnet-sepolia}
+    volumes:
+      - blast-data:/blast:rw
+      - ../config/fixturenet-blast/genesis.json:/blast/genesis.json
+    entrypoint: /bin/sh
+    command: >
+      -c "[ ! -d /blast/${GETH_DATA_DIR:-blast-geth-data}/geth ] && /usr/local/bin/geth init --datadir=/blast/${GETH_DATA_DIR:-blast-geth-data} /blast/genesis.json || exit 0"
+    depends_on:
+      generate-jwt:
+        condition: service_completed_successfully
+    env_file: 
+      - ../config/fixturenet-blast/${NETWORK:-fixturenet}.config
+  blast-geth:
+    image: blastio/blast-geth:${NETWORK:-testnet-sepolia}
+    volumes:
+      - blast-data:/blast
+    ports:
+      - "9545"
+      - "9546"
+    command: >
+      --datadir=/blast/${GETH_DATA_DIR:-blast-geth-data}
+      --http
+      --http.corsdomain="*"
+      --http.vhosts="*"
+      --http.addr=0.0.0.0
+      --http.port=9545
+      --http.api=web3,debug,eth,txpool,net,engine
+      --ws
+      --ws.addr=0.0.0.0
+      --ws.port=9546
+      --ws.origins="*"
+      --ws.api=debug,eth,txpool,net,engine
+      --authrpc.addr="0.0.0.0"
+      --authrpc.port="8551"
+      --authrpc.vhosts="*"
+      --authrpc.jwtsecret=/blast/jwt.txt
+      --syncmode=full
+      --gcmode=archive
+      --nodiscover
+      --maxpeers=0
+      --rollup.disabletxpoolgossip=true
+    env_file: 
+      - ../config/fixturenet-blast/${NETWORK:-fixturenet}.config
+    depends_on:
+      geth-init:
+        condition: service_completed_successfully
+  op-node:
+    image: blastio/blast-optimism:${NETWORK:-testnet-sepolia}
+    volumes:
+      - blast-data:/blast
+      - ../config/fixturenet-blast/rollup.json:/blast/rollup.json
+    ports:
+      - "9003"
+    command: >
+      op-node
+      --l1="${CERC_L1_RPC}"
+      --l1.rpckind="any"
+      --l1.trustrpc=true
+      --l2="http://blast-geth:8551"
+      --l2.jwt-secret=/blast/jwt.txt
+      --rollup.config="/blast/rollup.json"
+    depends_on:
+      - blast-geth
+    env_file: 
+      - ../config/fixturenet-blast/${NETWORK:-fixturenet}.config
+
+volumes:
+  blast-data:
@@ -3,6 +3,9 @@ services:
    restart: unless-stopped
    image: cerc/laconicd:local
    command: ["sh", "/docker-entrypoint-scripts.d/create-fixturenet.sh"]
+    environment:
+      TEST_AUCTION_ENABLED: ${TEST_AUCTION_ENABLED}
+      TEST_REGISTRY_EXPIRY: ${TEST_REGISTRY_EXPIRY}
    volumes:
      # The cosmos-sdk node's database directory:
      - laconicd-data:/root/.laconicd
@@ -25,6 +28,7 @@ services:
    image: cerc/laconic-registry-cli:local
    volumes:
      - ../config/fixturenet-laconicd/registry-cli-config-template.yml:/registry-cli-config-template.yml
+      - ${BASE_DIR:-~/cerc}/laconic-registry-cli:/laconic-registry-cli

 volumes:
  laconicd-data:
@@ -0,0 +1,86 @@
+
+# From: https://raw.githubusercontent.com/blast-io/deployment/master/docker-compose.yml
+services:
+  # generate jwt.txt if it's absent
+  generate-jwt:
+    image: blastio/openssl
+    volumes:
+      - blast-data:/blast:rw
+    command: >
+      sh -c "[ ! -f /blast/jwt.txt ] && openssl rand -hex 32 | tr -d '\n' > /blast/jwt.txt || exit 0"
+  # initialise geth db
+  geth-init:
+    image: blastio/blast-geth:${NETWORK:-mainnet}
+    volumes:
+      - blast-data:/blast:rw
+    entrypoint: /bin/sh
+    command: >
+      -c "[ ! -d /blast/${GETH_DATA_DIR:-blast-geth-data}/geth ] && /usr/local/bin/geth init --datadir=/blast/${GETH_DATA_DIR:-blast-geth-data} /blast/genesis.json || exit 0"
+    depends_on:
+      generate-jwt:
+        condition: service_completed_successfully
+    env_file: 
+      - ../config/mainnet-blast/${NETWORK:-mainnet}.config
+  blast-geth:
+    image: blastio/blast-geth:${NETWORK:-mainnet}
+    volumes:
+      - blast-data:/blast
+    ports:
+      - "9545"
+      - "9546"
+      - "6060"
+    command: >
+      --datadir=/blast/${GETH_DATA_DIR:-blast-geth-data}
+      --http
+      --http.corsdomain="*"
+      --http.vhosts="*"
+      --http.addr=0.0.0.0
+      --http.port=9545
+      --http.api=web3,debug,eth,txpool,net,engine
+      --ws
+      --ws.addr=0.0.0.0
+      --ws.port=9546
+      --ws.origins="*"
+      --ws.api=debug,eth,txpool,net,engine
+      --authrpc.addr="0.0.0.0"
+      --authrpc.port="8551"
+      --authrpc.vhosts="*"
+      --authrpc.jwtsecret=/blast/jwt.txt
+      --syncmode=full
+      --metrics
+      --metrics.addr=0.0.0.0
+      --gcmode=archive
+      --nodiscover
+      --maxpeers=0
+      --rollup.disabletxpoolgossip=true
+      --override.canyon="${OVERRIDE_CANYON:-0}"
+      --override.ecotone="${OVERRIDE_ECOTONE:-1716843599}"
+    env_file: 
+      - ../config/mainnet-blast/${NETWORK:-mainnet}.config
+    depends_on:
+      geth-init:
+        condition: service_completed_successfully
+  op-node:
+    image: blastio/blast-optimism:${NETWORK:-mainnet}
+    volumes:
+      - blast-data:/blast
+    ports:
+      - "9003"
+      - "7300"
+    command: >
+      op-node
+      --l1="${L1_RPC:-https://eth-mainnet-1.vdb.to/}"
+      --l1.beacon="${L1_RPC_BEACON:-https://eth-mainnet-1.vdb.to/}"
+      --l1.rpckind="any"
+      --l1.trustrpc=true
+      --l2="${L2_RPC:-http://blast-geth:8551}"
+      --l2.jwt-secret=/blast/jwt.txt
+      --rollup.config="/blast/rollup.json"
+      --metrics.enabled
+    depends_on:
+      - blast-geth
+    env_file: 
+      - ../config/mainnet-blast/${NETWORK:-mainnet}.config
+
+volumes:
+  blast-data:
@@ -0,0 +1,76 @@
+version: '3.2'
+
+services:
+  ajna-watcher-db:
+    restart: unless-stopped
+    image: postgres:14-alpine
+    environment:
+      - POSTGRES_USER=vdbm
+      - POSTGRES_MULTIPLE_DATABASES=ajna-watcher,ajna-watcher-job-queue
+      - POSTGRES_EXTENSION=ajna-watcher-job-queue:pgcrypto
+      - POSTGRES_PASSWORD=password
+    volumes:
+      - ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
+      - ajna_watcher_db_data:/var/lib/postgresql/data
+    ports:
+      - "5432"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 10s
+
+  ajna-watcher-job-runner:
+    restart: unless-stopped
+    depends_on:
+      ajna-watcher-db:
+        condition: service_healthy
+    image: cerc/watcher-ajna:local
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+    command: ["bash", "./start-job-runner.sh"]
+    volumes:
+      - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
+      - ../config/watcher-ajna/start-job-runner.sh:/app/start-job-runner.sh
+    ports:
+      - "9000"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  ajna-watcher-server:
+    restart: unless-stopped
+    depends_on:
+      ajna-watcher-db:
+        condition: service_healthy
+      ajna-watcher-job-runner:
+        condition: service_healthy
+    image: cerc/watcher-ajna:local
+    environment:
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
+      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+    command: ["bash", "./start-server.sh"]
+    volumes:
+      - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
+      - ../config/watcher-ajna/start-server.sh:/app/start-server.sh
+    ports:
+      - "3008"
+      - "9001"
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      interval: 20s
+      timeout: 5s
+      retries: 15
+      start_period: 5s
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+volumes:
+  ajna_watcher_db_data:
@@ -0,0 +1,2 @@
+GETH_ROLLUP_SEQUENCERHTTP=https://sequencer.s2.testblast.io
+OP_NODE_P2P_BOOTNODES=enr:-J-4QM3GLUFfKMSJQuP1UvuKQe8DyovE7Eaiit0l6By4zjTodkR4V8NWXJxNmlg8t8rP-Q-wp3jVmeAOml8cjMj__ROGAYznzb_HgmlkgnY0gmlwhA-cZ_eHb3BzdGFja4X947FQAIlzZWNwMjU2azGhAiuDqvB-AsVSRmnnWr6OHfjgY8YfNclFy9p02flKzXnOg3RjcIJ2YYN1ZHCCdmE,enr:-J-4QDCVpByqQ8nFqCS9aHicqwUfXgzFDslvpEyYz19lvkHLIdtcIGp2d4q5dxHdjRNTO6HXCsnIKxUeuZSPcEbyVQCGAYznzz0RgmlkgnY0gmlwhANiQfuHb3BzdGFja4X947FQAIlzZWNwMjU2azGhAy3AtF2Jh_aPdOohg506Hjmtx-fQ1AKmu71C7PfkWAw9g3RjcIJ2YYN1ZHCCdmE
@@ -0,0 +1,57 @@
+{
+  "config": {
+    "chainId": 608943043,
+    "homesteadBlock": 0,
+    "eip150Block": 0,
+    "eip155Block": 0,
+    "eip158Block": 0,
+    "byzantiumBlock": 0,
+    "constantinopleBlock": 0,
+    "petersburgBlock": 0,
+    "istanbulBlock": 0,
+    "muirGlacierBlock": 0,
+    "berlinBlock": 0,
+    "londonBlock": 0,
+    "arrowGlacierBlock": 0,
+    "grayGlacierBlock": 0,
+    "mergeNetsplitBlock": 0,
+    "shanghaiTime": 0,
+    "bedrockBlock": 0,
+    "regolithTime": 0,
+    "canyonTime": 0,
+    "terminalTotalDifficulty": 0,
+    "terminalTotalDifficultyPassed": true,
+    "optimism": {
+      "eip1559Elasticity": 6,
+      "eip1559Denominator": 50,
+      "eip1559DenominatorCanyon": 250
+    }
+  },
+  "alloc": {
+    "0000000000000000000000000000000000000000": {
+      "balance": "0x1"
+    },
+    "4200000000000000000000000000000000000000": {
+      "code": "0x60806040526004361061004e5760003560e01c80633659cfe6146100655780634f1ef286146100855780635c60da1b146100ae5780638f283970146100db578063f851a440146100fb5761005d565b3661005d5761005b610110565b005b61005b610110565b34801561007157600080fd5b5061005b610080366004610521565b6101c8565b61009861009336600461053c565b61020e565b6040516100a591906105bf565b60405180910390f35b3480156100ba57600080fd5b506100c361033e565b6040516001600160a01b0390911681526020016100a5565b3480156100e757600080fd5b5061005b6100f6366004610521565b6103a9565b34801561010757600080fd5b506100c36103e4565b600061013a7f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc5490565b90506001600160a01b0381166101a55760405162461bcd60e51b815260206004820152602560248201527f50726f78793a20696d706c656d656e746174696f6e206e6f7420696e697469616044820152641b1a5e995960da1b60648201526084015b60405180910390fd5b3660008037600080366000845af43d6000803e806101c2573d6000fd5b503d6000f35b600080516020610625833981519152546001600160a01b0316336001600160a01b031614806101f5575033155b156102065761020381610432565b50565b610203610110565b60606102266000805160206106258339815191525490565b6001600160a01b0316336001600160a01b03161480610243575033155b1561032f5761025184610432565b600080856001600160a01b0316858560405161026e929190610614565b600060405180830381855af49150503d80600081146102a9576040519150601f19603f3d011682016040523d82523d6000602084013e6102ae565b606091505b5091509150816103265760405162461bcd60e51b815260206004820152603960248201527f50726f78793a2064656c656761746563616c6c20746f206e657720696d706c6560448201527f6d656e746174696f6e20636f6e7472616374206661696c656400000000000000606482015260840161019c565b91506103379050565b610337610110565b9392505050565b60006103566000805160206106258339815191525490565b6001600160a01b0316336001600160a01b03161480610373575033155b1561039e57507f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc5490565b6103a6610110565b90565b600080516020610625833981519152546001600160a01b0316336001600160a01b031614806103d6575033155b15610206576102038161048e565b60006103fc6000805160206106258339815191525490565b6001600160a01b0316336001600160a01b03161480610419575033155b1561039e57506000805160206106258339815191525490565b7f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc8181556040516001600160a01b038316907fbc7cd75a20ee27fd9adebab32041f755214dbc6bffa90cc0225b39da2e5c2d3b90600090a25050565b60006104a66000805160206106258339815191525490565b600080516020610625833981519152838155604080516001600160a01b0380851682528616602082015292935090917f7e644d79422f17c01e4894b5f4f588d331ebfa28653d42ae832dc59e38c9798f910160405180910390a1505050565b80356001600160a01b038116811461051c57600080fd5b919050565b60006020828403121561053357600080fd5b61033782610505565b60008060006040848603121561055157600080fd5b61055a84610505565b9250602084013567ffffffffffffffff8082111561057757600080fd5b818601915086601f83011261058b57600080fd5b81358181111561059a57600080fd5b8760208285010111156105ac57600080fd5b6020830194508093505050509250925092565b600060208083528351808285015260005b818110156105ec578581018301518582016040015282016105d0565b818111156105fe576000604083870101525b50601f01601f1916929092016040019392505050565b818382376000910190815291905056feb53127684a568b3173ae13b9f8a6016e243e63b6e8ee1178d6a717850b5d6103a164736f6c634300080f000a",
+      "storage": {
+        "0x360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc": "0x000000000000000000000000c0d3c0d3c0d3c0d3c0d3c0d3c0d3c0d3c0d30000",
+        "0xb53127684a568b3173ae13b9f8a6016e243e63b6e8ee1178d6a717850b5d6103": "0x0000000000000000000000004200000000000000000000000000000000000018"
+      },
+      "balance": "0x0",
+      "flags": 1
+    }
+  },
+  "nonce": "0x0",
+  "timestamp": "0x659b7460",
+  "extraData": "0x424544524f434b",
+  "gasLimit": "0x1c9c380",
+  "difficulty": "0x0",
+  "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+  "coinbase": "0x4200000000000000000000000000000000000011",
+  "number": "0x0",
+  "gasUsed": "0x0",
+  "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+  "baseFeePerGas": "0x3b9aca00",
+  "excessBlobGas": null,
+  "blobGasUsed": null
+}
@@ -0,0 +1,31 @@
+{
+  "genesis": {
+    "l1": {
+      "hash": "0x17728cf4d8e0b4f292d2390a869fd7c632d39e72efb00ca3462b4387c6aa2437",
+      "number": 5044255
+    },
+    "l2": {
+      "hash": "0x26a1c0faad7b041f34569a1bb383f00ab74b335883a44bed53e9f41ced5fd906",
+      "number": 0
+    },
+    "l2_time": 1704686688,
+    "system_config": {
+      "batcherAddr": "0xba26fee2fa917443e05e65de8d4350bcd2f59222",
+      "overhead": "0x00000000000000000000000000000000000000000000000000000000000000bc",
+      "scalar": "0x00000000000000000000000000000000000000000000000000000000000a6fe0",
+      "gasLimit": 30000000
+    }
+  },
+  "block_time": 2,
+  "max_sequencer_drift": 600,
+  "seq_window_size": 3600,
+  "channel_timeout": 300,
+  "l1_chain_id": 11155111,
+  "l2_chain_id": 608943043,
+  "regolith_time": 0,
+  "canyon_time": 0,
+  "batch_inbox_address": "0x1c3b85a2108784eab6a4bf56cdd6f722e415b331",
+  "deposit_contract_address": "0x2757e4430e694f27b73ec9c02257cab3a498c8c5",
+  "l1_system_config_address": "0x329faf078c364a316e08bf6a17b7eee6ae75a613",
+  "protocol_versions_address": "0x0000000000000000000000000000000000000000"
+}
@@ -0,0 +1,32 @@
+POSTGRES_DB=keycloak
+POSTGRES_USER=keycloak
+POSTGRES_PASSWORD=keycloak
+# Don't change this unless you also change the healthcheck in docker-compose-mainnet-eth-keycloak.yml
+PGPORT=35432
+KC_DB=postgres
+KC_DB_URL_HOST=keycloak-db
+KC_DB_URL_PORT=${PGPORT}
+KC_DB_URL_DATABASE=${POSTGRES_DB}
+KC_DB_USERNAME=${POSTGRES_USER}
+KC_DB_PASSWORD=${POSTGRES_PASSWORD}
+KC_DB_SCHEMA=public
+KC_HOSTNAME=localhost
+KC_HTTP_ENABLED="true"
+KC_HTTP_RELATIVE_PATH="/auth"
+KC_HOSTNAME_STRICT_HTTPS="false"
+KEYCLOAK_ADMIN=admin
+KEYCLOAK_ADMIN_PASSWORD=admin
+X_API_CHECK_REALM=cerc
+X_API_CHECK_CLIENT_ID="%user_id%"
+
+
+# keycloak-reg-api
+CERC_KCUSERREG_LISTEN_PORT=9292
+CERC_KCUSERREG_LISTEN_ADDR='0.0.0.0'
+CERC_KCUSERREG_API_URL='http://keycloak:8080/auth'
+CERC_KCUSERREG_REG_USER="${KEYCLOAK_ADMIN}"
+CERC_KCUSERREG_REG_PW="${KEYCLOAK_ADMIN_PASSWORD}"
+CERC_KCUSERREG_REG_CLIENT_ID='admin-cli'
+CERC_KCUSERREG_TARGET_REALM=cerc
+CERC_KCUSERREG_TARGET_GROUPS=eth
+CERC_KCUSERREG_CREATE_ENABLED=true
@@ -0,0 +1,33 @@
+# Enable startup script debug output.
+CERC_SCRIPT_DEBUG=false
+
+# Specify any other lighthouse CLI options.
+LIGHTHOUSE_OPTS=""
+
+# Override the advertised public IP (optional)
+# --enr-address
+#LIGHTHOUSE_ENR_ADDRESS=""
+
+# --checkpoint-sync-url
+LIGHTHOUSE_CHECKPOINT_SYNC_URL="https://beaconstate.ethstaker.cc"
+
+# --checkpoint-sync-url-timeout
+LIGHTHOUSE_CHECKPOINT_SYNC_URL_TIMEOUT=300
+
+# --datadir
+LIGHTHOUSE_DATADIR=/data
+
+# --debug-level
+LIGHTHOUSE_DEBUG_LEVEL=info
+
+# --http-port
+LIGHTHOUSE_HTTP_PORT=5052
+
+# --execution-jwt
+LIGHTHOUSE_JWTSECRET=/etc/mainnet-eth/jwtsecret
+
+# --metrics-port
+LIGHTHOUSE_METRICS_PORT=5054
+
+# --port --enr-udp-port --enr-tcp-port
+LIGHTHOUSE_NETWORK_PORT=9000
@@ -0,0 +1,2 @@
+GETH_ROLLUP_SEQUENCERHTTP=https://sequencer.blast.io
+OP_NODE_P2P_BOOTNODES=enr:-J64QGwHl9uYLfC_cnmxSA6wQH811nkOWJDWjzxqkEUlJoZHWvI66u-BXgVcPCeMUmg0dBpFQAPotFchG67FHJMZ9OSGAY3d6wevgmlkgnY0gmlwhANizeSHb3BzdGFja4Sx_AQAiXNlY3AyNTZrMaECg4pk0cskPAyJ7pOmo9E6RqGBwV-Lex4VS9a3MQvu7PWDdGNwgnZhg3VkcIJ2YQ,enr:-J64QDge2jYBQtcNEpRqmKfci5E5BHAhNBjgv4WSdwH1_wPqbueq2bDj38-TSW8asjy5lJj1Xftui6Or8lnaYFCqCI-GAY3d6wf3gmlkgnY0gmlwhCO2D9yHb3BzdGFja4Sx_AQAiXNlY3AyNTZrMaEDo4aCTq7pCEN8om9U5n_VyWdambGnQhwHNwKc8o-OicaDdGNwgnZhg3VkcIJ2YQ
@@ -0,0 +1,32 @@
+{
+    "genesis": {
+      "l1": {
+        "hash": "0xfcfb8d586bdae763f1189988789211c69eb893a895e7ba48be3ca6289f0941b7",
+        "number": 19300102
+      },
+      "l2": {
+        "hash": "0xb689b35ef29d0bec5816938e0e52683c7257d2e325420ea69b739a2be4754b89",
+        "number": 0
+      },
+      "l2_time": 1708809815,
+      "system_config": {
+        "batcherAddr": "0x415c8893d514f9bc5211d36eeda4183226b84aa7",
+        "overhead": "0x00000000000000000000000000000000000000000000000000000000000000bc",
+        "scalar": "0x00000000000000000000000000000000000000000000000000000000000a6fe0",
+        "gasLimit": 30000000
+      }
+    },
+    "block_time": 2,
+    "max_sequencer_drift": 600,
+    "seq_window_size": 3600,
+    "channel_timeout": 300,
+    "l1_chain_id": 1,
+    "l2_chain_id": 81457,
+    "regolith_time": 0,
+    "canyon_time": 0,
+    "batch_inbox_address": "0xff00000000000000000000000000000000081457",
+    "deposit_contract_address": "0x0ec68c5b10f21effb74f2a5c61dfe6b08c0db6cb",
+    "l1_system_config_address": "0x5531dcff39ec1ec727c4c5d2fc49835368f805a9",
+    "protocol_versions_address": "0x0000000000000000000000000000000000000000"
+  }
+  
@@ -0,0 +1,30 @@
+#!/bin/bash
+if [[ "true" == "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi
+
+ENR_OPTS=""
+if [[ -n "$LIGHTHOUSE_ENR_ADDRESS" ]]; then
+  ENR_OPTS="--enr-address $LIGHTHOUSE_ENR_ADDRESS"
+fi
+
+exec lighthouse bn \
+  --checkpoint-sync-url "$LIGHTHOUSE_CHECKPOINT_SYNC_URL" \
+  --checkpoint-sync-url-timeout ${LIGHTHOUSE_CHECKPOINT_SYNC_URL_TIMEOUT} \
+  --datadir "$LIGHTHOUSE_DATADIR" \
+  --debug-level $LIGHTHOUSE_DEBUG_LEVEL \
+  --disable-deposit-contract-sync \
+  --disable-upnp \
+  --enr-tcp-port $LIGHTHOUSE_NETWORK_PORT \
+  --enr-udp-port $LIGHTHOUSE_NETWORK_PORT \
+  --execution-endpoint "$LIGHTHOUSE_EXECUTION_ENDPOINT" \
+  --execution-jwt /etc/mainnet-eth/jwtsecret \
+  --http \
+  --http-address 0.0.0.0 \
+  --http-port $LIGHTHOUSE_HTTP_PORT \
+  --metrics \
+  --metrics-address=0.0.0.0 \
+  --metrics-port $LIGHTHOUSE_METRICS_PORT \
+  --network mainnet \
+  --port $LIGHTHOUSE_NETWORK_PORT \
+  $ENR_OPTS $LIGHTHOUSE_OPTS
@@ -50,22 +50,6 @@ groups:
              legendFormat: __auto
              range: false
              refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
          - refId: condition
            relativeTimeRange:
              from: 600
@@ -142,22 +126,6 @@ groups:
              legendFormat: __auto
              range: false
              refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
          - refId: condition
            relativeTimeRange:
              from: 600
@@ -234,22 +202,6 @@ groups:
              legendFormat: __auto
              range: false
              refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
          - refId: condition
            relativeTimeRange:
              from: 600
@@ -326,22 +278,6 @@ groups:
              legendFormat: __auto
              range: false
              refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
          - refId: condition
            relativeTimeRange:
              from: 600
@@ -418,22 +354,6 @@ groups:
              legendFormat: __auto
              range: false
              refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
          - refId: condition
            relativeTimeRange:
              from: 600
@@ -510,22 +430,6 @@ groups:
              legendFormat: __auto
              range: false
              refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
          - refId: condition
            relativeTimeRange:
              from: 600
@@ -602,22 +506,6 @@ groups:
              legendFormat: __auto
              range: false
              refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
          - refId: condition
            relativeTimeRange:
              from: 600
@@ -694,22 +582,6 @@ groups:
              legendFormat: __auto
              range: false
              refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
          - refId: condition
            relativeTimeRange:
              from: 600
@@ -788,22 +660,6 @@ groups:
              legendFormat: __auto
              range: false
              refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
          - refId: condition
            relativeTimeRange:
              from: 600
@@ -880,22 +736,6 @@ groups:
              legendFormat: __auto
              range: false
              refId: latest_external
-          - refId: latest_indexed
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_indexed
          - refId: condition
            relativeTimeRange:
              from: 600
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+set -u
+
+echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+
+# Read in the config template TOML file and modify it
+WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+
+# Write the modified content to a new file
+echo "$WATCHER_CONFIG" > environments/local.toml
+
+echo "Running job-runner..."
+DEBUG=vulcanize:* exec node --enable-source-maps dist/job-runner.js
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+set -u
+
+echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+
+# Read in the config template TOML file and modify it
+WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+
+# Write the modified content to a new file
+echo "$WATCHER_CONFIG" > environments/local.toml
+
+echo "Running server..."
+DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js
@@ -0,0 +1,98 @@
+[server]
+  host = "0.0.0.0"
+  port = 3008
+  kind = "active"
+  gqlPath = "/"
+
+  # Checkpointing state.
+  checkpointing = true
+
+  # Checkpoint interval in number of blocks.
+  checkpointInterval = 2000
+
+  # Enable state creation
+  # CAUTION: Disable only if state creation is not desired or can be filled subsequently
+  enableState = false
+
+  subgraphPath = "./subgraph-build"
+
+  # Interval to restart wasm instance periodically
+  wasmRestartBlocksInterval = 20
+
+  # Interval in number of blocks at which to clear entities cache.
+  clearEntitiesCacheInterval = 1000
+
+  # Max block range for which to return events in eventsInRange GQL query.
+  # Use -1 for skipping check on block range.
+  maxEventsBlockRange = 1000
+
+  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
+  rpcSupportsBlockHashParam = false
+
+  # GQL cache settings
+  [server.gqlCache]
+    enabled = true
+
+    # Max in-memory cache size (in bytes) (default 8 MB)
+    # maxCacheSize
+
+    # GQL cache-control max-age settings (in seconds)
+    maxAge = 15
+    timeTravelMaxAge = 86400 # 1 day
+
+[metrics]
+  host = "0.0.0.0"
+  port = 9000
+  [metrics.gql]
+    port = 9001
+
+[database]
+  type = "postgres"
+  host = "ajna-watcher-db"
+  port = 5432
+  database = "ajna-watcher"
+  username = "vdbm"
+  password = "password"
+  synchronize = true
+  logging = false
+
+[upstream]
+  [upstream.ethServer]
+    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+
+    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
+    rpcClient = true
+
+    # Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint
+    isFEVM = true
+
+    # Boolean flag to filter event logs by contracts
+    filterLogsByAddresses = true
+    # Boolean flag to filter event logs by topics
+    filterLogsByTopics = true
+
+  [upstream.cache]
+    name = "requests"
+    enabled = false
+    deleteOnStart = false
+
+[jobQueue]
+  dbConnectionString = "postgres://vdbm:password@ajna-watcher-db/ajna-watcher-job-queue"
+  maxCompletionLagInSecs = 300
+  jobDelayInMilliSecs = 100
+  eventsInBatch = 50
+  subgraphEventsOrder = true
+  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
+  blockDelayInMilliSecs = 30000
+
+  # Boolean to switch between modes of processing events when starting the server.
+  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
+  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
+  useBlockRanges = true
+
+  # Block range in which logs are fetched during historical blocks processing
+  historicalLogsBlockRange = 2000
+
+  # Max block range of historical processing after which it waits for completion of events processing
+  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
+  historicalMaxFetchAhead = 10000
@@ -11,8 +11,14 @@ CERC_CONTAINER_BUILD_DOCKERFILE=${CERC_CONTAINER_BUILD_DOCKERFILE:-$SCRIPT_DIR/D
 CERC_CONTAINER_BUILD_TAG=${CERC_CONTAINER_BUILD_TAG:-cerc/nextjs-base:local}

 docker build -t $CERC_CONTAINER_BUILD_TAG ${build_command_args} -f $CERC_CONTAINER_BUILD_DOCKERFILE $CERC_CONTAINER_BUILD_WORK_DIR
+rc=$?

-if [ $? -eq 0 ] && [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/nextjs-base:local" ]; then
+if [ $rc -ne 0 ]; then
+  echo "BUILD FAILED" 1>&2
+  exit $rc
+fi
+
+if [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/nextjs-base:local" ]; then
  cat <<EOF

 #################################################################
@@ -115,7 +115,7 @@ if [ "$CERC_NEXT_VERSION" != "keep" ] && [ "$CUR_NEXT_VERSION" != "$CERC_NEXT_VE
  mv package.json.$$ package.json
 fi

-$CERC_BUILD_TOOL install || exit 1
+time $CERC_BUILD_TOOL install || exit 1

 CUR_NEXT_VERSION=`jq -r '.version' node_modules/next/package.json`

@@ -138,9 +138,9 @@ to use for the build with:
 EOF
  cat package.json | jq ".dependencies.next = \"^$CERC_MIN_NEXTVER\"" > package.json.$$
  mv package.json.$$ package.json
-  $CERC_BUILD_TOOL install || exit 1
+  time $CERC_BUILD_TOOL install || exit 1
 fi

-$CERC_BUILD_TOOL run cerc_compile || exit 1
+time $CERC_BUILD_TOOL run cerc_compile || exit 1

 exit 0
@@ -0,0 +1,10 @@
+FROM node:18.17.1-alpine3.18
+
+RUN apk --update --no-cache add git python3 alpine-sdk bash curl jq
+
+WORKDIR /app
+
+COPY . .
+
+RUN echo "Installing dependencies and building ajna-watcher-ts" && \
+  yarn && yarn build
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# Build cerc/watcher-ajna
+
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/watcher-ajna:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/ajna-watcher-ts
@@ -1,6 +1,6 @@
 # Originally from: https://github.com/devcontainers/images/blob/main/src/javascript-node/.devcontainer/Dockerfile
 # [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
-ARG VARIANT=20-bullseye
+ARG VARIANT=20-bullseye-slim
 FROM node:${VARIANT}

 ARG USERNAME=node
@@ -11,8 +11,14 @@ CERC_CONTAINER_BUILD_DOCKERFILE=${CERC_CONTAINER_BUILD_DOCKERFILE:-$SCRIPT_DIR/D
 CERC_CONTAINER_BUILD_TAG=${CERC_CONTAINER_BUILD_TAG:-cerc/webapp-base:local}

 docker build -t $CERC_CONTAINER_BUILD_TAG ${build_command_args} -f $CERC_CONTAINER_BUILD_DOCKERFILE $CERC_CONTAINER_BUILD_WORK_DIR
+rc=$?

-if [ $? -eq 0 ] && [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/webapp-base:local" ]; then
+if [ $rc -ne 0 ]; then
+  echo "BUILD FAILED" 1>&2
+  exit $rc
+fi
+
+if [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/webapp-base:local" ]; then
  cat <<EOF

 #################################################################
@@ -32,8 +32,8 @@ elif [ -f "${WORK_DIR}/package.json" ]; then
    fi
  fi

-  $CERC_BUILD_TOOL install || exit 1
-  $CERC_BUILD_TOOL build || exit 1
+  time $CERC_BUILD_TOOL install || exit 1
+  time $CERC_BUILD_TOOL build || exit 1

  rm -rf "${DEST_DIR}"
  if [ -z "${CERC_BUILD_OUTPUT_DIR}" ]; then
@@ -0,0 +1,118 @@
+# Ajna Watcher
+
+## Setup
+
+Clone required repositories:
+
+```bash
+laconic-so --stack ajna setup-repositories --git-ssh --pull
+```
+
+Build the container images:
+
+```bash
+laconic-so --stack ajna build-containers
+```
+
+## Deploy
+
+Create a spec file for the deployment:
+
+```bash
+laconic-so --stack ajna deploy init --output ajna-spec.yml
+```
+
+### Ports
+
+Edit `network` in the spec file to map container ports to host ports as required:
+
+```yml
+...
+network:
+  ports:
+    ajna-watcher-db:
+     - 15432:5432
+    ajna-watcher-job-runner:
+     - 9000:9000
+    ajna-watcher-server:
+     - 3008:3008
+     - 9001:9001
+```
+
+### Create a deployment
+
+Create a deployment from the spec file:
+
+```bash
+laconic-so --stack ajna deploy create --spec-file ajna-spec.yml --deployment-dir ajna-deployment
+```
+
+### Configuration
+
+Inside deployment directory, open the `config.env` file  and set following env variables:
+
+```bash
+# External Filecoin (ETH RPC) endpoint to point the watcher to
+CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
+```
+
+### Start the deployment
+
+```bash
+laconic-so deployment --dir ajna-deployment start
+```
+
+* To list down and monitor the running containers:
+
+  ```bash
+  # With status
+  docker ps -a
+
+  # Check logs for a container
+  docker logs -f <CONTAINER_ID>
+  ```
+
+* Open the GQL playground at <http://localhost:3008/graphql>
+
+  ```graphql
+  # Example query
+  query {
+    _meta {
+      block {
+        hash
+        number
+        timestamp
+      }
+      deployment
+      hasIndexingErrors
+    }
+
+    accounts {
+      id
+      txCount
+      tokensDelegated
+      rewardsClaimed
+    }
+  }
+  ```
+
+## Clean up
+
+Stop all the ajna services running in background:
+
+```bash
+# Only stop the docker containers
+laconic-so deployment --dir ajna-deployment stop
+
+# Run 'start' to restart the deployment
+```
+
+To stop all the ajna services and also delete data:
+
+```bash
+# Stop the docker containers
+laconic-so deployment --dir ajna-deployment stop --delete-volumes
+
+# Remove deployment directory (deployment will have to be recreated for a re-run)
+rm -r ajna-deployment
+```
@@ -0,0 +1,9 @@
+version: "1.0"
+name: ajna
+description: "Ajna watcher stack"
+repos:
+  - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.1
+containers:
+  - cerc/watcher-ajna
+pods:
+  - watcher-ajna
@@ -0,0 +1,26 @@
+# Blast stack
+
+## Clone required repositories
+```
+$ laconic-so --stack fixturenet-blast setup-repositories
+```
+## Build the stack's containers
+```
+$ laconic-so --stack fixturenet-blast build-containers
+```
+## Create a deployment of the stack
+```
+$ laconic-so --stack fixturenet-blast deploy init --map-ports-to-host any-same --output blast-spec.yml
+```
+[Insert details on how to configure the stack]
+```
+$ laconic-so --stack fixturenet-blast deploy create --deployment-dir blast-deployment --spec-file blast-spec.yml
+```
+## Start the stack
+```
+$ laconic-so deployment --dir blast-deployment start
+```
+Check logs:
+```
+$ laconic-so deployment --dir blast-deployment logs
+```
@@ -0,0 +1,17 @@
+version: "1.0"
+name: fixturenet-blast
+description: "A blast devnet stack"
+repos:
+  - github.com/blast-io/blast
+  - git.vdb.to/cerc-io/lighthouse
+containers:
+  - cerc/webapp-base
+  - cerc/lighthouse
+  - cerc/lighthouse-cli
+  - cerc/foundry
+  - cerc/fixturenet-eth-lighthouse
+
+pods:
+  - fixturenet-blast
+  - foundry
+  
@@ -0,0 +1,26 @@
+# Blast stack
+
+## Clone required repositories
+```
+$ laconic-so --stack mainnet-blast setup-repositories
+```
+## Build the stack's containers
+```
+$ laconic-so --stack mainnet-blast build-containers
+```
+## Create a deployment of the stack
+```
+$ laconic-so --stack mainnet-blast deploy init --map-ports-to-host any-same --output blast-spec.yml
+```
+[Insert details on how to configure the stack]
+```
+$ laconic-so --stack mainnet-blast deploy create --deployment-dir blast-deployment --spec-file blast-spec.yml
+```
+## Start the stack
+```
+$ laconic-so deployment --dir blast-deployment start
+```
+Check logs:
+```
+$ laconic-so deployment --dir blast-deployment logs
+```
@@ -0,0 +1,39 @@
+# Copyright © 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+
+from pathlib import Path
+from shutil import copy
+import yaml
+
+
+def create(context, extra_args):
+    # Our goal here is just to copy the json files for blast
+    yml_path = context.deployment_dir.joinpath("spec.yml")
+    with open(yml_path, 'r') as file:
+        data = yaml.safe_load(file)
+
+    mount_point = data['volumes']['blast-data']
+    if mount_point[0] == "/":
+        deploy_dir = Path(mount_point)
+    else:
+        deploy_dir = context.deployment_dir.joinpath(mount_point)
+
+    command_context = extra_args[2]
+    compose_file = [f for f in command_context.cluster_context.compose_files if "mainnet-blast" in f][0]
+    source_config_file = Path(compose_file).parent.parent.joinpath("config", "mainnet-blast", "genesis.json")
+    copy(source_config_file, deploy_dir)
+    source_config_file = Path(compose_file).parent.parent.joinpath("config", "mainnet-blast", "rollup.json")
+    copy(source_config_file, deploy_dir)
@@ -0,0 +1,12 @@
+version: "1.0"
+name: mainnet-blast
+description: "A blast stack"
+repos:
+  - github.com/blast-io/blast
+  - git.vdb.to/cerc-io/lighthouse
+containers:
+  - cerc/webapp-base
+  - cerc/lighthouse
+  - cerc/lighthouse-cli
+pods:
+  - mainnet-blast
@@ -16,26 +16,55 @@ laconic-so --stack merkl-sushiswap-v3 build-containers

 ## Deploy

-### Configuration
-
-Create and update an env file to be used in the next step:
-
-  ```bash
-  # External Filecoin (ETH RPC) endpoint to point the watcher
-  CERC_ETH_RPC_ENDPOINT=
-  ```
-
-### Deploy the stack
+Create a spec file for the deployment:

 ```bash
-laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 --env-file <PATH_TO_ENV_FILE> up
+laconic-so --stack merkl-sushiswap-v3 deploy init --output merkl-sushiswap-v3-spec.yml
+```
+
+### Ports
+
+Edit `network` in the spec file to map container ports to host ports as required:
+
+```
+...
+network:
+  ports:
+    merkl-sushiswap-v3-watcher-db:
+     - '5432'
+    merkl-sushiswap-v3-watcher-job-runner:
+     - 9002:9000
+    merkl-sushiswap-v3-watcher-server:
+     - 127.0.0.1:3007:3008
+     - 9003:9001
+```
+
+### Create a deployment
+
+Create a deployment from the spec file:
+
+```bash
+laconic-so --stack merkl-sushiswap-v3 deploy create --spec-file merkl-sushiswap-v3-spec.yml --deployment-dir merkl-sushiswap-v3-deployment
+```
+
+### Configuration
+
+Inside deployment directory, open the `config.env` file  and set following env variables:
+
+```bash
+# External Filecoin (ETH RPC) endpoint to point the watcher to
+CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
+```
+
+### Start the deployment
+
+```bash
+laconic-so deployment --dir merkl-sushiswap-v3-deployment start
 ```

 * To list down and monitor the running containers:

  ```bash
-  laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 ps
-
  # With status
  docker ps -a

@@ -46,6 +75,7 @@ laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 --env-
 * Open the GQL playground at http://localhost:3007/graphql

  ```graphql
+  # Example query
  {
    _meta {
      block {
@@ -54,7 +84,7 @@ laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 --env-
      }
      hasIndexingErrors
    }
-    
+
    factories {
      id
      poolCount
@@ -64,18 +94,21 @@ laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 --env-

 ## Clean up

-Stop all the services running in background:
+Stop all the merkl-sushiswap-v3 services running in background:

 ```bash
-laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 down
+# Only stop the docker containers
+laconic-so deployment --dir merkl-sushiswap-v3-deployment stop
+
+# Run 'start' to restart the deployment
 ```

-Clear volumes created by this stack:
+To stop all the merkl-sushiswap-v3 services and also delete data:

 ```bash
-# List all relevant volumes
-docker volume ls -q --filter "name=merkl_sushiswap_v3"
+# Stop the docker containers
+laconic-so deployment --dir merkl-sushiswap-v3-deployment stop --delete-volumes

-# Remove all the listed volumes
-docker volume rm $(docker volume ls -q --filter "name=merkl_sushiswap_v3")
+# Remove deployment directory (deployment will have to be recreated for a re-run)
+rm -r merkl-sushiswap-v3-deployment
 ```
@@ -2,7 +2,7 @@ version: "1.0"
 name: merkl-sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.6
+  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.7
 containers:
  - cerc/watcher-merkl-sushiswap-v3
 pods:
@@ -1,7 +1,7 @@
 version: "0.1"
 name: monitoring
 repos:
-  - github.com/cerc-io/watcher-ts@v0.2.79
+  - github.com/cerc-io/watcher-ts@v0.2.81
 containers:
  - cerc/watcher-ts
 pods:
@@ -55,7 +55,7 @@ ports:
 Create deployment:

 ```bash
-laconic-so deploy create --spec-file sushiswap-subgraph-spec.yml --deployment-dir sushiswap-subgraph-deployment
+laconic-so --stack sushiswap-subgraph deploy create --spec-file sushiswap-subgraph-spec.yml --deployment-dir sushiswap-subgraph-deployment
 ```

 ## Start the stack
@@ -16,26 +16,55 @@ laconic-so --stack sushiswap-v3 build-containers

 ## Deploy

-### Configuration
-
-Create and update an env file to be used in the next step:
-
-  ```bash
-  # External Filecoin (ETH RPC) endpoint to point the watcher
-  CERC_ETH_RPC_ENDPOINT=
-  ```
-
-### Deploy the stack
+Create a spec file for the deployment:

 ```bash
-laconic-so --stack sushiswap-v3 deploy --cluster sushiswap_v3 --env-file <PATH_TO_ENV_FILE> up
+laconic-so --stack sushiswap-v3 deploy init --output sushiswap-v3-spec.yml
+```
+
+### Ports
+
+Edit `network` in the spec file to map container ports to host ports as required:
+
+```
+...
+network:
+  ports:
+    sushiswap-v3-watcher-db:
+     - '5432'
+    sushiswap-v3-watcher-job-runner:
+     - 9000:9000
+    sushiswap-v3-watcher-server:
+     - 127.0.0.1:3008:3008
+     - 9001:9001
+```
+
+### Create a deployment
+
+Create a deployment from the spec file:
+
+```bash
+laconic-so --stack sushiswap-v3 deploy create --spec-file sushiswap-v3-spec.yml --deployment-dir sushiswap-v3-deployment
+```
+
+### Configuration
+
+Inside deployment directory, open the `config.env` file  and set following env variables:
+
+```bash
+# External Filecoin (ETH RPC) endpoint to point the watcher to
+CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
+```
+
+### Start the deployment
+
+```bash
+laconic-so deployment --dir sushiswap-v3-deployment start
 ```

 * To list down and monitor the running containers:

  ```bash
-  laconic-so --stack sushiswap-v3 deploy --cluster sushiswap_v3 ps
-
  # With status
  docker ps -a

@@ -43,20 +72,43 @@ laconic-so --stack sushiswap-v3 deploy --cluster sushiswap_v3 --env-file <PATH_T
  docker logs -f <CONTAINER_ID>
  ```

+* Open the GQL playground at http://localhost:3008/graphql
+
+  ```graphql
+  # Example query
+  {
+    _meta {
+      block {
+        number
+        timestamp
+      }
+      hasIndexingErrors
+    }
+
+    factories {
+      id
+      poolCount
+    }
+  }
+  ```
+
 ## Clean up

-Stop all the services running in background:
+Stop all the sushiswap-v3 services running in background:

 ```bash
-laconic-so --stack sushiswap-v3 deploy --cluster sushiswap_v3 down
+# Only stop the docker containers
+laconic-so deployment --dir sushiswap-v3-deployment stop
+
+# Run 'start' to restart the deployment
 ```

-Clear volumes created by this stack:
+To stop all the sushiswap-v3 services and also delete data:

 ```bash
-# List all relevant volumes
-docker volume ls -q --filter "name=sushiswap_v3"
+# Stop the docker containers
+laconic-so deployment --dir sushiswap-v3-deployment stop --delete-volumes

-# Remove all the listed volumes
-docker volume rm $(docker volume ls -q --filter "name=sushiswap_v3")
+# Remove deployment directory (deployment will have to be recreated for a re-run)
+rm -r sushiswap-v3-deployment
 ```
@@ -2,7 +2,7 @@ version: "1.0"
 name: sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.6
+  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.7
 containers:
  - cerc/watcher-sushiswap-v3
 pods:
@@ -101,7 +101,7 @@ class ClusterInfo:
        )
        return service

-    def get_ingress(self, use_tls=False):
+    def get_ingress(self, use_tls=False, certificate=None, cluster_issuer="letsencrypt-prod"):
        # No ingress for a deployment that has no http-proxy defined, for now
        http_proxy_info_list = self.spec.get_http_proxy()
        ingress = None
@@ -114,8 +114,8 @@ class ClusterInfo:
            host_name = http_proxy_info["host-name"]
            rules = []
            tls = [client.V1IngressTLS(
-                hosts=[host_name],
-                secret_name=f"{self.app_name}-tls"
+                hosts=certificate["spec"]["dnsNames"] if certificate else [host_name],
+                secret_name=certificate["spec"]["secretName"] if certificate else f"{self.app_name}-tls"
            )] if use_tls else None
            paths = []
            for route in http_proxy_info["routes"]:
@@ -147,13 +147,17 @@ class ClusterInfo:
                tls=tls,
                rules=rules
            )
+
+            ingress_annotations = {
+                "kubernetes.io/ingress.class": "nginx",
+            }
+            if not certificate:
+                ingress_annotations["cert-manager.io/cluster-issuer"] = cluster_issuer
+
            ingress = client.V1Ingress(
                metadata=client.V1ObjectMeta(
                    name=f"{self.app_name}-ingress",
-                    annotations={
-                        "kubernetes.io/ingress.class": "nginx",
-                        "cert-manager.io/cluster-issuer": "letsencrypt-prod"
-                    }
+                    annotations=ingress_annotations
                ),
                spec=spec
            )
@@ -169,6 +169,39 @@ class K8sDeployer(Deployer):
                print("Service created:")
                print(f"{service_resp}")

+    def _find_certificate_for_host_name(self, host_name):
+        all_certificates = self.custom_obj_api.list_namespaced_custom_object(
+            group="cert-manager.io",
+            version="v1",
+            namespace=self.k8s_namespace,
+            plural="certificates"
+        )
+
+        host_parts = host_name.split(".", 1)
+        host_as_wild = None
+        if len(host_parts) == 2:
+            host_as_wild = f"*.{host_parts[1]}"
+
+        now = datetime.utcnow().replace(tzinfo=timezone.utc)
+        fmt = "%Y-%m-%dT%H:%M:%S%z"
+
+        # Walk over all the configured certificates.
+        for cert in all_certificates["items"]:
+            dns = cert["spec"]["dnsNames"]
+            # Check for an exact hostname match or a wildcard match.
+            if host_name in dns or host_as_wild in dns:
+                status = cert.get("status", {})
+                # Check the certificate date.
+                if "notAfter" in status and "notBefore" in status:
+                    before = datetime.strptime(status["notBefore"], fmt)
+                    after = datetime.strptime(status["notAfter"], fmt)
+                    if before < now < after:
+                        # Check the status is Ready
+                        for condition in status.get("conditions", []):
+                            if "True" == condition.get("status") and "Ready" == condition.get("type"):
+                                return cert
+        return None
+
    def up(self, detach, services):
        if not opts.o.dry_run:
            if self.is_kind():
@@ -189,8 +222,15 @@ class K8sDeployer(Deployer):
        self._create_volume_data()
        self._create_deployment()

+        http_proxy_info = self.cluster_info.spec.get_http_proxy()
        # Note: at present we don't support tls for kind (and enabling tls causes errors)
-        ingress: client.V1Ingress = self.cluster_info.get_ingress(use_tls=not self.is_kind())
+        use_tls = http_proxy_info and not self.is_kind()
+        certificate = self._find_certificate_for_host_name(http_proxy_info[0]["host-name"]) if use_tls else None
+        if opts.o.debug:
+            if certificate:
+                print(f"Using existing certificate: {certificate}")
+
+        ingress: client.V1Ingress = self.cluster_info.get_ingress(use_tls=use_tls, certificate=certificate)
        if ingress:
            if opts.o.debug:
                print(f"Sending this ingress: {ingress}")
@@ -350,9 +390,11 @@ class K8sDeployer(Deployer):
                name=ingress.spec.tls[0].secret_name
            )

-            hostname = ingress.spec.tls[0].hosts[0]
+            hostname = ingress.spec.rules[0].host
            ip = ingress.status.load_balancer.ingress[0].ip
-            tls = "notBefore: %s, notAfter: %s" % (cert["status"]["notBefore"], cert["status"]["notAfter"])
+            tls = "notBefore: %s; notAfter: %s; names: %s" % (
+                cert["status"]["notBefore"], cert["status"]["notAfter"], ingress.spec.tls[0].hosts
+            )
        except:  # noqa: E722
            pass

@@ -27,7 +27,9 @@ class ResourceLimits:
    memory: int = None
    storage: int = None

-    def __init__(self, obj={}):
+    def __init__(self, obj=None):
+        if obj is None:
+            obj = {}
        if "cpus" in obj:
            self.cpus = float(obj["cpus"])
        if "memory" in obj:
@@ -50,7 +52,9 @@ class Resources:
    limits: ResourceLimits = None
    reservations: ResourceLimits = None

-    def __init__(self, obj={}):
+    def __init__(self, obj=None):
+        if obj is None:
+            obj = {}
        if "reservations" in obj:
            self.reservations = ResourceLimits(obj["reservations"])
        if "limits" in obj:
@@ -72,7 +76,9 @@ class Spec:
    obj: typing.Any
    file_path: Path

-    def __init__(self, file_path: Path = None, obj={}) -> None:
+    def __init__(self, file_path: Path = None, obj=None) -> None:
+        if obj is None:
+            obj = {}
        self.file_path = file_path
        self.obj = obj

@@ -91,49 +97,41 @@ class Spec:
            self.file_path = file_path

    def get_image_registry(self):
-        return (self.obj[constants.image_registry_key]
-                if self.obj and constants.image_registry_key in self.obj
-                else None)
+        return self.obj.get(constants.image_registry_key)

    def get_volumes(self):
-        return (self.obj["volumes"]
-                if self.obj and "volumes" in self.obj
-                else {})
+        return self.obj.get(constants.volumes_key, {})

    def get_configmaps(self):
-        return (self.obj["configmaps"]
-                if self.obj and "configmaps" in self.obj
-                else {})
+        return self.obj.get(constants.configmaps_key, {})

    def get_container_resources(self):
-        return Resources(self.obj.get("resources", {}).get("containers", {}))
+        return Resources(self.obj.get(constants.resources_key, {}).get("containers", {}))

    def get_volume_resources(self):
-        return Resources(self.obj.get("resources", {}).get("volumes", {}))
+        return Resources(self.obj.get(constants.resources_key, {}).get(constants.volumes_key, {}))

    def get_http_proxy(self):
-        return (self.obj[constants.network_key][constants.http_proxy_key]
-                if self.obj and constants.network_key in self.obj
-                and constants.http_proxy_key in self.obj[constants.network_key]
-                else None)
+        return self.obj.get(constants.network_key, {}).get(constants.http_proxy_key, [])

    def get_annotations(self):
-        return self.obj.get("annotations", {})
+        return self.obj.get(constants.annotations_key, {})

    def get_labels(self):
-        return self.obj.get("labels", {})
+        return self.obj.get(constants.labels_key, {})

    def get_privileged(self):
-        return "true" == str(self.obj.get("security", {}).get("privileged", "false")).lower()
+        return "true" == str(self.obj.get(constants.security_key, {}).get("privileged", "false")).lower()

    def get_capabilities(self):
-        return self.obj.get("security", {}).get("capabilities", [])
+        return self.obj.get(constants.security_key, {}).get("capabilities", [])

    def get_deployment_type(self):
-        return self.obj[constants.deploy_to_key]
+        return self.obj.get(constants.deploy_to_key)

    def is_kubernetes_deployment(self):
-        return self.get_deployment_type() in [constants.k8s_kind_deploy_type, constants.k8s_deploy_type]
+        return self.get_deployment_type() in [constants.k8s_kind_deploy_type,
+                                              constants.k8s_deploy_type]

    def is_kind_deployment(self):
        return self.get_deployment_type() in [constants.k8s_kind_deploy_type]
@@ -26,7 +26,7 @@ import click

 from stack_orchestrator.deploy.images import remote_image_exists, add_tags_to_image
 from stack_orchestrator.deploy.webapp import deploy_webapp
-from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient,
+from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient, TimedLogger,
                                                   build_container_image, push_container_image,
                                                   file_hash, deploy_to_k8s, publish_deployment,
                                                   hostname_for_deployment_request, generate_hostname_for_app,
@@ -34,7 +34,6 @@ from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient,


 def process_app_deployment_request(
-    run_id,
    ctx,
    laconic: LaconicRegistryClient,
    app_deployment_request,
@@ -44,14 +43,18 @@ def process_app_deployment_request(
    deployment_parent_dir,
    kube_config,
    image_registry,
-    force_rebuild=False,
-    log_file=None
+    force_rebuild,
+    logger
 ):
+    logger.log("BEGIN - process_app_deployment_request")
+
    # 1. look up application
    app = laconic.get_record(app_deployment_request.attributes.application, require=True)
+    logger.log(f"Retrieved app record {app_deployment_request.attributes.application}")

    # 2. determine dns
    requested_name = hostname_for_deployment_request(app_deployment_request, laconic)
+    logger.log(f"Determined requested name: {requested_name}")

    # HACK
    if "." in requested_name:
@@ -69,7 +72,7 @@ def process_app_deployment_request(
            matched_owner = match_owner(app_deployment_request, laconic.get_record(dns_record.attributes.request, require=True))

        if matched_owner:
-            print("Matched DnsRecord ownership:", matched_owner)
+            logger.log(f"Matched DnsRecord ownership: {matched_owner}")
        else:
            raise Exception("Unable to confirm ownership of DnsRecord %s for request %s" %
                            (dns_record.id, app_deployment_request.id))
@@ -115,15 +118,21 @@ def process_app_deployment_request(
        shared_tag_exists = remote_image_exists(image_registry, app_image_shared_tag)
        if shared_tag_exists and not force_rebuild:
            # simply add our unique tag to the existing image and we are done
-            print(f"Using existing app image {app_image_shared_tag} for {deployment_container_tag}", file=log_file)
+            logger.log(f"Using existing app image {app_image_shared_tag} for {deployment_container_tag}")
            add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_tag)
+            logger.log("Tag complete")
        else:
            extra_build_args = []  # TODO: pull from request
-            build_container_image(app, deployment_container_tag, extra_build_args, log_file)
-            push_container_image(deployment_dir, log_file)
+            logger.log(f"Building container image {deployment_container_tag}")
+            build_container_image(app, deployment_container_tag, extra_build_args, logger)
+            logger.log("Build complete")
+            logger.log(f"Pushing container image {deployment_container_tag}")
+            push_container_image(deployment_dir, logger)
+            logger.log("Push complete")
            # The build/push commands above will use the unique deployment tag, so now we need to add the shared tag.
-            print(f"Updating app image tag {app_image_shared_tag} from build of {deployment_container_tag}", file=log_file)
+            logger.log(f"Updating app image tag {app_image_shared_tag} from build of {deployment_container_tag}")
            add_tags_to_image(image_registry, deployment_container_tag, app_image_shared_tag)
+            logger.log("Tag complete")

    # 7. update config (if needed)
    if not deployment_record or file_hash(deployment_config_file) != deployment_record.attributes.meta.config:
@@ -131,13 +140,13 @@ def process_app_deployment_request(

    # 8. update k8s deployment
    if needs_k8s_deploy:
-        print("Deploying to k8s")
        deploy_to_k8s(
            deployment_record,
            deployment_dir,
-            log_file
+            logger
        )

+    logger.log("Publishing deployment to registry.")
    publish_deployment(
        laconic,
        app,
@@ -146,8 +155,11 @@ def process_app_deployment_request(
        dns_record,
        dns_crn,
        deployment_dir,
-        app_deployment_request
+        app_deployment_request,
+        logger
    )
+    logger.log("Publication complete.")
+    logger.log("END - process_app_deployment_request")


 def load_known_requests(filename):
@@ -309,8 +321,9 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
                    run_log_file = open(run_log_file_path, "wt")
                    run_reg_client = LaconicRegistryClient(laconic_config, log_file=run_log_file)

+                logger = TimedLogger(run_id, run_log_file)
+                logger.log("Processing ...")
                process_app_deployment_request(
-                    run_id,
                    ctx,
                    run_reg_client,
                    r,
@@ -321,12 +334,14 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
                    kube_config,
                    image_registry,
                    force_rebuild,
-                    run_log_file
+                    logger
                )
                status = "DEPLOYED"
            except Exception as e:
-                print("ERROR: " + str(e), file=run_log_file)
+                logger.log("ERROR: " + str(e))
            finally:
+                if logger:
+                    logger.log(f"DONE with status {status}", show_step_time=False, show_total_time=True)
                dump_known_requests(state_file, [r], status)
                if run_log_file:
                    run_log_file.close()
@@ -13,6 +13,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.

+import datetime
 import hashlib
 import json
 import os
@@ -39,6 +40,25 @@ class AttrDict(dict):
            return v


+class TimedLogger:
+    def __init__(self, id="", file=None):
+        self.start = datetime.datetime.now()
+        self.last = self.start
+        self.id = id
+        self.file = file
+
+    def log(self, msg, show_step_time=True, show_total_time=False):
+        prefix = f"{datetime.datetime.utcnow()} - {self.id}"
+        if show_step_time:
+            prefix += f" - {datetime.datetime.now() - self.last} (step)"
+        if show_total_time:
+            prefix += f" - {datetime.datetime.now() - self.start} (total)"
+        print(f"{prefix}: {msg}", file=self.file)
+        if self.file:
+            self.file.flush()
+        self.last = datetime.datetime.now()
+
+
 def logged_cmd(log_file, *vargs):
    result = None
    try:
@@ -219,7 +239,7 @@ def determine_base_container(clone_dir, app_type="webapp"):
    return base_container


-def build_container_image(app_record, tag, extra_build_args=[], log_file=None):
+def build_container_image(app_record, tag, extra_build_args=[], logger=None):
    tmpdir = tempfile.mkdtemp()

    try:
@@ -228,31 +248,33 @@ def build_container_image(app_record, tag, extra_build_args=[], log_file=None):
        repo = random.choice(app_record.attributes.repository)
        clone_dir = os.path.join(tmpdir, record_id)

-        print(f"Cloning repository {repo} to {clone_dir} ...")
+        logger.log(f"Cloning repository {repo} to {clone_dir} ...")
        if ref:
            # TODO: Determing branch or hash, and use depth 1 if we can.
            git_env = dict(os.environ.copy())
            # Never prompt
            git_env["GIT_TERMINAL_PROMPT"] = "0"
            try:
-                subprocess.check_call(["git", "clone", repo, clone_dir], env=git_env, stdout=log_file, stderr=log_file)
+                subprocess.check_call(["git", "clone", repo, clone_dir], env=git_env, stdout=logger.file, stderr=logger.file)
            except Exception as e:
-                print(f"git clone failed.  Is the repository {repo} private?", file=log_file)
+                logger.log(f"git clone failed.  Is the repository {repo} private?")
                raise e
            try:
-                subprocess.check_call(["git", "checkout", ref], cwd=clone_dir, env=git_env, stdout=log_file, stderr=log_file)
+                subprocess.check_call(["git", "checkout", ref], cwd=clone_dir, env=git_env, stdout=logger.file, stderr=logger.file)
            except Exception as e:
-                print(f"git checkout failed.  Does ref {ref} exist?", file=log_file)
+                logger.log(f"git checkout failed.  Does ref {ref} exist?")
                raise e
        else:
-            result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir], stdout=log_file, stderr=log_file)
+            result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir], stdout=logger.file, stderr=logger.file)
            result.check_returncode()

        base_container = determine_base_container(clone_dir, app_record.attributes.app_type)

-        print("Building webapp ...")
+        logger.log("Building webapp ...")
        build_command = [
-            sys.argv[0], "build-webapp",
+            sys.argv[0],
+            "--verbose",
+            "build-webapp",
            "--source-repo", clone_dir,
            "--tag", tag,
            "--base-container", base_container
@@ -261,28 +283,31 @@ def build_container_image(app_record, tag, extra_build_args=[], log_file=None):
            build_command.append("--extra-build-args")
            build_command.append(" ".join(extra_build_args))

-        result = subprocess.run(build_command, stdout=log_file, stderr=log_file)
+        result = subprocess.run(build_command, stdout=logger.file, stderr=logger.file)
        result.check_returncode()
    finally:
-        logged_cmd(log_file, "rm", "-rf", tmpdir)
+        logged_cmd(logger.file, "rm", "-rf", tmpdir)


-def push_container_image(deployment_dir, log_file=None):
-    print("Pushing image ...")
+def push_container_image(deployment_dir, logger):
+    logger.log("Pushing images ...")
    result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, "push-images"],
-                            stdout=log_file, stderr=log_file)
+                            stdout=logger.file, stderr=logger.file)
    result.check_returncode()
+    logger.log("Finished pushing images.")


-def deploy_to_k8s(deploy_record, deployment_dir, log_file=None):
+def deploy_to_k8s(deploy_record, deployment_dir, logger):
    if not deploy_record:
        command = "up"
    else:
        command = "update"

+    logger.log("Deploying to k8s ...")
    result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command],
-                            stdout=log_file, stderr=log_file)
+                            stdout=logger.file, stderr=logger.file)
    result.check_returncode()
+    logger.log("Finished deploying to k8s.")


 def publish_deployment(laconic: LaconicRegistryClient,
@@ -292,7 +317,8 @@ def publish_deployment(laconic: LaconicRegistryClient,
                       dns_record,
                       dns_crn,
                       deployment_dir,
-                       app_deployment_request=None):
+                       app_deployment_request=None,
+                       logger=None):
    if not deploy_record:
        deploy_ver = "0.0.1"
    else:
@@ -322,6 +348,8 @@ def publish_deployment(laconic: LaconicRegistryClient,
    if app_deployment_request:
        new_dns_record["record"]["request"] = app_deployment_request.id

+    if logger:
+        logger.log("Publishing DnsRecord.")
    dns_id = laconic.publish(new_dns_record, [dns_crn])

    new_deployment_record = {
@@ -341,6 +369,8 @@ def publish_deployment(laconic: LaconicRegistryClient,
    if app_deployment_request:
        new_deployment_record["record"]["request"] = app_deployment_request.id

+    if logger:
+        logger.log("Publishing ApplicationDeploymentRecord.")
    deployment_id = laconic.publish(new_deployment_record, [deployment_crn])
    return {"dns": dns_id, "deployment": deployment_id}

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+echo "$(date +"%Y-%m-%d %T"): Running stack-orchestrator Laconic registry CLI tests"
+env
+cat /etc/hosts
+# Bit of a hack, test the most recent package
+TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
+
+export CERC_REPO_BASE_DIR=$(mktemp -d $(pwd)/stack-orchestrator-fixturenet-laconicd-test.XXXXXXXXXX)
+echo "$(date +"%Y-%m-%d %T"): Cloning laconic-registry-cli repository into: $CERC_REPO_BASE_DIR"
+$TEST_TARGET_SO --stack fixturenet-laconicd setup-repositories --include git.vdb.to/cerc-io/laconic-registry-cli
+
+echo "$(date +"%Y-%m-%d %T"): Starting stack"
+TEST_AUCTION_ENABLED=true BASE_DIR=${CERC_REPO_BASE_DIR} $TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd up
+echo "$(date +"%Y-%m-%d %T"): Stack started"
+
+# Verify that the fixturenet is up and running
+$TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd ps
+
+# Get the fixturenet account address
+laconicd_account_address=$(docker exec laconicd-laconicd-1 laconicd keys list | awk '/- address:/ {print $3}')
+
+# Copy over config
+docker exec laconicd-cli-1 cp config.yml laconic-registry-cli/
+
+# Wait for the laconid endpoint to come up
+echo "Waiting for the RPC endpoint to come up"
+docker exec laconicd-laconicd-1 sh -c "curl --retry 20 --retry-delay 3 --retry-connrefused http://127.0.0.1:9473/api"
+
+# Run the tests
+echo "Running the tests"
+docker exec -e TEST_ACCOUNT=$laconicd_account_address laconicd-cli-1 sh -c 'cd laconic-registry-cli && yarn && yarn test'
+
+# Clean up
+$TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd down --delete-volumes
+echo "$(date +"%Y-%m-%d %T"): Removing cloned repositories"
+rm -rf $CERC_REPO_BASE_DIR
+echo "$(date +"%Y-%m-%d %T"): Test finished"
@@ -12,6 +12,7 @@ cat /etc/hosts
 TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
 # Set a new unique repo dir
 export CERC_REPO_BASE_DIR=$(mktemp -d $(pwd)/stack-orchestrator-fixturenet-laconicd-test.XXXXXXXXXX)
+
 echo "$(date +"%Y-%m-%d %T"): Testing this package: $TEST_TARGET_SO"
 echo "$(date +"%Y-%m-%d %T"): Test version command"
 reported_version_string=$( $TEST_TARGET_SO version )
@@ -32,14 +32,14 @@ set +e

 CONTAINER_ID=$(docker run -p 3000:80 -d -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG cerc/test-progressive-web-app:local)
 sleep 3
-wget -t 7 -O test.before -m http://localhost:3000
+wget --tries 20 --retry-connrefused --waitretry=3 -O test.before -m http://localhost:3000

 docker logs $CONTAINER_ID
 docker remove -f $CONTAINER_ID

 CONTAINER_ID=$(docker run -p 3000:80 -e CERC_WEBAPP_DEBUG=$CHECK -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG -d cerc/test-progressive-web-app:local)
 sleep 3
-wget -t 7 -O test.after -m http://localhost:3000
+wget --tries 20 --retry-connrefused --waitretry=3 -O test.after -m http://localhost:3000

 docker logs $CONTAINER_ID
 docker remove -f $CONTAINER_ID