Reduce alert pending period for watcher alerts

Part of https://www.notion.so/Monitoring-and-alerting-for-laconicd-86727c3b4dde4dc993d87d6e29f935fe

- Add a laconicd Grafana dashboard
  - Update fixturenet-laconicd script to expose metrics
- Upgrade Grafana version to avoid errors while saving changes made to a dashboard (see [thread](https://community.grafana.com/t/error-cannot-add-property-ishandled-object-is-not-extensible/109268))
-  Add an alert rule for Ajna watcher

Reviewed-on: #799
Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com>
Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>

.gitea/workflows/fixturenet-eth-plugeth-arm-test.yml

@@ -43,3 +43,19 @@ jobs:
         run: ./scripts/build_shiv_package.sh
       - name: "Run fixturenet-eth tests"
         run: ./tests/fixturenet-eth-plugeth/run-test.sh
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/fixturenet-eth-plugeth-test.yml

@@ -47,3 +47,19 @@ jobs:
           sleep 5
       - name: "Run fixturenet-eth tests"
         run: ./tests/fixturenet-eth-plugeth/run-test.sh
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/fixturenet-eth-test.yml

@@ -45,4 +45,19 @@ jobs:
           sleep 5
       - name: "Run fixturenet-eth tests"
         run: ./tests/fixturenet-eth/run-test.sh
-
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/fixturenet-laconicd-test.yml

@@ -11,7 +11,7 @@ on:
 
 jobs:
   test:
-    name: "Run an Laconicd fixturenet test"
+    name: "Run Laconicd fixturenet and Laconic CLI tests"
     runs-on: ubuntu-latest
     steps:
       - name: 'Update'
@@ -46,3 +46,21 @@ jobs:
         run: ./scripts/build_shiv_package.sh
       - name: "Run fixturenet-laconicd tests"
         run: ./tests/fixturenet-laconicd/run-test.sh
+      - name: "Run laconic CLI tests"
+        run: ./tests/fixturenet-laconicd/run-cli-test.sh
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/lint.yml

@@ -19,3 +19,19 @@ jobs:
           python-version: '3.8'
       - name : "Run flake8"
         uses: py-actions/flake8@v2
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/publish.yml

@@ -54,3 +54,19 @@ jobs:
           # Hack using endsWith to workaround Gitea sometimes sending "publish-test" vs "refs/heads/publish-test"
           draft: ${{ endsWith('publish-test', github.ref ) }}
           files: ./laconic-so
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/test-container-registry.yml

@@ -51,4 +51,19 @@ jobs:
           source /opt/bash-utils/cgroup-helper.sh
           join_cgroup
           ./tests/container-registry/run-test.sh
-
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/test-database.yml

@@ -49,4 +49,19 @@ jobs:
           source /opt/bash-utils/cgroup-helper.sh
           join_cgroup
           ./tests/database/run-test.sh
-
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/test-deploy.yml

@@ -47,3 +47,19 @@ jobs:
           sleep 5
       - name: "Run deploy tests"
         run: ./tests/deploy/run-deploy-test.sh
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/test-k8s-deploy.yml

@@ -51,4 +51,19 @@ jobs:
           source /opt/bash-utils/cgroup-helper.sh
           join_cgroup
           ./tests/k8s-deploy/run-deploy-test.sh
-
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/test-webapp.yml

@@ -49,3 +49,19 @@ jobs:
           sleep 5
       - name: "Run webapp tests"
         run: ./tests/webapp-test/run-webapp-test.sh
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/test.yml

@@ -47,5 +47,19 @@ jobs:
           sleep 5
       - name: "Run smoke tests"
         run: ./tests/smoke-test/run-smoke-test.sh
-
-
+      - name: Notify Vulcanize Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.VULCANIZE_SLACK_CI_ALERTS }}
+      - name: Notify DeepStack Slack on CI failure
+        if: ${{ always() && github.ref_name == 'main' }}
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.DEEPSTACK_SLACK_CI_ALERTS }}

.gitea/workflows/triggers/fixturenet-laconicd-test

@@ -1,3 +1,6 @@
 Change this file to trigger running the fixturenet-laconicd-test CI job
 Trigger
 Trigger
+Trigger
+Trigger
+Trigger

stack_orchestrator/data/compose/docker-compose-fixturenet-blast.yml

@@ -0,0 +1,80 @@
+
+# From: https://raw.githubusercontent.com/blast-io/deployment/master/docker-compose.yml
+services:
+  # generate jwt.txt if it's absent
+  generate-jwt:
+    image: blastio/openssl
+    volumes:
+      - blast-data:/blast:rw
+    command: >
+      sh -c "[ ! -f /blast/jwt.txt ] && openssl rand -hex 32 | tr -d '\n' > /blast/jwt.txt || exit 0"
+  # initialise geth db
+  geth-init:
+    image: blastio/blast-geth:${NETWORK:-testnet-sepolia}
+    volumes:
+      - blast-data:/blast:rw
+      - ../config/fixturenet-blast/genesis.json:/blast/genesis.json
+    entrypoint: /bin/sh
+    command: >
+      -c "[ ! -d /blast/${GETH_DATA_DIR:-blast-geth-data}/geth ] && /usr/local/bin/geth init --datadir=/blast/${GETH_DATA_DIR:-blast-geth-data} /blast/genesis.json || exit 0"
+    depends_on:
+      generate-jwt:
+        condition: service_completed_successfully
+    env_file: 
+      - ../config/fixturenet-blast/${NETWORK:-fixturenet}.config
+  blast-geth:
+    image: blastio/blast-geth:${NETWORK:-testnet-sepolia}
+    volumes:
+      - blast-data:/blast
+    ports:
+      - "9545"
+      - "9546"
+    command: >
+      --datadir=/blast/${GETH_DATA_DIR:-blast-geth-data}
+      --http
+      --http.corsdomain="*"
+      --http.vhosts="*"
+      --http.addr=0.0.0.0
+      --http.port=9545
+      --http.api=web3,debug,eth,txpool,net,engine
+      --ws
+      --ws.addr=0.0.0.0
+      --ws.port=9546
+      --ws.origins="*"
+      --ws.api=debug,eth,txpool,net,engine
+      --authrpc.addr="0.0.0.0"
+      --authrpc.port="8551"
+      --authrpc.vhosts="*"
+      --authrpc.jwtsecret=/blast/jwt.txt
+      --syncmode=full
+      --gcmode=archive
+      --nodiscover
+      --maxpeers=0
+      --rollup.disabletxpoolgossip=true
+    env_file: 
+      - ../config/fixturenet-blast/${NETWORK:-fixturenet}.config
+    depends_on:
+      geth-init:
+        condition: service_completed_successfully
+  op-node:
+    image: blastio/blast-optimism:${NETWORK:-testnet-sepolia}
+    volumes:
+      - blast-data:/blast
+      - ../config/fixturenet-blast/rollup.json:/blast/rollup.json
+    ports:
+      - "9003"
+    command: >
+      op-node
+      --l1="${CERC_L1_RPC}"
+      --l1.rpckind="any"
+      --l1.trustrpc=true
+      --l2="http://blast-geth:8551"
+      --l2.jwt-secret=/blast/jwt.txt
+      --rollup.config="/blast/rollup.json"
+    depends_on:
+      - blast-geth
+    env_file: 
+      - ../config/fixturenet-blast/${NETWORK:-fixturenet}.config
+
+volumes:
+  blast-data:

stack_orchestrator/data/compose/docker-compose-fixturenet-laconicd.yml

@@ -3,6 +3,9 @@ services:
     restart: unless-stopped
     image: cerc/laconicd:local
     command: ["sh", "/docker-entrypoint-scripts.d/create-fixturenet.sh"]
+    environment:
+      TEST_AUCTION_ENABLED: ${TEST_AUCTION_ENABLED}
+      TEST_REGISTRY_EXPIRY: ${TEST_REGISTRY_EXPIRY}
     volumes:
       # The cosmos-sdk node's database directory:
       - laconicd-data:/root/.laconicd
@@ -25,6 +28,7 @@ services:
     image: cerc/laconic-registry-cli:local
     volumes:
       - ../config/fixturenet-laconicd/registry-cli-config-template.yml:/registry-cli-config-template.yml
+      - ${BASE_DIR:-~/cerc}/laconic-registry-cli:/laconic-registry-cli
 
 volumes:
   laconicd-data:

stack_orchestrator/data/compose/docker-compose-fixturenet-plugeth.yml

@@ -6,6 +6,7 @@ services:
       - ../config/fixturenet-eth/fixturenet-eth.env
     environment:
       RUN_BOOTNODE: "true"
+      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
     image: cerc/fixturenet-plugeth-plugeth:local
     volumes:
       - fixturenet_plugeth_bootnode_geth_data:/root/ethdata

stack_orchestrator/data/compose/docker-compose-grafana.yml

@@ -2,7 +2,7 @@ version: "3.7"
 
 services:
   grafana:
-    image: grafana/grafana:10.2.2
+    image: grafana/grafana:10.2.3
     restart: always
     environment:
       GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL}

stack_orchestrator/data/compose/docker-compose-mainnet-blast.yml

@@ -0,0 +1,83 @@
+
+# From: https://raw.githubusercontent.com/blast-io/deployment/master/docker-compose.yml
+services:
+  # generate jwt.txt if it's absent
+  generate-jwt:
+    image: blastio/openssl
+    volumes:
+      - blast-data:/blast:rw
+    command: >
+      sh -c "[ ! -f /blast/jwt.txt ] && openssl rand -hex 32 | tr -d '\n' > /blast/jwt.txt || exit 0"
+  # initialise geth db
+  geth-init:
+    image: blastio/blast-geth:${NETWORK:-mainnet}
+    volumes:
+      - blast-data:/blast:rw
+    entrypoint: /bin/sh
+    command: >
+      -c "[ ! -d /blast/${GETH_DATA_DIR:-blast-geth-data}/geth ] && /usr/local/bin/geth init --datadir=/blast/${GETH_DATA_DIR:-blast-geth-data} /blast/genesis.json || exit 0"
+    depends_on:
+      generate-jwt:
+        condition: service_completed_successfully
+    env_file: 
+      - ../config/mainnet-blast/${NETWORK:-mainnet}.config
+  blast-geth:
+    image: blastio/blast-geth:${NETWORK:-mainnet}
+    volumes:
+      - blast-data:/blast
+    ports:
+      - "9545"
+      - "9546"
+      - "6060"
+    command: >
+      --datadir=/blast/${GETH_DATA_DIR:-blast-geth-data}
+      --http
+      --http.corsdomain="*"
+      --http.vhosts="*"
+      --http.addr=0.0.0.0
+      --http.port=9545
+      --http.api=web3,debug,eth,txpool,net,engine
+      --ws
+      --ws.addr=0.0.0.0
+      --ws.port=9546
+      --ws.origins="*"
+      --ws.api=debug,eth,txpool,net,engine
+      --authrpc.addr="0.0.0.0"
+      --authrpc.port="8551"
+      --authrpc.vhosts="*"
+      --authrpc.jwtsecret=/blast/jwt.txt
+      --syncmode=full
+      --metrics
+      --metrics.addr=0.0.0.0
+      --gcmode=archive
+      --nodiscover
+      --maxpeers=0
+      --rollup.disabletxpoolgossip=true
+    env_file: 
+      - ../config/mainnet-blast/${NETWORK:-mainnet}.config
+    depends_on:
+      geth-init:
+        condition: service_completed_successfully
+  op-node:
+    image: blastio/blast-optimism:${NETWORK:-mainnet}
+    volumes:
+      - blast-data:/blast
+    ports:
+      - "9003"
+      - "7300"
+    command: >
+      op-node
+      --l1="https://eth-mainnet-1.vdb.to/"
+      --metrics.enabled
+      --l1.rpckind="any"
+      --l1.trustrpc=true
+      --l2="http://blast-geth:8551"
+      --l2.jwt-secret=/blast/jwt.txt
+      --rollup.config="/blast/rollup.json"
+    depends_on:
+      - blast-geth
+    env_file: 
+      - ../config/mainnet-blast/${NETWORK:-mainnet}.config
+
+volumes:
+  blast-data:

stack_orchestrator/data/config/fixturenet-blast/fixturenet.config

@@ -0,0 +1,2 @@
+GETH_ROLLUP_SEQUENCERHTTP=https://sequencer.s2.testblast.io
+OP_NODE_P2P_BOOTNODES=enr:-J-4QM3GLUFfKMSJQuP1UvuKQe8DyovE7Eaiit0l6By4zjTodkR4V8NWXJxNmlg8t8rP-Q-wp3jVmeAOml8cjMj__ROGAYznzb_HgmlkgnY0gmlwhA-cZ_eHb3BzdGFja4X947FQAIlzZWNwMjU2azGhAiuDqvB-AsVSRmnnWr6OHfjgY8YfNclFy9p02flKzXnOg3RjcIJ2YYN1ZHCCdmE,enr:-J-4QDCVpByqQ8nFqCS9aHicqwUfXgzFDslvpEyYz19lvkHLIdtcIGp2d4q5dxHdjRNTO6HXCsnIKxUeuZSPcEbyVQCGAYznzz0RgmlkgnY0gmlwhANiQfuHb3BzdGFja4X947FQAIlzZWNwMjU2azGhAy3AtF2Jh_aPdOohg506Hjmtx-fQ1AKmu71C7PfkWAw9g3RjcIJ2YYN1ZHCCdmE

stack_orchestrator/data/config/fixturenet-blast/genesis.json

@@ -0,0 +1,57 @@
+{
+  "config": {
+    "chainId": 608943043,
+    "homesteadBlock": 0,
+    "eip150Block": 0,
+    "eip155Block": 0,
+    "eip158Block": 0,
+    "byzantiumBlock": 0,
+    "constantinopleBlock": 0,
+    "petersburgBlock": 0,
+    "istanbulBlock": 0,
+    "muirGlacierBlock": 0,
+    "berlinBlock": 0,
+    "londonBlock": 0,
+    "arrowGlacierBlock": 0,
+    "grayGlacierBlock": 0,
+    "mergeNetsplitBlock": 0,
+    "shanghaiTime": 0,
+    "bedrockBlock": 0,
+    "regolithTime": 0,
+    "canyonTime": 0,
+    "terminalTotalDifficulty": 0,
+    "terminalTotalDifficultyPassed": true,
+    "optimism": {
+      "eip1559Elasticity": 6,
+      "eip1559Denominator": 50,
+      "eip1559DenominatorCanyon": 250
+    }
+  },
+  "alloc": {
+    "0000000000000000000000000000000000000000": {
+      "balance": "0x1"
+    },
+    "4200000000000000000000000000000000000000": {
+      "code": "0x60806040526004361061004e5760003560e01c80633659cfe6146100655780634f1ef286146100855780635c60da1b146100ae5780638f283970146100db578063f851a440146100fb5761005d565b3661005d5761005b610110565b005b61005b610110565b34801561007157600080fd5b5061005b610080366004610521565b6101c8565b61009861009336600461053c565b61020e565b6040516100a591906105bf565b60405180910390f35b3480156100ba57600080fd5b506100c361033e565b6040516001600160a01b0390911681526020016100a5565b3480156100e757600080fd5b5061005b6100f6366004610521565b6103a9565b34801561010757600080fd5b506100c36103e4565b600061013a7f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc5490565b90506001600160a01b0381166101a55760405162461bcd60e51b815260206004820152602560248201527f50726f78793a20696d706c656d656e746174696f6e206e6f7420696e697469616044820152641b1a5e995960da1b60648201526084015b60405180910390fd5b3660008037600080366000845af43d6000803e806101c2573d6000fd5b503d6000f35b600080516020610625833981519152546001600160a01b0316336001600160a01b031614806101f5575033155b156102065761020381610432565b50565b610203610110565b60606102266000805160206106258339815191525490565b6001600160a01b0316336001600160a01b03161480610243575033155b1561032f5761025184610432565b600080856001600160a01b0316858560405161026e929190610614565b600060405180830381855af49150503d80600081146102a9576040519150601f19603f3d011682016040523d82523d6000602084013e6102ae565b606091505b5091509150816103265760405162461bcd60e51b815260206004820152603960248201527f50726f78793a2064656c656761746563616c6c20746f206e657720696d706c6560448201527f6d656e746174696f6e20636f6e7472616374206661696c656400000000000000606482015260840161019c565b91506103379050565b610337610110565b9392505050565b60006103566000805160206106258339815191525490565b6001600160a01b0316336001600160a01b03161480610373575033155b1561039e57507f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc5490565b6103a6610110565b90565b600080516020610625833981519152546001600160a01b0316336001600160a01b031614806103d6575033155b15610206576102038161048e565b60006103fc6000805160206106258339815191525490565b6001600160a01b0316336001600160a01b03161480610419575033155b1561039e57506000805160206106258339815191525490565b7f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc8181556040516001600160a01b038316907fbc7cd75a20ee27fd9adebab32041f755214dbc6bffa90cc0225b39da2e5c2d3b90600090a25050565b60006104a66000805160206106258339815191525490565b600080516020610625833981519152838155604080516001600160a01b0380851682528616602082015292935090917f7e644d79422f17c01e4894b5f4f588d331ebfa28653d42ae832dc59e38c9798f910160405180910390a1505050565b80356001600160a01b038116811461051c57600080fd5b919050565b60006020828403121561053357600080fd5b61033782610505565b60008060006040848603121561055157600080fd5b61055a84610505565b9250602084013567ffffffffffffffff8082111561057757600080fd5b818601915086601f83011261058b57600080fd5b81358181111561059a57600080fd5b8760208285010111156105ac57600080fd5b6020830194508093505050509250925092565b600060208083528351808285015260005b818110156105ec578581018301518582016040015282016105d0565b818111156105fe576000604083870101525b50601f01601f1916929092016040019392505050565b818382376000910190815291905056feb53127684a568b3173ae13b9f8a6016e243e63b6e8ee1178d6a717850b5d6103a164736f6c634300080f000a",
+      "storage": {
+        "0x360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc": "0x000000000000000000000000c0d3c0d3c0d3c0d3c0d3c0d3c0d3c0d3c0d30000",
+        "0xb53127684a568b3173ae13b9f8a6016e243e63b6e8ee1178d6a717850b5d6103": "0x0000000000000000000000004200000000000000000000000000000000000018"
+      },
+      "balance": "0x0",
+      "flags": 1
+    }
+  },
+  "nonce": "0x0",
+  "timestamp": "0x659b7460",
+  "extraData": "0x424544524f434b",
+  "gasLimit": "0x1c9c380",
+  "difficulty": "0x0",
+  "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+  "coinbase": "0x4200000000000000000000000000000000000011",
+  "number": "0x0",
+  "gasUsed": "0x0",
+  "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+  "baseFeePerGas": "0x3b9aca00",
+  "excessBlobGas": null,
+  "blobGasUsed": null
+}

stack_orchestrator/data/config/fixturenet-blast/rollup.json

@@ -0,0 +1,31 @@
+{
+  "genesis": {
+    "l1": {
+      "hash": "0x17728cf4d8e0b4f292d2390a869fd7c632d39e72efb00ca3462b4387c6aa2437",
+      "number": 5044255
+    },
+    "l2": {
+      "hash": "0x26a1c0faad7b041f34569a1bb383f00ab74b335883a44bed53e9f41ced5fd906",
+      "number": 0
+    },
+    "l2_time": 1704686688,
+    "system_config": {
+      "batcherAddr": "0xba26fee2fa917443e05e65de8d4350bcd2f59222",
+      "overhead": "0x00000000000000000000000000000000000000000000000000000000000000bc",
+      "scalar": "0x00000000000000000000000000000000000000000000000000000000000a6fe0",
+      "gasLimit": 30000000
+    }
+  },
+  "block_time": 2,
+  "max_sequencer_drift": 600,
+  "seq_window_size": 3600,
+  "channel_timeout": 300,
+  "l1_chain_id": 11155111,
+  "l2_chain_id": 608943043,
+  "regolith_time": 0,
+  "canyon_time": 0,
+  "batch_inbox_address": "0x1c3b85a2108784eab6a4bf56cdd6f722e415b331",
+  "deposit_contract_address": "0x2757e4430e694f27b73ec9c02257cab3a498c8c5",
+  "l1_system_config_address": "0x329faf078c364a316e08bf6a17b7eee6ae75a613",
+  "protocol_versions_address": "0x0000000000000000000000000000000000000000"
+}

stack_orchestrator/data/config/fixturenet-eth/fixturenet-eth.env

@@ -22,4 +22,7 @@ CERC_STATEDIFF_DB_LOG_STATEMENTS="${CERC_STATEDIFF_DB_LOG_STATEMENTS:-false}"
 CERC_STATEDIFF_WORKERS=2
 
 CERC_GETH_VMODULE="statediff/*=5,rpc/*=5"
-CERC_GETH_VERBOSITY=${CERC_GETH_VERBOSITY:-3}
+CERC_GETH_VERBOSITY=${CERC_GETH_VERBOSITY:-3}
+
+# Used by Lighthouse
+SECONDS_PER_ETH1_BLOCK=${SECONDS_PER_ETH1_BLOCK:-3}

stack_orchestrator/data/config/fixturenet-laconicd/create-fixturenet.sh

@@ -102,6 +102,17 @@ if [ "$1" == "clean" ] || [ ! -d "$HOME/.laconicd/data/blockstore.db" ]; then
     fi
   fi
 
+  # Enable telemetry (prometheus metrics: http://localhost:1317/metrics?format=prometheus)
+  if [[ "$OSTYPE" == "darwin"* ]]; then
+    sed -i '' 's/enabled = false/enabled = true/g' $HOME/.laconicd/config/app.toml
+    sed -i '' 's/prometheus-retention-time = 0/prometheus-retention-time = 60/g' $HOME/.laconicd/config/app.toml
+    sed -i '' 's/prometheus = false/prometheus = true/g' $HOME/.laconicd/config/config.toml
+  else
+    sed -i 's/enabled = false/enabled = true/g' $HOME/.laconicd/config/app.toml
+    sed -i 's/prometheus-retention-time = 0/prometheus-retention-time = 60/g' $HOME/.laconicd/config/app.toml
+    sed -i 's/prometheus = false/prometheus = true/g' $HOME/.laconicd/config/config.toml
+  fi
+
   # Allocate genesis accounts (cosmos formatted addresses)
   laconicd add-genesis-account $KEY 100000000000000000000000000aphoton --keyring-backend $KEYRING
 

stack_orchestrator/data/config/mainnet-blast/keycloak.env

@@ -0,0 +1,32 @@
+POSTGRES_DB=keycloak
+POSTGRES_USER=keycloak
+POSTGRES_PASSWORD=keycloak
+# Don't change this unless you also change the healthcheck in docker-compose-mainnet-eth-keycloak.yml
+PGPORT=35432
+KC_DB=postgres
+KC_DB_URL_HOST=keycloak-db
+KC_DB_URL_PORT=${PGPORT}
+KC_DB_URL_DATABASE=${POSTGRES_DB}
+KC_DB_USERNAME=${POSTGRES_USER}
+KC_DB_PASSWORD=${POSTGRES_PASSWORD}
+KC_DB_SCHEMA=public
+KC_HOSTNAME=localhost
+KC_HTTP_ENABLED="true"
+KC_HTTP_RELATIVE_PATH="/auth"
+KC_HOSTNAME_STRICT_HTTPS="false"
+KEYCLOAK_ADMIN=admin
+KEYCLOAK_ADMIN_PASSWORD=admin
+X_API_CHECK_REALM=cerc
+X_API_CHECK_CLIENT_ID="%user_id%"
+
+
+# keycloak-reg-api
+CERC_KCUSERREG_LISTEN_PORT=9292
+CERC_KCUSERREG_LISTEN_ADDR='0.0.0.0'
+CERC_KCUSERREG_API_URL='http://keycloak:8080/auth'
+CERC_KCUSERREG_REG_USER="${KEYCLOAK_ADMIN}"
+CERC_KCUSERREG_REG_PW="${KEYCLOAK_ADMIN_PASSWORD}"
+CERC_KCUSERREG_REG_CLIENT_ID='admin-cli'
+CERC_KCUSERREG_TARGET_REALM=cerc
+CERC_KCUSERREG_TARGET_GROUPS=eth
+CERC_KCUSERREG_CREATE_ENABLED=true

stack_orchestrator/data/config/mainnet-blast/lighthouse.env

@@ -0,0 +1,33 @@
+# Enable startup script debug output.
+CERC_SCRIPT_DEBUG=false
+
+# Specify any other lighthouse CLI options.
+LIGHTHOUSE_OPTS=""
+
+# Override the advertised public IP (optional)
+# --enr-address
+#LIGHTHOUSE_ENR_ADDRESS=""
+
+# --checkpoint-sync-url
+LIGHTHOUSE_CHECKPOINT_SYNC_URL="https://beaconstate.ethstaker.cc"
+
+# --checkpoint-sync-url-timeout
+LIGHTHOUSE_CHECKPOINT_SYNC_URL_TIMEOUT=300
+
+# --datadir
+LIGHTHOUSE_DATADIR=/data
+
+# --debug-level
+LIGHTHOUSE_DEBUG_LEVEL=info
+
+# --http-port
+LIGHTHOUSE_HTTP_PORT=5052
+
+# --execution-jwt
+LIGHTHOUSE_JWTSECRET=/etc/mainnet-eth/jwtsecret
+
+# --metrics-port
+LIGHTHOUSE_METRICS_PORT=5054
+
+# --port --enr-udp-port --enr-tcp-port
+LIGHTHOUSE_NETWORK_PORT=9000

stack_orchestrator/data/config/mainnet-blast/mainnet.config

@@ -0,0 +1,2 @@
+GETH_ROLLUP_SEQUENCERHTTP=https://sequencer.blast.io
+OP_NODE_P2P_BOOTNODES=enr:-J64QGwHl9uYLfC_cnmxSA6wQH811nkOWJDWjzxqkEUlJoZHWvI66u-BXgVcPCeMUmg0dBpFQAPotFchG67FHJMZ9OSGAY3d6wevgmlkgnY0gmlwhANizeSHb3BzdGFja4Sx_AQAiXNlY3AyNTZrMaECg4pk0cskPAyJ7pOmo9E6RqGBwV-Lex4VS9a3MQvu7PWDdGNwgnZhg3VkcIJ2YQ,enr:-J64QDge2jYBQtcNEpRqmKfci5E5BHAhNBjgv4WSdwH1_wPqbueq2bDj38-TSW8asjy5lJj1Xftui6Or8lnaYFCqCI-GAY3d6wf3gmlkgnY0gmlwhCO2D9yHb3BzdGFja4Sx_AQAiXNlY3AyNTZrMaEDo4aCTq7pCEN8om9U5n_VyWdambGnQhwHNwKc8o-OicaDdGNwgnZhg3VkcIJ2YQ

stack_orchestrator/data/config/mainnet-blast/rollup.json

@@ -0,0 +1,32 @@
+{
+    "genesis": {
+      "l1": {
+        "hash": "0xfcfb8d586bdae763f1189988789211c69eb893a895e7ba48be3ca6289f0941b7",
+        "number": 19300102
+      },
+      "l2": {
+        "hash": "0xb689b35ef29d0bec5816938e0e52683c7257d2e325420ea69b739a2be4754b89",
+        "number": 0
+      },
+      "l2_time": 1708809815,
+      "system_config": {
+        "batcherAddr": "0x415c8893d514f9bc5211d36eeda4183226b84aa7",
+        "overhead": "0x00000000000000000000000000000000000000000000000000000000000000bc",
+        "scalar": "0x00000000000000000000000000000000000000000000000000000000000a6fe0",
+        "gasLimit": 30000000
+      }
+    },
+    "block_time": 2,
+    "max_sequencer_drift": 600,
+    "seq_window_size": 3600,
+    "channel_timeout": 300,
+    "l1_chain_id": 1,
+    "l2_chain_id": 81457,
+    "regolith_time": 0,
+    "canyon_time": 0,
+    "batch_inbox_address": "0xff00000000000000000000000000000000081457",
+    "deposit_contract_address": "0x0ec68c5b10f21effb74f2a5c61dfe6b08c0db6cb",
+    "l1_system_config_address": "0x5531dcff39ec1ec727c4c5d2fc49835368f805a9",
+    "protocol_versions_address": "0x0000000000000000000000000000000000000000"
+  }
+  

stack_orchestrator/data/config/mainnet-blast/run-lighthouse.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+if [[ "true" == "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi
+
+ENR_OPTS=""
+if [[ -n "$LIGHTHOUSE_ENR_ADDRESS" ]]; then
+  ENR_OPTS="--enr-address $LIGHTHOUSE_ENR_ADDRESS"
+fi
+
+exec lighthouse bn \
+  --checkpoint-sync-url "$LIGHTHOUSE_CHECKPOINT_SYNC_URL" \
+  --checkpoint-sync-url-timeout ${LIGHTHOUSE_CHECKPOINT_SYNC_URL_TIMEOUT} \
+  --datadir "$LIGHTHOUSE_DATADIR" \
+  --debug-level $LIGHTHOUSE_DEBUG_LEVEL \
+  --disable-deposit-contract-sync \
+  --disable-upnp \
+  --enr-tcp-port $LIGHTHOUSE_NETWORK_PORT \
+  --enr-udp-port $LIGHTHOUSE_NETWORK_PORT \
+  --execution-endpoint "$LIGHTHOUSE_EXECUTION_ENDPOINT" \
+  --execution-jwt /etc/mainnet-eth/jwtsecret \
+  --http \
+  --http-address 0.0.0.0 \
+  --http-port $LIGHTHOUSE_HTTP_PORT \
+  --metrics \
+  --metrics-address=0.0.0.0 \
+  --metrics-port $LIGHTHOUSE_METRICS_PORT \
+  --network mainnet \
+  --port $LIGHTHOUSE_NETWORK_PORT \
+  $ENR_OPTS $LIGHTHOUSE_OPTS

stack_orchestrator/data/config/monitoring/blackbox-alert-rules.yml

@@ -0,0 +1,147 @@
+apiVersion: 1
+groups:
+    - orgId: 1
+      name: blackbox
+      folder: BlackboxAlerts
+      interval: 30s
+      rules:
+        # Azimuth Gateway endpoint
+        - uid: azimuth_gateway
+          title: azimuth_gateway_endpoint_tracking
+          condition: condition
+          data:
+            - refId: probe
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: PBFA97CFB590B2093
+              model:
+                editorMode: code
+                expr: probe_success{destination="azimuth_gateway"}
+                instant: true
+                intervalMs: 1000
+                legendFormat: __auto
+                maxDataPoints: 43200
+                range: false
+                refId: probe
+            - refId: http_status_code
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: PBFA97CFB590B2093
+              model:
+                editorMode: code
+                expr: probe_http_status_code{destination="azimuth_gateway"}
+                instant: true
+                intervalMs: 1000
+                legendFormat: __auto
+                maxDataPoints: 43200
+                range: false
+                refId: http_status_code
+            - refId: condition
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: __expr__
+              model:
+                conditions:
+                    - evaluator:
+                        params:
+                            - 0
+                            - 0
+                        type: gt
+                      operator:
+                        type: and
+                      query:
+                        params: []
+                      reducer:
+                        params: []
+                        type: avg
+                      type: query
+                datasource:
+                    name: Expression
+                    type: __expr__
+                    uid: __expr__
+                expression: ${probe} != 1 ||  ${http_status_code} != 200
+                intervalMs: 1000
+                maxDataPoints: 43200
+                refId: condition
+                type: math
+          noDataState: Alerting
+          execErrState: Alerting
+          for: 5m
+          annotations:
+            summary: Probe failed for Azimuth gateway endpoint
+          labels:
+            probe_success: '{{ index $values "probe" }}'
+          isPaused: false
+        # Laconicd GQL endpoint
+        - uid: laconicd_gql
+          title: laconicd_gql_endpoint_tracking
+          condition: condition
+          data:
+            - refId: probe
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: PBFA97CFB590B2093
+              model:
+                editorMode: code
+                expr: probe_success{destination="laconicd_gql"}
+                instant: true
+                intervalMs: 1000
+                legendFormat: __auto
+                maxDataPoints: 43200
+                range: false
+                refId: probe
+            - refId: http_status_code
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: PBFA97CFB590B2093
+              model:
+                editorMode: code
+                expr: probe_http_status_code{destination="laconicd_gql"}
+                instant: true
+                intervalMs: 1000
+                legendFormat: __auto
+                maxDataPoints: 43200
+                range: false
+                refId: http_status_code
+            - refId: condition
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: __expr__
+              model:
+                conditions:
+                    - evaluator:
+                        params:
+                            - 0
+                            - 0
+                        type: gt
+                      operator:
+                        type: and
+                      query:
+                        params: []
+                      reducer:
+                        params: []
+                        type: avg
+                      type: query
+                datasource:
+                    name: Expression
+                    type: __expr__
+                    uid: __expr__
+                expression: ${probe} != 1 ||  ${http_status_code} != 200
+                intervalMs: 1000
+                maxDataPoints: 43200
+                refId: condition
+                type: math
+          noDataState: Alerting
+          execErrState: Alerting
+          for: 5m
+          annotations:
+            summary: Probe failed for Laconicd GQL endpoint
+          labels:
+            probe_success: '{{ index $values "probe" }}'
+          isPaused: false

stack_orchestrator/data/config/monitoring/prometheus/prometheus.yml

@@ -24,9 +24,10 @@ scrape_configs:
     params:
       module: [http_2xx]
     static_configs:
-      # Add URLs to be monitored below
-      - targets:
-        # - https://github.com
+      # Add URLs for targets to be monitored below
+      # - targets: [https://github.com]
+      #   labels:
+      #     destination: 'github'
     relabel_configs:
       - source_labels: [__address__]
         regex: (.*)(:80)?
@@ -65,3 +66,12 @@ scrape_configs:
         target_label: instance
       - target_label: __address__
         replacement: postgres-exporter:9187
+
+  - job_name: laconicd
+    metrics_path: /metrics
+    scrape_interval: 30s
+    static_configs:
+      # Add laconicd REST endpoint target with host and port (1317)
+      # - targets: ['example-host:1317']
+    params:
+      format: ['prometheus']

stack_orchestrator/data/config/monitoring/watcher-alert-rules.yml

@@ -59,29 +59,29 @@ groups:
               conditions:
                 - evaluator:
                     params:
-                      - 0
+                      - 15
                       - 0
                     type: gt
                   operator:
-                    type: and
+                    type: when
                   query:
-                    params: []
+                    params:
+                      - diff
                   reducer:
                     params: []
-                    type: avg
+                    type: last
                   type: query
               datasource:
                 name: Expression
                 type: __expr__
                 uid: __expr__
-              expression: ${diff} >= 16
-              intervalMs: 1000
-              maxDataPoints: 43200
+              expression: ""
+              hide: false
               refId: condition
-              type: math
+              type: classic_conditions
         noDataState: Alerting
         execErrState: Alerting
-        for: 15m
+        for: 5m
         annotations:
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
@@ -135,29 +135,29 @@ groups:
               conditions:
                 - evaluator:
                     params:
-                      - 0
+                      - 15
                       - 0
                     type: gt
                   operator:
-                    type: and
+                    type: when
                   query:
-                    params: []
+                    params:
+                      - diff
                   reducer:
                     params: []
-                    type: avg
+                    type: last
                   type: query
               datasource:
                 name: Expression
                 type: __expr__
                 uid: __expr__
-              expression: ${diff} >= 16
-              intervalMs: 1000
-              maxDataPoints: 43200
+              expression: ""
+              hide: false
               refId: condition
-              type: math
+              type: classic_conditions
         noDataState: Alerting
         execErrState: Alerting
-        for: 15m
+        for: 5m
         annotations:
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
@@ -202,7 +202,7 @@ groups:
               legendFormat: __auto
               range: false
               refId: latest_external
-          - refId: condition
+           - refId: condition
             relativeTimeRange:
               from: 600
               to: 0
@@ -211,29 +211,29 @@ groups:
               conditions:
                 - evaluator:
                     params:
-                      - 0
+                      - 15
                       - 0
                     type: gt
                   operator:
-                    type: and
+                    type: when
                   query:
-                    params: []
+                    params:
+                      - diff
                   reducer:
                     params: []
-                    type: avg
+                    type: last
                   type: query
               datasource:
                 name: Expression
                 type: __expr__
                 uid: __expr__
-              expression: ${diff} >= 16
-              intervalMs: 1000
-              maxDataPoints: 43200
+              expression: ""
+              hide: false
               refId: condition
-              type: math
+              type: classic_conditions
         noDataState: Alerting
         execErrState: Alerting
-        for: 15m
+        for: 5m
         annotations:
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
@@ -278,7 +278,7 @@ groups:
               legendFormat: __auto
               range: false
               refId: latest_external
-          - refId: condition
+           - refId: condition
             relativeTimeRange:
               from: 600
               to: 0
@@ -287,29 +287,29 @@ groups:
               conditions:
                 - evaluator:
                     params:
-                      - 0
+                      - 15
                       - 0
                     type: gt
                   operator:
-                    type: and
+                    type: when
                   query:
-                    params: []
+                    params:
+                      - diff
                   reducer:
                     params: []
-                    type: avg
+                    type: last
                   type: query
               datasource:
                 name: Expression
                 type: __expr__
                 uid: __expr__
-              expression: ${diff} >= 16
-              intervalMs: 1000
-              maxDataPoints: 43200
+              expression: ""
+              hide: false
               refId: condition
-              type: math
+              type: classic_conditions
         noDataState: Alerting
         execErrState: Alerting
-        for: 15m
+        for: 5m
         annotations:
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
@@ -363,29 +363,29 @@ groups:
               conditions:
                 - evaluator:
                     params:
-                      - 0
+                      - 15
                       - 0
                     type: gt
                   operator:
-                    type: and
+                    type: when
                   query:
-                    params: []
+                    params:
+                      - diff
                   reducer:
                     params: []
-                    type: avg
+                    type: last
                   type: query
               datasource:
                 name: Expression
                 type: __expr__
                 uid: __expr__
-              expression: ${diff} >= 16
-              intervalMs: 1000
-              maxDataPoints: 43200
+              expression: ""
+              hide: false
               refId: condition
-              type: math
+              type: classic_conditions
         noDataState: Alerting
         execErrState: Alerting
-        for: 15m
+        for: 5m
         annotations:
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
@@ -439,29 +439,29 @@ groups:
               conditions:
                 - evaluator:
                     params:
-                      - 0
+                      - 15
                       - 0
                     type: gt
                   operator:
-                    type: and
+                    type: when
                   query:
-                    params: []
+                    params:
+                      - diff
                   reducer:
                     params: []
-                    type: avg
+                    type: last
                   type: query
               datasource:
                 name: Expression
                 type: __expr__
                 uid: __expr__
-              expression: ${diff} >= 16
-              intervalMs: 1000
-              maxDataPoints: 43200
+              expression: ""
+              hide: false
               refId: condition
-              type: math
+              type: classic_conditions
         noDataState: Alerting
         execErrState: Alerting
-        for: 15m
+        for: 5m
         annotations:
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
@@ -515,29 +515,29 @@ groups:
               conditions:
                 - evaluator:
                     params:
-                      - 0
+                      - 15
                       - 0
                     type: gt
                   operator:
-                    type: and
+                    type: when
                   query:
-                    params: []
+                    params:
+                      - diff
                   reducer:
                     params: []
-                    type: avg
+                    type: last
                   type: query
               datasource:
                 name: Expression
                 type: __expr__
                 uid: __expr__
-              expression: ${diff} >= 16
-              intervalMs: 1000
-              maxDataPoints: 43200
+              expression: ""
+              hide: false
               refId: condition
-              type: math
+              type: classic_conditions
         noDataState: Alerting
         execErrState: Alerting
-        for: 15m
+        for: 5m
         annotations:
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
@@ -591,29 +591,29 @@ groups:
               conditions:
                 - evaluator:
                     params:
-                      - 0
+                      - 15
                       - 0
                     type: gt
                   operator:
-                    type: and
+                    type: when
                   query:
-                    params: []
+                    params:
+                      - diff
                   reducer:
                     params: []
-                    type: avg
+                    type: last
                   type: query
               datasource:
                 name: Expression
                 type: __expr__
                 uid: __expr__
-              expression: ${diff} >= 16
-              intervalMs: 1000
-              maxDataPoints: 43200
+              expression: ""
+              hide: false
               refId: condition
-              type: math
+              type: classic_conditions
         noDataState: Alerting
         execErrState: Alerting
-        for: 15m
+        for: 5m
         annotations:
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
@@ -669,29 +669,29 @@ groups:
               conditions:
                 - evaluator:
                     params:
-                      - 0
+                      - 15
                       - 0
                     type: gt
                   operator:
-                    type: and
+                    type: when
                   query:
-                    params: []
+                    params:
+                      - diff
                   reducer:
                     params: []
-                    type: avg
+                    type: last
                   type: query
               datasource:
                 name: Expression
                 type: __expr__
                 uid: __expr__
-              expression: ${diff} >= 16
-              intervalMs: 1000
-              maxDataPoints: 43200
+              expression: ""
+              hide: false
               refId: condition
-              type: math
+              type: classic_conditions
         noDataState: Alerting
         execErrState: Alerting
-        for: 15m
+        for: 5m
         annotations:
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false
@@ -745,29 +745,107 @@ groups:
               conditions:
                 - evaluator:
                     params:
-                      - 0
+                      - 15
                       - 0
                     type: gt
                   operator:
-                    type: and
+                    type: when
                   query:
-                    params: []
+                    params:
+                      - diff
                   reducer:
                     params: []
-                    type: avg
+                    type: last
                   type: query
               datasource:
                 name: Expression
                 type: __expr__
                 uid: __expr__
-              expression: ${diff} >= 16
-              intervalMs: 1000
-              maxDataPoints: 43200
+              expression: ""
+              hide: false
               refId: condition
-              type: math
+              type: classic_conditions
         noDataState: Alerting
         execErrState: Alerting
-        for: 15m
+        for: 5m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+
+      # Ajna
+      - uid: ajna_diff_external
+        title: ajna_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="ajna", instance="ajna", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 15
+                      - 0
+                    type: gt
+                  operator:
+                    type: when
+                  query:
+                    params:
+                      - diff
+                  reducer:
+                    params: []
+                    type: last
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ""
+              hide: false
+              refId: condition
+              type: classic_conditions
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 5m
         annotations:
           summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
         isPaused: false

stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/Dockerfile

@@ -1,26 +1,23 @@
-FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen
+FROM ethpandaops/ethereum-genesis-generator:3.0.0 AS ethgen
 
 FROM golang:1.20-alpine as builder
 
-RUN apk add --no-cache python3 py3-pip
-
-COPY genesis /opt/genesis
+RUN apk add --no-cache python3 py3-pip make bash envsubst jq
 
 # Install ethereum-genesis-generator tools
-COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/
-COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/
 COPY --from=ethgen /apps /apps
 RUN cd /apps/el-gen && pip3 install --break-system-packages -r requirements.txt
-# web3==5.24.0 used by el-gen is broken on python 3.11
-RUN pip3 install --break-system-packages --upgrade "web3==6.5.0"
+RUN pip3 install --break-system-packages --upgrade "web3==v6.15.1"
 RUN pip3 install --break-system-packages --upgrade "typing-extensions"
 
+# Install tool to generate initial block
+RUN go install github.com/cerc-io/eth-dump-genblock@b29516740fc01cf1d1d623acbfd0e9a2b6440a96
+
 # Build genesis config
-RUN apk add --no-cache make bash envsubst jq
+COPY genesis /opt/genesis
 RUN cd /opt/genesis && make genesis-el
 
 # Snag the genesis block info.
-RUN go install github.com/cerc-io/eth-dump-genblock@latest
 RUN eth-dump-genblock /opt/genesis/build/el/geth.json > /opt/genesis/build/el/genesis_block.json
 
 FROM alpine:latest

stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/genesis/el/build_el.sh

@@ -9,32 +9,8 @@ mkdir -p ../build/el
 tmp_dir=$(mktemp -d -t ci-XXXXXXXXXX)
 envsubst < el-config.yaml > $tmp_dir/genesis-config.yaml
 
-ttd=`cat $tmp_dir/genesis-config.yaml | grep terminal_total_difficulty | awk '{ print $2 }'`
-homestead_block=`cat $tmp_dir/genesis-config.yaml | grep homestead_block | awk '{ print $2 }'`
-eip150_block=`cat $tmp_dir/genesis-config.yaml | grep eip150_block | awk '{ print $2 }'`
-eip155_block=`cat $tmp_dir/genesis-config.yaml | grep eip155_block | awk '{ print $2 }'`
-eip158_block=`cat $tmp_dir/genesis-config.yaml | grep eip158_block | awk '{ print $2 }'`
-byzantium_block=`cat $tmp_dir/genesis-config.yaml | grep byzantium_block | awk '{ print $2 }'`
-constantinople_block=`cat $tmp_dir/genesis-config.yaml | grep constantinople_block | awk '{ print $2 }'`
-petersburg_block=`cat $tmp_dir/genesis-config.yaml | grep petersburg_block | awk '{ print $2 }'`
-istanbul_block=`cat $tmp_dir/genesis-config.yaml | grep istanbul_block | awk '{ print $2 }'`
-berlin_block=`cat $tmp_dir/genesis-config.yaml | grep berlin_block | awk '{ print $2 }'`
-london_block=`cat $tmp_dir/genesis-config.yaml | grep london_block | awk '{ print $2 }'`
-merge_fork_block=`cat $tmp_dir/genesis-config.yaml | grep merge_fork_block | awk '{ print $2 }'`
-
 python3 /apps/el-gen/genesis_geth.py $tmp_dir/genesis-config.yaml | \
-  jq ".config.terminalTotalDifficulty=$ttd" | \
-  jq ".config.homesteadBlock=$homestead_block" | \
-  jq ".config.eip150Block=$eip150_block" | \
-  jq ".config.eip155Block=$eip155_block" | \
-  jq ".config.eip158Block=$eip158_block" | \
-  jq ".config.byzantiumBlock=$byzantium_block" | \
-  jq ".config.constantinopleBlock=$constantinople_block" | \
-  jq ".config.petersburgBlock=$petersburg_block" | \
-  jq ".config.istanbulBlock=$istanbul_block" | \
-  jq ".config.berlinBlock=$berlin_block" | \
-  jq ".config.londonBlock=$london_block" | \
-  jq ".config.mergeForkBlock=$merge_fork_block" | \
-  jq ".config.mergeNetsplitBlock=$merge_fork_block" \
+  jq 'del(.config.pragueTime)' \
   > ../build/el/geth.json
+
 python3 ../accounts/mnemonic_to_csv.py $tmp_dir/genesis-config.yaml > ../build/el/accounts.csv

stack_orchestrator/data/container-build/cerc-fixturenet-eth-genesis/genesis/el/el-config.yaml

@@ -10,22 +10,8 @@ el_premine_addrs: {}
 chain_id: 1212
 deposit_contract_address: "0x1212121212121212121212121212121212121212"
 genesis_timestamp: 0
-terminal_total_difficulty: 1000
-homestead_block: 1
-eip150_block: 1
-eip155_block: 1
-eip158_block: 1
-byzantium_block: 1
-constantinople_block: 1
-petersburg_block: 1
-istanbul_block: 1
-berlin_block: 1
-london_block: 1
-merge_fork_block: 1
-
-clique:
-  enabled: false
-  signers:
-    - 36d56343bc308d4ffaac2f793d121aba905fa6cc
-    - 5e762d4a3847cadaf40a4b0c39574b0ff6698c78
-    - 15d7acc1019fdf8ab4f0f7bd31ec1487ecb5a2bd
+genesis_delay: 0
+deneb_fork_epoch: 0
+# note: only needed as workaround https://github.com/ethpandaops/ethereum-genesis-generator/pull/105
+electra_fork_epoch: 0
+slot_duration_in_seconds: 3

stack_orchestrator/data/container-build/cerc-fixturenet-eth-geth/run-el.sh

@@ -6,7 +6,7 @@ fi
 
 ETHERBASE=`cat /opt/testnet/build/el/accounts.csv | head -1 | cut -d',' -f2`
 NETWORK_ID=`cat /opt/testnet/el/el-config.yaml | grep 'chain_id' | awk '{ print $2 }'`
-NETRESTRICT=`ip addr | grep inet | grep -v '127.0' | awk '{print $2}'`
+NETRESTRICT=`ip addr | grep -w inet | grep -v '127.0' | awk '{print $2}'`
 CERC_ETH_DATADIR="${CERC_ETH_DATADIR:-$HOME/ethdata}"
 CERC_PLUGINS_DIR="${CERC_PLUGINS_DIR:-/usr/local/lib/plugeth}"
 
@@ -102,6 +102,13 @@ else
       fi
     fi
 
+    OTHER_OPTS=""
+    # miner options were removed in v1.12
+    GETH_VERSION=$(geth --version | grep -io '[0-9][0-9a-z.-]*')
+    if echo -e "$GETH_VERSION\n1.12" | sort -Vc; then
+      OTHER_OPTS="--miner.threads=1"
+    fi
+
     $START_CMD \
       --datadir="${CERC_ETH_DATADIR}" \
       --bootnodes="${ENODE}" \
@@ -126,12 +133,12 @@ else
       --cache.preimages \
       --syncmode=full \
       --mine \
-      --miner.threads=1 \
       --metrics \
       --metrics.addr="0.0.0.0" \
       --verbosity=${CERC_GETH_VERBOSITY:-3} \
       --log.vmodule="${CERC_GETH_VMODULE:-statediff/*=5}" \
       --miner.etherbase="${ETHERBASE}" \
+      ${OTHER_OPTS} \
       ${STATEDIFF_OPTS} \
       &
 

stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/Dockerfile

@@ -1,5 +1,4 @@
 FROM cerc/lighthouse-cli:local AS lcli
-FROM skylenet/ethereum-genesis-generator@sha256:210353ce7c898686bc5092f16c61220a76d357f51eff9c451e9ad1b9ad03d4d3 AS ethgen
 FROM cerc/fixturenet-eth-genesis:local AS fnetgen
 
 FROM cerc/lighthouse:local
@@ -12,16 +11,13 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-reco
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 
-COPY genesis /opt/testnet
-COPY run-cl.sh /opt/testnet/run.sh
-
 COPY --from=lcli /usr/local/bin/lcli /usr/local/bin/lcli
-COPY --from=ethgen /usr/local/bin/eth2-testnet-genesis /usr/local/bin/eth2-testnet-genesis
-COPY --from=ethgen /usr/local/bin/eth2-val-tools /usr/local/bin/eth2-val-tools
-COPY --from=ethgen /apps /apps
 COPY --from=fnetgen /opt/genesis/el /opt/testnet/el
 COPY --from=fnetgen /opt/genesis/build/el /opt/testnet/build/el
 
+COPY genesis /opt/testnet
+COPY run-cl.sh /opt/testnet/run.sh
+
 RUN cd /opt/testnet && make genesis-cl
 
 # Work around some bugs in lcli where the default path is always used.

stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/beacon_node.sh

@@ -10,7 +10,6 @@ set -Eeuo pipefail
 source ./vars.env
 
 SUBSCRIBE_ALL_SUBNETS=
-DEBUG_LEVEL=${DEBUG_LEVEL:-debug}
 
 # Get positional arguments
 data_dir=$DATADIR/node_${NODE_NUMBER}

stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/bootnode.sh

@@ -9,8 +9,6 @@ set -Eeuo pipefail
 
 source ./vars.env
 
-DEBUG_LEVEL=${1:-info}
-
 echo "Starting bootnode"
 
 # Clean up existing ENR dir to avoid node connectivity issues on a restart

stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/build_cl.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+# See https://github.com/sigp/lighthouse/scripts/local_testnet/setup.sh
 #
 # Deploys the deposit contract and makes deposits for $VALIDATOR_COUNT insecure deterministic validators.
 # Produces a testnet specification and a genesis state where the genesis time
@@ -24,23 +25,26 @@ echo "(Note: errors of the form 'WARN: Scrypt parameters are too weak...' below
 lcli \
   new-testnet \
   --spec $SPEC_PRESET \
-  --deposit-contract-address $ETH1_DEPOSIT_CONTRACT_ADDRESS \
   --testnet-dir $TESTNET_DIR \
+  --deposit-contract-address $ETH1_DEPOSIT_CONTRACT_ADDRESS \
   --min-genesis-active-validator-count $GENESIS_VALIDATOR_COUNT \
   --validator-count $VALIDATOR_COUNT \
   --min-genesis-time $GENESIS_TIME \
   --genesis-delay $GENESIS_DELAY \
   --genesis-fork-version $GENESIS_FORK_VERSION \
   --altair-fork-epoch $ALTAIR_FORK_EPOCH \
-  --bellatrix-fork-epoch $MERGE_FORK_EPOCH \
+  --bellatrix-fork-epoch $BELLATRIX_FORK_EPOCH \
+  --capella-fork-epoch $CAPELLA_FORK_EPOCH \
+  --deneb-fork-epoch $DENEB_FORK_EPOCH \
   --eth1-id $ETH1_CHAIN_ID \
   --eth1-block-hash $ETH1_BLOCK_HASH \
   --eth1-follow-distance 1 \
   --seconds-per-slot $SECONDS_PER_SLOT \
   --seconds-per-eth1-block $SECONDS_PER_ETH1_BLOCK \
+  --interop-genesis-state \
   --force
 
-echo Specification generated at $TESTNET_DIR.
+echo Specification and genesis.ssz generated at $TESTNET_DIR.
 echo "Generating $VALIDATOR_COUNT validators concurrently... (this may take a while)"
 
 lcli \
@@ -50,13 +54,3 @@ lcli \
   --node-count $BN_COUNT
 
 echo Validators generated with keystore passwords at $DATADIR.
-echo "Building genesis state... (this might take a while)"
-
-lcli \
-  interop-genesis \
-  --spec $SPEC_PRESET \
-  --genesis-time $GENESIS_TIME \
-  --testnet-dir $TESTNET_DIR \
-  $GENESIS_VALIDATOR_COUNT
-
-echo Created genesis state in $TESTNET_DIR

stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/validator_client.sh

@@ -8,8 +8,6 @@ set -Eeuo pipefail
 
 source ./vars.env
 
-DEBUG_LEVEL=info
-
 BUILDER_PROPOSALS=
 
 # Get options

stack_orchestrator/data/container-build/cerc-fixturenet-eth-lighthouse/genesis/cl/vars.env

@@ -25,7 +25,9 @@ BOOTNODE_PORT=${BOOTNODE_PORT:-4242}
 
 # Hard fork configuration
 ALTAIR_FORK_EPOCH=${ALTAIR_FORK_EPOCH:-0}
-MERGE_FORK_EPOCH=${MERGE_FORK_EPOCH:-0}
+BELLATRIX_FORK_EPOCH=${BELLATRIX_FORK_EPOCH:-0}
+CAPELLA_FORK_EPOCH=${CAPELLA_FORK_EPOCH:-0}
+DENEB_FORK_EPOCH=${DENEB_FORK_EPOCH:-0}
 
 # Spec version (mainnet or minimal)
 SPEC_PRESET=${SPEC_PRESET:-mainnet}
@@ -51,3 +53,6 @@ ETH1_TTD=${ETH1_TTD:-`cat $ETH1_GENESIS_JSON | jq -r '.config.terminalTotalDiffi
 ETH1_DEPOSIT_CONTRACT_ADDRESS=${ETH1_DEPOSIT_CONTRACT_ADDRESS:-`cat $ETH1_CONFIG_YAML | grep 'deposit_contract_address' | awk '{ print $2 }' | sed 's/"//g'`}
 ETH1_DEPOSIT_CONTRACT_BLOCK=${ETH1_DEPOSIT_CONTRACT_BLOCK:-0x0}
 SUGGESTED_FEE_RECIPIENT=`cat ../build/el/accounts.csv | head -1 | cut -d',' -f2`
+
+# --debug-level
+DEBUG_LEVEL=${LIGHTHOUSE_DEBUG_LEVEL:-debug}

stack_orchestrator/data/container-build/cerc-lighthouse/Dockerfile

@@ -1,9 +1,10 @@
-ARG TAG_SUFFIX="-modern"
-FROM sigp/lighthouse:v4.3.0${TAG_SUFFIX}
+FROM sigp/lighthouse:v5.1.2
 
-RUN apt-get update; apt-get install bash netcat curl less jq wget -y;
+RUN apt-get update && apt-get -y upgrade \
+    && apt-get -y install bash netcat curl less jq wget \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
 
-WORKDIR /root/
+WORKDIR /root
 ADD start-lighthouse.sh .
 
 ENTRYPOINT [ "./start-lighthouse.sh" ]

stack_orchestrator/data/container-build/cerc-lighthouse/build.sh

@@ -6,4 +6,4 @@ source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 # See: https://stackoverflow.com/a/246128/1701505
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-docker build -t cerc/lighthouse:local ${build_command_args} --build-arg TAG_SUFFIX="" ${SCRIPT_DIR}
+docker build -t cerc/lighthouse:local ${build_command_args} ${SCRIPT_DIR}

stack_orchestrator/data/stacks/fixturenet-blast/README.md

@@ -0,0 +1,26 @@
+# Blast stack
+
+## Clone required repositories
+```
+$ laconic-so --stack fixturenet-blast setup-repositories
+```
+## Build the stack's containers
+```
+$ laconic-so --stack fixturenet-blast build-containers
+```
+## Create a deployment of the stack
+```
+$ laconic-so --stack fixturenet-blast deploy init --map-ports-to-host any-same --output blast-spec.yml
+```
+[Insert details on how to configure the stack]
+```
+$ laconic-so --stack fixturenet-blast deploy create --deployment-dir blast-deployment --spec-file blast-spec.yml
+```
+## Start the stack
+```
+$ laconic-so deployment --dir blast-deployment start
+```
+Check logs:
+```
+$ laconic-so deployment --dir blast-deployment logs
+```

stack_orchestrator/data/stacks/fixturenet-blast/stack.yml

@@ -0,0 +1,17 @@
+version: "1.0"
+name: fixturenet-blast
+description: "A blast devnet stack"
+repos:
+  - github.com/blast-io/blast
+  - git.vdb.to/cerc-io/lighthouse
+containers:
+  - cerc/webapp-base
+  - cerc/lighthouse
+  - cerc/lighthouse-cli
+  - cerc/foundry
+  - cerc/fixturenet-eth-lighthouse
+
+pods:
+  - fixturenet-blast
+  - foundry
+  

stack_orchestrator/data/stacks/mainnet-blast/README.md

@@ -0,0 +1,26 @@
+# Blast stack
+
+## Clone required repositories
+```
+$ laconic-so --stack mainnet-blast setup-repositories
+```
+## Build the stack's containers
+```
+$ laconic-so --stack mainnet-blast build-containers
+```
+## Create a deployment of the stack
+```
+$ laconic-so --stack mainnet-blast deploy init --map-ports-to-host any-same --output blast-spec.yml
+```
+[Insert details on how to configure the stack]
+```
+$ laconic-so --stack mainnet-blast deploy create --deployment-dir blast-deployment --spec-file blast-spec.yml
+```
+## Start the stack
+```
+$ laconic-so deployment --dir blast-deployment start
+```
+Check logs:
+```
+$ laconic-so deployment --dir blast-deployment logs
+```

stack_orchestrator/data/stacks/mainnet-blast/deploy/commands.py

@@ -0,0 +1,39 @@
+# Copyright © 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+
+from pathlib import Path
+from shutil import copy
+import yaml
+
+
+def create(context, extra_args):
+    # Our goal here is just to copy the json files for blast
+    yml_path = context.deployment_dir.joinpath("spec.yml")
+    with open(yml_path, 'r') as file:
+        data = yaml.safe_load(file)
+
+    mount_point = data['volumes']['blast-data']
+    if mount_point[0] == "/":
+        deploy_dir = Path(mount_point)
+    else:
+        deploy_dir = context.deployment_dir.joinpath(mount_point)
+
+    command_context = extra_args[2]
+    compose_file = [f for f in command_context.cluster_context.compose_files if "mainnet-blast" in f][0]
+    source_config_file = Path(compose_file).parent.parent.joinpath("config", "mainnet-blast", "genesis.json")
+    copy(source_config_file, deploy_dir)
+    source_config_file = Path(compose_file).parent.parent.joinpath("config", "mainnet-blast", "rollup.json")
+    copy(source_config_file, deploy_dir)

stack_orchestrator/data/stacks/mainnet-blast/stack.yml

@@ -0,0 +1,12 @@
+version: "1.0"
+name: mainnet-blast
+description: "A blast stack"
+repos:
+  - github.com/blast-io/blast
+  - git.vdb.to/cerc-io/lighthouse
+containers:
+  - cerc/webapp-base
+  - cerc/lighthouse
+  - cerc/lighthouse-cli
+pods:
+  - mainnet-blast

stack_orchestrator/data/stacks/monitoring/README.md

@@ -4,6 +4,7 @@
 * Comes with the following built-in exporters / dashboards:
   * Chain Head Exporter - for tracking chain heads given external ETH RPC endpoints
   * Watchers dashboard
+  * laconicd dashboard
   * [Prometheus Blackbox](https://grafana.com/grafana/dashboards/7587-prometheus-blackbox-exporter/) - for tracking HTTP endpoints
   * [NodeJS Application Dashboard](https://grafana.com/grafana/dashboards/11159-nodejs-application-dashboard/) - for default NodeJS metrics
   * [PostgreSQL Database](https://grafana.com/grafana/dashboards/9628-postgresql-database/) - for monitoring Postgres dbs
@@ -99,6 +100,7 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --de
       - targets:
         - <HTTP_ENDPOINT_1>
         - <HTTP_ENDPOINT_2>
+        - <LACONICD_GQL_ENDPOINT>
   ```
 
 * Postgres (in-stack exporter):
@@ -116,6 +118,17 @@ laconic-so --stack monitoring deploy create --spec-file monitoring-spec.yml --de
     ```
   * Add database credentials to be used in `auth_modules` in the postgres-exporter config file (`monitoring-deployment/config/monitoring/postgres-exporter.yml`)
 
+* laconicd: update the `laconicd` job with a laconicd node's REST endpoint host and port:
+
+  ```yml
+  ...
+  - job_name: laconicd
+    ...
+    static_configs:
+      - targets: ['example-host:1317']
+  ...
+  ```
+
 Note: Use `host.docker.internal` as host to access ports on the host machine
 
 ### Grafana Config

stack_orchestrator/data/stacks/monitoring/monitoring-watchers.md

@@ -44,8 +44,18 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
   - job_name: 'blackbox'
     ...
     static_configs:
-      - targets:
-        - <AZIMUTH_GATEWAY_GQL_ENDPOINT>
+      - targets: [<AZIMUTH_GATEWAY_GQL_ENDPOINT>]
+        labels:
+          # Add destination label for pre-configured alerts
+          destination: 'azimuth_gateway'
+      - targets: [<LACONICD_GQL_ENDPOINT>]
+        labels:
+          destination: 'laconicd_gql'
+  ...
+  - job_name: laconicd
+    ...
+    static_configs:
+      - targets: ['LACONICD_REST_HOST:LACONICD_REST_PORT']
   ...
   - job_name: azimuth
     scrape_interval: 10s
@@ -98,16 +108,28 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
         labels:
           instance: 'merkl_sushiswap'
           chain: 'filecoin'
+
+  - job_name: ajna
+    scrape_interval: 20s
+    metrics_path: /metrics
+    scheme: http
+    static_configs:
+      - targets: ['AJNA_WATCHER_HOST:AJNA_WATCHER_PORT']
+        labels:
+          instance: 'ajna'
+          chain: 'filecoin'
   ```
 
 Add scrape config as done above for any additional watcher to add it to the Watchers dashboard.
 
 ### Grafana alerts config
 
-Place the pre-configured watcher alerts rules in Grafana provisioning directory:
+Place the pre-configured watcher and blackbox endpoint alerts rules in Grafana provisioning directory:
 
   ```bash
   cp monitoring-watchers-deployment/config/monitoring/watcher-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
+
+  cp monitoring-watchers-deployment/config/monitoring/blackbox-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
   ```
 
 Update the alerting contact points config (`monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/contactpoints.yml`) with desired contact points
@@ -120,7 +142,7 @@ Add corresponding routes to the notification policies config (`monitoring-watche
       - receiver: SlackNotifier
       object_matchers:
         # Add matchers below
-        - ['grafana_folder', '=', 'WatcherAlerts']
+        - ['grafana_folder', '=~', 'WatcherAlerts|BlackboxAlerts']
   ```
 
 ### Env

tests/fixturenet-laconicd/run-cli-test.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+set -e
+if [ -n "$CERC_SCRIPT_DEBUG" ]; then
+  set -x
+fi
+
+echo "$(date +"%Y-%m-%d %T"): Running stack-orchestrator Laconic registry CLI tests"
+env
+cat /etc/hosts
+# Bit of a hack, test the most recent package
+TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
+
+export CERC_REPO_BASE_DIR=$(mktemp -d $(pwd)/stack-orchestrator-fixturenet-laconicd-test.XXXXXXXXXX)
+echo "$(date +"%Y-%m-%d %T"): Cloning laconic-registry-cli repository into: $CERC_REPO_BASE_DIR"
+$TEST_TARGET_SO --stack fixturenet-laconicd setup-repositories --include git.vdb.to/cerc-io/laconic-registry-cli
+
+echo "$(date +"%Y-%m-%d %T"): Starting stack"
+TEST_AUCTION_ENABLED=true BASE_DIR=${CERC_REPO_BASE_DIR} $TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd up
+echo "$(date +"%Y-%m-%d %T"): Stack started"
+
+# Verify that the fixturenet is up and running
+$TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd ps
+
+# Get the fixturenet account address
+laconicd_account_address=$(docker exec laconicd-laconicd-1 laconicd keys list | awk '/- address:/ {print $3}')
+
+# Copy over config
+docker exec laconicd-cli-1 cp config.yml laconic-registry-cli/
+
+# Wait for the laconid endpoint to come up
+echo "Waiting for the RPC endpoint to come up"
+docker exec laconicd-laconicd-1 sh -c "curl --retry 20 --retry-delay 3 --retry-connrefused http://127.0.0.1:9473/api"
+
+# Run the tests
+echo "Running the tests"
+docker exec -e TEST_ACCOUNT=$laconicd_account_address laconicd-cli-1 sh -c 'cd laconic-registry-cli && yarn && yarn test'
+
+# Clean up
+$TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd down --delete-volumes
+echo "$(date +"%Y-%m-%d %T"): Removing cloned repositories"
+rm -rf $CERC_REPO_BASE_DIR
+echo "$(date +"%Y-%m-%d %T"): Test finished"

tests/fixturenet-laconicd/run-test.sh

@@ -12,6 +12,7 @@ cat /etc/hosts
 TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
 # Set a new unique repo dir
 export CERC_REPO_BASE_DIR=$(mktemp -d $(pwd)/stack-orchestrator-fixturenet-laconicd-test.XXXXXXXXXX)
+
 echo "$(date +"%Y-%m-%d %T"): Testing this package: $TEST_TARGET_SO"
 echo "$(date +"%Y-%m-%d %T"): Test version command"
 reported_version_string=$( $TEST_TARGET_SO version )

tests/webapp-test/run-webapp-test.sh

@@ -32,14 +32,14 @@ set +e
 
 CONTAINER_ID=$(docker run -p 3000:80 -d -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG cerc/test-progressive-web-app:local)
 sleep 3
-wget -t 7 -O test.before -m http://localhost:3000
+wget --tries 20 --retry-connrefused --waitretry=3 -O test.before -m http://localhost:3000
 
 docker logs $CONTAINER_ID
 docker remove -f $CONTAINER_ID
 
 CONTAINER_ID=$(docker run -p 3000:80 -e CERC_WEBAPP_DEBUG=$CHECK -e CERC_SCRIPT_DEBUG=$CERC_SCRIPT_DEBUG -d cerc/test-progressive-web-app:local)
 sleep 3
-wget -t 7 -O test.after -m http://localhost:3000
+wget --tries 20 --retry-connrefused --waitretry=3 -O test.after -m http://localhost:3000
 
 docker logs $CONTAINER_ID
 docker remove -f $CONTAINER_ID