Add git to webapp base container.

Merge pull request 'Fix mainnet laconic deploy setup' (#850 ) from dboreham/fix-mainnet-laconic-init into main
Reviewed-on: cerc-io/stack-orchestrator#850
2024-06-24 23:57:15 -05:00 · 2024-06-22 01:25:07 +00:00 · 2024-06-21 19:24:33 -06:00 · 2024-06-20 09:27:23 +00:00 · 2024-06-19 10:40:54 +00:00 · 2024-06-19 06:41:58 +00:00
67 changed files with 3949 additions and 716 deletions
@@ -9,10 +9,6 @@ on:
  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
    - cron: '2 14 * * *'

-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-

 jobs:
  test:
@@ -9,10 +9,6 @@ on:
  schedule: # Note: coordinate with other tests to not overload runners at the same time of day
    - cron: '2 14 * * *'

-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-

 jobs:
  test:
@@ -41,10 +37,6 @@ jobs:
        run: ./scripts/create_build_tag_file.sh
      - name: "Build local shiv package"
        run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
      - name: "Run fixturenet-eth tests"
        run: ./tests/fixturenet-eth-plugeth/run-test.sh
      - name: Notify Vulcanize Slack on CI failure
@@ -7,10 +7,6 @@ on:
      - '!**'
      - '.gitea/workflows/triggers/fixturenet-eth-test'

-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-

 jobs:
  test:
@@ -39,10 +35,6 @@ jobs:
        run: ./scripts/create_build_tag_file.sh
      - name: "Build local shiv package"
        run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
      - name: "Run fixturenet-eth tests"
        run: ./tests/fixturenet-eth/run-test.sh
      - name: Notify Vulcanize Slack on CI failure
@@ -10,9 +10,6 @@ on:
    paths-ignore:
      - '.gitea/workflows/triggers/*'

-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock

 jobs:
  test:
@@ -41,10 +38,6 @@ jobs:
        run: ./scripts/create_build_tag_file.sh
      - name: "Build local shiv package"
        run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
      - name: "Run deploy tests"
        run: ./tests/deploy/run-deploy-test.sh
      - name: Notify Vulcanize Slack on CI failure
@@ -10,10 +10,6 @@ on:
    paths-ignore:
      - '.gitea/workflows/triggers/*'

-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock
-
 jobs:
  test:
    name: "Run webapp test suite"
@@ -43,10 +39,6 @@ jobs:
        run: ./scripts/build_shiv_package.sh
      - name: "Install wget" # 20240109 - Only needed until the executors are updated.
        run: apt update && apt install -y wget
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
      - name: "Run webapp tests"
        run: ./tests/webapp-test/run-webapp-test.sh
      - name: Notify Vulcanize Slack on CI failure
@@ -10,9 +10,6 @@ on:
    paths-ignore:
      - '.gitea/workflows/triggers/*'

-# Needed until we can incorporate docker startup into the executor container
-env:
-  DOCKER_HOST: unix:///var/run/dind.sock

 jobs:
  test:
@@ -41,10 +38,6 @@ jobs:
        run: ./scripts/create_build_tag_file.sh
      - name: "Build local shiv package"
        run: ./scripts/build_shiv_package.sh
-      - name: Start dockerd # Also needed until we can incorporate into the executor
-        run: |
-          dockerd -H $DOCKER_HOST --userland-proxy=false &
-          sleep 5
      - name: "Run smoke tests"
        run: ./tests/smoke-test/run-smoke-test.sh
      - name: Notify Vulcanize Slack on CI failure
@@ -4,3 +4,4 @@ Trigger
 Trigger
 Trigger
 Trigger
+Trigger
@@ -3,7 +3,7 @@ services:
    restart: always
    hostname: fixturenet-eth-bootnode-geth
    env_file:
-      - ../config/fixturenet-plugeth/fixturenet-eth.env
+      - ../config/fixturenet-eth/fixturenet-eth.env
    environment:
      RUN_BOOTNODE: "true"
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
@@ -24,9 +24,8 @@ services:
      CERC_RUN_STATEDIFF: ${CERC_RUN_STATEDIFF:-detect}
      CERC_STATEDIFF_DB_NODE_ID: 1
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ALLOW_UNPROTECTED_TXS: ${CERC_ALLOW_UNPROTECTED_TXS:-false}
    env_file:
-      - ../config/fixturenet-plugeth/fixturenet-eth.env
+      - ../config/fixturenet-eth/fixturenet-eth.env
    image: cerc/fixturenet-plugeth-plugeth:local
    volumes:
      - fixturenet_plugeth_geth_1_data:/root/ethdata
@@ -56,7 +55,7 @@ services:
    environment:
      CERC_KEEP_RUNNING_AFTER_GETH_EXIT: "true"
    env_file:
-      - ../config/fixturenet-plugeth/fixturenet-eth.env
+      - ../config/fixturenet-eth/fixturenet-eth.env
    image: cerc/fixturenet-plugeth-plugeth:local
    depends_on:
      - fixturenet-eth-bootnode-geth
@@ -83,7 +82,7 @@ services:
      retries: 10
      start_period: 30s
    env_file:
-      - ../config/fixturenet-plugeth/fixturenet-eth.env
+      - ../config/fixturenet-eth/fixturenet-eth.env
    environment:
      NODE_NUMBER: "1"
      ETH1_ENDPOINT: "http://fixturenet-eth-geth-1:8545"
@@ -109,7 +108,7 @@ services:
      retries: 10
      start_period: 30s
    env_file:
-      - ../config/fixturenet-plugeth/fixturenet-eth.env
+      - ../config/fixturenet-eth/fixturenet-eth.env
    environment:
      NODE_NUMBER: "2"
      ETH1_ENDPOINT: "http://fixturenet-eth-geth-2:8545"
@@ -6,12 +6,20 @@ services:
    restart: always
    environment:
      GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL}
+      CERC_GRAFANA_ALERTS_SUBGRAPH_IDS: ${CERC_GRAFANA_ALERTS_SUBGRAPH_IDS}
    volumes:
      - ../config/monitoring/grafana/provisioning:/etc/grafana/provisioning
      - ../config/monitoring/grafana/dashboards:/etc/grafana/dashboards
+      - ../config/monitoring/update-grafana-alerts-config.sh:/update-grafana-alerts-config.sh
      - grafana_storage:/var/lib/grafana
+    user: root
+    entrypoint: ["bash", "-c"]
+    command: |
+      "/update-grafana-alerts-config.sh && /run.sh"
    ports:
      - "3000"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
    healthcheck:
      test: ["CMD", "nc", "-vz", "localhost", "3000"]
      interval: 30s
@@ -22,6 +22,7 @@ services:
      GRAPH_ETHEREUM_JSON_RPC_TIMEOUT: ${GRAPH_ETHEREUM_JSON_RPC_TIMEOUT:-180}
      GRAPH_ETHEREUM_REQUEST_RETRIES: ${GRAPH_ETHEREUM_REQUEST_RETRIES:-10}
      GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE: ${GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE:-2000}
+      GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS: ${GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS:-1000}
    entrypoint: ["bash", "-c"]
    # Wait for ETH RPC endpoint to be up when running with fixturenet-lotus
    command: |
@@ -31,6 +32,7 @@ services:
      - "8001"
      - "8020"
      - "8030"
+      - "8040"
    healthcheck:
      test: ["CMD", "nc", "-vz", "localhost", "8020"]
      interval: 30s
@@ -28,15 +28,37 @@ services:
    extra_hosts:
      - "host.docker.internal:host-gateway"

-  chain-head-exporter:
+  ethereum-chain-head-exporter:
    image: cerc/watcher-ts:local
    restart: always
    working_dir: /app/packages/cli
    environment:
-      ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
-      FIL_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT}
+      ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT:-https://mainnet.infura.io/v3}
      ETH_RPC_API_KEY: ${CERC_INFURA_KEY}
-      PORT: ${CERC_METRICS_PORT}
+    command: ["sh", "-c", "yarn export-metrics:chain-heads"]
+    ports:
+      - '5000'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  filecoin-chain-head-exporter:
+    image: cerc/watcher-ts:local
+    restart: always
+    working_dir: /app/packages/cli
+    environment:
+      ETH_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT:-https://api.node.glif.io/rpc/v1}
+    command: ["sh", "-c", "yarn export-metrics:chain-heads"]
+    ports:
+      - '5000'
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  graph-node-upstream-head-exporter:
+    image: cerc/watcher-ts:local
+    restart: always
+    working_dir: /app/packages/cli
+    environment:
+      ETH_RPC_ENDPOINT: ${GRAPH_NODE_RPC_ENDPOINT}
    command: ["sh", "-c", "yarn export-metrics:chain-heads"]
    ports:
      - '5000'
@@ -1,13 +0,0 @@
-services:
-  snowballtools-base-backend:
-    image: cerc/snowballtools-base-backend:local
-    restart: always
-    volumes:
-      - data:/data
-      - config:/config:ro
-    ports:
-      - 8000
-
-volumes:
-  data:
-  config:
@@ -29,7 +29,7 @@ services:
    image: cerc/watcher-ajna:local
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
    command: ["bash", "./start-job-runner.sh"]
    volumes:
      - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -37,7 +37,7 @@ services:
    ports:
      - "9000"
    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -55,16 +55,17 @@ services:
    image: cerc/watcher-ajna:local
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
    command: ["bash", "./start-server.sh"]
    volumes:
      - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
      - ../config/watcher-ajna/start-server.sh:/app/start-server.sh
+      - ajna_watcher_gql_logs_data:/app/gql-logs
    ports:
      - "3008"
      - "9001"
    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -74,3 +75,4 @@ services:

 volumes:
  ajna_watcher_db_data:
+  ajna_watcher_gql_logs_data:
@@ -32,8 +32,8 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
      CERC_HISTORICAL_BLOCK_RANGE: 500
      CONTRACT_ADDRESS: 0x223c067F8CF28ae173EE5CafEa60cA44C335fecB
      CONTRACT_NAME: Azimuth
@@ -47,7 +47,7 @@ services:
    ports:
      - "9000"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -66,18 +66,20 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
    working_dir: /app/packages/azimuth-watcher
    command: "./start-server.sh"
    volumes:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/azimuth-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/azimuth-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/azimuth-watcher/start-server.sh
+      - azimuth_watcher_gql_logs_data:/app/packages/azimuth-watcher/gql-logs
    ports:
      - "3001"
+      - "9001"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3001"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3001"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -94,8 +96,8 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
      CONTRACT_ADDRESS: 0x325f68d32BdEe6Ed86E7235ff2480e2A433D6189
      CONTRACT_NAME: Censures
      STARTING_BLOCK: 6784954
@@ -108,7 +110,7 @@ services:
    ports:
      - "9002"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9002"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9002"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -127,18 +129,20 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
    working_dir: /app/packages/censures-watcher
    command: "./start-server.sh"
    volumes:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/censures-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/censures-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/censures-watcher/start-server.sh
+      - censures_watcher_gql_logs_data:/app/packages/censures-watcher/gql-logs
    ports:
      - "3002"
+      - "9003"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3002"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3002"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -155,8 +159,8 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
      CONTRACT_ADDRESS: 0xe7e7f69b34D7d9Bd8d61Fb22C33b22708947971A
      CONTRACT_NAME: Claims
      STARTING_BLOCK: 6784941
@@ -169,7 +173,7 @@ services:
    ports:
      - "9004"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9004"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9004"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -188,18 +192,20 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
    working_dir: /app/packages/claims-watcher
    command: "./start-server.sh"
    volumes:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/claims-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/claims-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/claims-watcher/start-server.sh
+      - claims_watcher_gql_logs_data:/app/packages/claims-watcher/gql-logs
    ports:
      - "3003"
+      - "9005"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3003"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3003"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -216,8 +222,8 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
      CONTRACT_ADDRESS: 0x8C241098C3D3498Fe1261421633FD57986D74AeA
      CONTRACT_NAME: ConditionalStarRelease
      STARTING_BLOCK: 6828004
@@ -230,7 +236,7 @@ services:
    ports:
      - "9006"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9006"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9006"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -249,18 +255,20 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
    working_dir: /app/packages/conditional-star-release-watcher
    command: "./start-server.sh"
    volumes:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/conditional-star-release-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/conditional-star-release-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/conditional-star-release-watcher/start-server.sh
+      - conditional_star_release_watcher_gql_logs_data:/app/packages/conditional-star-release-watcher/gql-logs
    ports:
      - "3004"
+      - "9007"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3004"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3004"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -277,8 +285,8 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
      CONTRACT_ADDRESS: 0xf6b461fE1aD4bd2ce25B23Fe0aff2ac19B3dFA76
      CONTRACT_NAME: DelegatedSending
      STARTING_BLOCK: 6784956
@@ -291,7 +299,7 @@ services:
    ports:
      - "9008"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9008"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -310,18 +318,20 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
    working_dir: /app/packages/delegated-sending-watcher
    command: "./start-server.sh"
    volumes:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/delegated-sending-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/delegated-sending-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/delegated-sending-watcher/start-server.sh
+      - delegated_sending_watcher_gql_logs_data:/app/packages/delegated-sending-watcher/gql-logs
    ports:
      - "3005"
+      - "9009"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3005"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3005"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -338,8 +348,8 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
      CONTRACT_ADDRESS: 0x33EeCbf908478C10614626A9D304bfe18B78DD73
      CONTRACT_NAME: Ecliptic
      STARTING_BLOCK: 13692129
@@ -352,7 +362,7 @@ services:
    ports:
      - "9010"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9010"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9010"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -371,18 +381,20 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
    working_dir: /app/packages/ecliptic-watcher
    command: "./start-server.sh"
    volumes:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/ecliptic-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/ecliptic-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/ecliptic-watcher/start-server.sh
+      - ecliptic_watcher_gql_logs_data:/app/packages/ecliptic-watcher/gql-logs
    ports:
      - "3006"
+      - "9011"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3006"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3006"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -399,8 +411,8 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
      CONTRACT_ADDRESS: 0x86cd9cd0992F04231751E3761De45cEceA5d1801
      CONTRACT_NAME: LinearStarRelease
      STARTING_BLOCK: 6784943
@@ -413,7 +425,7 @@ services:
    ports:
      - "9012"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9012"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9012"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -432,18 +444,20 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
    working_dir: /app/packages/linear-star-release-watcher
    command: "./start-server.sh"
    volumes:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/linear-star-release-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/linear-star-release-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/linear-star-release-watcher/start-server.sh
+      - linear_star_release_watcher_gql_logs_data:/app/packages/linear-star-release-watcher/gql-logs
    ports:
      - "3007"
+      - "9013"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3007"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3007"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -460,8 +474,8 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
      CONTRACT_ADDRESS: 0x7fEcaB617c868Bb5996d99D95200D2Fa708218e4
      CONTRACT_NAME: Polls
      STARTING_BLOCK: 6784912
@@ -474,7 +488,7 @@ services:
    ports:
      - "9014"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "9014"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9014"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -493,18 +507,20 @@ services:
        condition: service_healthy
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
-      CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
+      CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
    working_dir: /app/packages/polls-watcher
    command: "./start-server.sh"
    volumes:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/polls-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/polls-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/polls-watcher/start-server.sh
+      - polls_watcher_gql_logs_data:/app/packages/polls-watcher/gql-logs
    ports:
      - "3008"
+      - "9015"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "3008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -542,7 +558,7 @@ services:
    ports:
      - "0.0.0.0:4000:4000"
    healthcheck:
-      test: ["CMD", "nc", "-vz", "localhost", "4000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "4000"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -552,3 +568,11 @@ services:

 volumes:
  watcher_db_data:
+  azimuth_watcher_gql_logs_data:
+  censures_watcher_gql_logs_data:
+  claims_watcher_gql_logs_data:
+  conditional_star_release_watcher_gql_logs_data:
+  delegated_sending_watcher_gql_logs_data:
+  ecliptic_watcher_gql_logs_data:
+  linear_star_release_watcher_gql_logs_data:
+  polls_watcher_gql_logs_data:
@@ -29,7 +29,7 @@ services:
    image: cerc/watcher-merkl-sushiswap-v3:local
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
    command: ["bash", "./start-job-runner.sh"]
    volumes:
      - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -37,7 +37,7 @@ services:
    ports:
      - "9002:9000"
    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -55,16 +55,17 @@ services:
    image: cerc/watcher-merkl-sushiswap-v3:local
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
    command: ["bash", "./start-server.sh"]
    volumes:
      - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
      - ../config/watcher-merkl-sushiswap-v3/start-server.sh:/app/start-server.sh
+      - merkl_sushiswap_v3_watcher_gql_logs_data:/app/gql-logs
    ports:
      - "127.0.0.1:3007:3008"
      - "9003:9001"
    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -74,3 +75,4 @@ services:

 volumes:
  merkl_sushiswap_v3_watcher_db_data:
+  merkl_sushiswap_v3_watcher_gql_logs_data:
@@ -29,7 +29,7 @@ services:
    image: cerc/watcher-sushiswap-v3:local
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
    command: ["bash", "./start-job-runner.sh"]
    volumes:
      - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -37,7 +37,7 @@ services:
    ports:
      - "9000:9000"
    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "9000"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -55,16 +55,17 @@ services:
    image: cerc/watcher-sushiswap-v3:local
    environment:
      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
-      CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
    command: ["bash", "./start-server.sh"]
    volumes:
      - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
      - ../config/watcher-sushiswap-v3/start-server.sh:/app/start-server.sh
+      - sushiswap_v3_watcher_gql_logs_data:/app/gql-logs
    ports:
      - "127.0.0.1:3008:3008"
      - "9001:9001"
    healthcheck:
-      test: ["CMD", "nc", "-v", "localhost", "3008"]
+      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
      interval: 20s
      timeout: 5s
      retries: 15
@@ -74,3 +75,4 @@ services:

 volumes:
  sushiswap_v3_watcher_db_data:
+  sushiswap_v3_watcher_gql_logs_data:
@@ -1,28 +0,0 @@
-# The password used to access test accounts (eg, via personal_unlockAccount).  The password is the same for all accounts.
-ACCOUNT_PASSWORD=secret1212
-
-# ENODE of the geth bootnode.
-BOOTNODE_KEY="b0ac22adcad37213c7c565810a50f1772291e7b0ce53fb73e7ec2a3c75bc13b5"
-ENODE="enode://af22c29c316ad069cf48a09a4ad5cf04a251b411e45098888d114c6dd7f489a13786620d5953738762afa13711d4ffb3b19aa5de772d8af72f851f7e9c5b164a@fixturenet-eth-bootnode-geth:30303"
-
-# JWT shared by geth and lighthouse for authentication.
-JWT="0x6cdcac3501046a08e186730dd8bd136cfaf0fdc1fc955f6e15ad3068c0ff2af0"
-
-# URL to download the ENR of the lighthouse bootnode (generated at first start).
-ENR_URL="http://fixturenet-eth-bootnode-lighthouse:3000/bootnode/enr.dat"
-
-# DB connection settings for statediffing (see docker-compose-db.yml)
-CERC_STATEDIFF_DB_HOST="ipld-eth-db"
-CERC_STATEDIFF_DB_PORT=5432
-CERC_STATEDIFF_DB_NAME="cerc_testing"
-CERC_STATEDIFF_DB_USER="vdbm"
-CERC_STATEDIFF_DB_PASSWORD="password"
-CERC_STATEDIFF_DB_GOOSE_MIN_VER=${CERC_STATEDIFF_DB_GOOSE_MIN_VER:-18}
-CERC_STATEDIFF_DB_LOG_STATEMENTS="${CERC_STATEDIFF_DB_LOG_STATEMENTS:-false}"
-CERC_STATEDIFF_WORKERS=2
-
-CERC_GETH_VMODULE="statediff/*=5,rpc/*=5"
-CERC_GETH_VERBOSITY=${CERC_GETH_VERBOSITY:-3}
-
-# Used by Lighthouse
-SECONDS_PER_ETH1_BLOCK=${SECONDS_PER_ETH1_BLOCK:-3}
@@ -0,0 +1,20 @@
+apiVersion: 1
+
+datasources:
+  - name: Graph Node Postgres
+    type: postgres
+    jsonData:
+      database: graph-node
+      sslmode: 'disable'
+      maxOpenConns: 100
+      maxIdleConns: 100
+      maxIdleConnsAuto: true
+      connMaxLifetime: 14400
+      postgresVersion: 1411 # 903=9.3, 1000=10, 1411=14.11
+      timescaledb: false
+    user: graph-node
+    # # Add URL for graph-node database
+    # url: graph-node-db:5432
+    # # Set password for graph-node database
+    # secureJsonData:
+    #   password: 'password'
@@ -45,7 +45,18 @@ scrape_configs:
    metrics_path: /metrics
    scheme: http
    static_configs:
-      - targets: ['chain-head-exporter:5000']
+      - targets: ['ethereum-chain-head-exporter:5000']
+        labels:
+          instance: 'external'
+          chain: 'ethereum'
+      - targets: ['filecoin-chain-head-exporter:5000']
+        labels:
+          instance: 'external'
+          chain: 'filecoin'
+      - targets: ['graph-node-upstream-head-exporter:5000']
+        labels:
+          instance: 'graph-node'
+          chain: 'filecoin'

  - job_name: 'postgres'
    scrape_interval: 30s
@@ -74,3 +85,11 @@ scrape_configs:
      # - targets: ['example-host:1317']
    params:
      format: ['prometheus']
+
+  - job_name: graph-node
+    metrics_path: /metrics
+    scrape_interval: 30s
+    scheme: http
+    static_configs:
+      # Add graph-node targets to be monitored below
+      # - targets: ['graph-node:8040']
@@ -0,0 +1,64 @@
+apiVersion: 1
+groups:
+  - orgId: 1
+    name: subgraph
+    folder: SubgraphAlerts
+    interval: 30s
+    rules:
+      - uid: b2a9144b-6104-46fc-92b5-352f4e643c4c
+        title: subgraph_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                  type: prometheus
+                  uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: ethereum_chain_head_number - on(network) group_right deployment_head{deployment=~"REPLACE_WITH_SUBGRAPH_IDS"}
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                  - evaluator:
+                      params:
+                          - 15
+                          - 0
+                      type: gt
+                    operator:
+                      type: and
+                    query:
+                      params: []
+                    reducer:
+                      params: []
+                      type: avg
+                    type: query
+              datasource:
+                  name: Expression
+                  type: __expr__
+                  uid: __expr__
+              expression: diff
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: threshold
+        noDataState: OK
+        execErrState: Alerting
+        for: 5m
+        annotations:
+          summary: Subgraph deployment {{ index $labels "deployment" }} is falling behind head by {{ index $values "diff" }}
+        labels: {}
+        isPaused: false
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+echo Using CERC_GRAFANA_ALERTS_SUBGRAPH_IDS ${CERC_GRAFANA_ALERTS_SUBGRAPH_IDS}
+
+# Replace subgraph ids in subgraph alerting config
+# Note: Requires the grafana container to be run with user root
+if [ -n "$CERC_GRAFANA_ALERTS_SUBGRAPH_IDS" ]; then
+  sed -i "s/REPLACE_WITH_SUBGRAPH_IDS/$CERC_GRAFANA_ALERTS_SUBGRAPH_IDS/g" /etc/grafana/provisioning/alerting/subgraph-alert-rules.yml
+fi
@@ -24,7 +24,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -100,7 +100,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -176,7 +176,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -252,7 +252,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -328,7 +328,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -404,7 +404,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -480,7 +480,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="linear_star_release", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="linear_star_release", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -556,7 +556,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -634,7 +634,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -710,7 +710,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -788,7 +788,85 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="ajna", instance="ajna", kind="latest_indexed"}
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="ajna", instance="ajna", kind="latest_indexed"}
+              fullMetaSearch: false
+              includeNullMetadata: true
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: diff
+              useBackend: false
+          - refId: latest_external
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              editorMode: code
+              expr: latest_block_number{chain="filecoin"}
+              hide: false
+              instant: true
+              legendFormat: __auto
+              range: false
+              refId: latest_external
+          - refId: condition
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 0
+                      - 0
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+              datasource:
+                name: Expression
+                type: __expr__
+                uid: __expr__
+              expression: ${diff} >= 16
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: condition
+              type: math
+        noDataState: Alerting
+        execErrState: Alerting
+        for: 15m
+        annotations:
+          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
+        isPaused: false
+      
+      # Secured Finance
+      - uid: secured_finance_diff_external
+        title: secured_finance_watcher_head_tracking
+        condition: condition
+        data:
+          - refId: diff
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: PBFA97CFB590B2093
+            model:
+              datasource:
+                type: prometheus
+                uid: PBFA97CFB590B2093
+              disableTextWrap: false
+              editorMode: code
+              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="secured-finance", instance="secured-finance", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u

-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"

 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")

 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml
@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u

-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"

 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")

 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml
@@ -2,7 +2,6 @@
  host = "0.0.0.0"
  port = 3008
  kind = "active"
-  gqlPath = "/"

  # Checkpointing state.
  checkpointing = true
@@ -22,23 +21,30 @@
  # Interval in number of blocks at which to clear entities cache.
  clearEntitiesCacheInterval = 1000

-  # Max block range for which to return events in eventsInRange GQL query.
-  # Use -1 for skipping check on block range.
-  maxEventsBlockRange = 1000
-
  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
  rpcSupportsBlockHashParam = false

-  # GQL cache settings
-  [server.gqlCache]
-    enabled = true
+  # Server GQL config
+  [server.gql]
+    path = "/"

-    # Max in-memory cache size (in bytes) (default 8 MB)
-    # maxCacheSize
+    # Max block range for which to return events in eventsInRange GQL query.
+    # Use -1 for skipping check on block range.
+    maxEventsBlockRange = 1000

-    # GQL cache-control max-age settings (in seconds)
-    maxAge = 15
-    timeTravelMaxAge = 86400 # 1 day
+    # Log directory for GQL requests
+    logDir = "./gql-logs"
+
+    # GQL cache settings
+    [server.gql.cache]
+      enabled = true
+
+      # Max in-memory cache size (in bytes) (default 8 MB)
+      # maxCacheSize
+
+      # GQL cache-control max-age settings (in seconds)
+      maxAge = 15
+      timeTravelMaxAge = 86400 # 1 day

 [metrics]
  host = "0.0.0.0"
@@ -58,7 +64,7 @@

 [upstream]
  [upstream.ethServer]
-    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS

    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
    rpcClient = true
@@ -85,6 +91,9 @@
  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
  blockDelayInMilliSecs = 30000

+  # Number of blocks by which block processing lags behind head
+  blockProcessingOffset = 0
+
  # Boolean to switch between modes of processing events when starting the server.
  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
@@ -96,3 +105,6 @@
  # Max block range of historical processing after which it waits for completion of events processing
  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
  historicalMaxFetchAhead = 10000
+
+  # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
+  maxNewBlockRetries = 3
@@ -4,16 +4,19 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
  set -x
 fi

-echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}"
-echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
+echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL_ENDPOINT}"
 echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}"

+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 # Replace env variables in template TOML file
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \
-          s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}|g; \
+          s|REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT|${CERC_IPLD_ETH_GQL_ENDPOINT}|g; \
          s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ")

 # Write the modified content to a new file
@@ -4,16 +4,19 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
  set -x
 fi

-echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}"
-echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
+echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL_ENDPOINT}"
 echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}"

+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 # Replace env variables in template TOML file
 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \
-          s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}|g; \
+          s|REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT|${CERC_IPLD_ETH_GQL_ENDPOINT}|g; \
          s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ")

 # Write the modified content to a new file
@@ -1,6 +1,7 @@
 [server]
  host = "0.0.0.0"
-  maxSimultaneousRequests = -1
+  [server.gql]
+    maxSimultaneousRequests = -1

 [metrics]
  host = "0.0.0.0"
@@ -13,8 +14,8 @@

 [upstream]
  [upstream.ethServer]
-    gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL"
-    rpcProviderEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_RPC"
+    gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT"
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS

 [jobQueue]
  historicalLogsBlockRange = REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE
@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u

-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"

 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")

 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml
@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u

-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"

 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")

 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml
@@ -2,7 +2,6 @@
  host = "0.0.0.0"
  port = 3008
  kind = "active"
-  gqlPath = '/'

  # Checkpointing state.
  checkpointing = true
@@ -22,23 +21,30 @@
  # Interval in number of blocks at which to clear entities cache.
  clearEntitiesCacheInterval = 1000

-  # Max block range for which to return events in eventsInRange GQL query.
-  # Use -1 for skipping check on block range.
-  maxEventsBlockRange = 1000
-
  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
  rpcSupportsBlockHashParam = false

-  # GQL cache settings
-  [server.gqlCache]
-    enabled = true
+  # Server GQL config
+  [server.gql]
+    path = "/"

-    # Max in-memory cache size (in bytes) (default 8 MB)
-    # maxCacheSize
+    # Max block range for which to return events in eventsInRange GQL query.
+    # Use -1 for skipping check on block range.
+    maxEventsBlockRange = 1000

-    # GQL cache-control max-age settings (in seconds)
-    maxAge = 15
-    timeTravelMaxAge = 86400 # 1 day
+    # Log directory for GQL requests
+    logDir = "./gql-logs"
+
+    # GQL cache settings
+    [server.gql.cache]
+      enabled = true
+
+      # Max in-memory cache size (in bytes) (default 8 MB)
+      # maxCacheSize
+
+      # GQL cache-control max-age settings (in seconds)
+      maxAge = 15
+      timeTravelMaxAge = 86400 # 1 day

 [metrics]
  host = "0.0.0.0"
@@ -58,7 +64,7 @@

 [upstream]
  [upstream.ethServer]
-    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS

    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
    rpcClient = true
@@ -69,7 +75,7 @@
    # Boolean flag to filter event logs by contracts
    filterLogsByAddresses = true
    # Boolean flag to filter event logs by topics
-    filterLogsByTopics = false
+    filterLogsByTopics = true

  [upstream.cache]
    name = "requests"
@@ -85,6 +91,9 @@
  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
  blockDelayInMilliSecs = 30000

+  # Number of blocks by which block processing lags behind head
+  blockProcessingOffset = 0
+
  # Boolean to switch between modes of processing events when starting the server.
  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
@@ -96,3 +105,6 @@
  # Max block range of historical processing after which it waits for completion of events processing
  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
  historicalMaxFetchAhead = 10000
+
+  # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
+  maxNewBlockRetries = 3
@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u

-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"

 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")

 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml
@@ -6,12 +6,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
 fi
 set -u

-echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
+echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"

 # Read in the config template TOML file and modify it
 WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
+
+# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
+RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
+
 WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
-  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
+  sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")

 # Write the modified content to a new file
 echo "$WATCHER_CONFIG" > environments/local.toml
@@ -2,7 +2,6 @@
  host = "0.0.0.0"
  port = 3008
  kind = "active"
-  gqlPath = "/"

  # Checkpointing state.
  checkpointing = true
@@ -22,23 +21,30 @@
  # Interval in number of blocks at which to clear entities cache.
  clearEntitiesCacheInterval = 1000

-  # Max block range for which to return events in eventsInRange GQL query.
-  # Use -1 for skipping check on block range.
-  maxEventsBlockRange = 1000
-
  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
  rpcSupportsBlockHashParam = false

-  # GQL cache settings
-  [server.gqlCache]
-    enabled = true
+  # Server GQL config
+  [server.gql]
+    path = "/"

-    # Max in-memory cache size (in bytes) (default 8 MB)
-    # maxCacheSize
+    # Max block range for which to return events in eventsInRange GQL query.
+    # Use -1 for skipping check on block range.
+    maxEventsBlockRange = 1000

-    # GQL cache-control max-age settings (in seconds)
-    maxAge = 15
-    timeTravelMaxAge = 86400 # 1 day
+    # Log directory for GQL requests
+    logDir = "./gql-logs"
+
+    # GQL cache settings
+    [server.gql.cache]
+      enabled = true
+
+      # Max in-memory cache size (in bytes) (default 8 MB)
+      # maxCacheSize
+
+      # GQL cache-control max-age settings (in seconds)
+      maxAge = 15
+      timeTravelMaxAge = 86400 # 1 day

 [metrics]
  host = "0.0.0.0"
@@ -58,7 +64,7 @@

 [upstream]
  [upstream.ethServer]
-    rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
+    rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS

    # Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
    rpcClient = true
@@ -85,6 +91,9 @@
  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
  blockDelayInMilliSecs = 30000

+  # Number of blocks by which block processing lags behind head
+  blockProcessingOffset = 0
+
  # Boolean to switch between modes of processing events when starting the server.
  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
@@ -96,3 +105,6 @@
  # Max block range of historical processing after which it waits for completion of events processing
  # If set to -1 historical processing does not wait for events processing and completes till latest canonical block
  historicalMaxFetchAhead = 10000
+
+  # Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
+  maxNewBlockRetries = 3
@@ -109,10 +109,6 @@ else
      OTHER_OPTS="--miner.threads=1"
    fi

-    if [ "$CERC_ALLOW_UNPROTECTED_TXS" == "true" ]; then
-      OTHER_OPTS+=" --rpc.allow-unprotected-txs"
-    fi
-
    $START_CMD \
      --datadir="${CERC_ETH_DATADIR}" \
      --bootnodes="${ENODE}" \
@@ -1,6 +0,0 @@
-FROM cerc/snowballtools-base-backend-base:local
-
-WORKDIR /app/packages/backend
-COPY run.sh .
-
-ENTRYPOINT ["./run.sh"]
@@ -1,26 +0,0 @@
-FROM ubuntu:22.04 as builder
-
-RUN apt update && \
-      apt install -y --no-install-recommends --no-install-suggests \
-      ca-certificates curl gnupg
-
-# Node
-ARG NODE_MAJOR=20
-RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
-      echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \
-      apt update && apt install -y nodejs
-
-# npm setup
-RUN npm config set @cerc-io:registry https://git.vdb.to/api/packages/cerc-io/npm/ && npm install -g yarn
-
-COPY . /app/
-WORKDIR /app/
-
-RUN find . -name 'node_modules' | xargs -n1 rm -rf
-RUN yarn && yarn build --ignore frontend
-
-FROM cerc/webapp-base:local
-
-COPY --from=builder /app /app
-
-WORKDIR /app/packages/backend
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-# Build cerc/webapp-deployer-backend
-
-source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
-
-# See: https://stackoverflow.com/a/246128/1701505
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-
-docker build -t cerc/snowballtools-base-backend-base:local ${build_command_args} -f ${SCRIPT_DIR}/Dockerfile-base ${CERC_REPO_BASE_DIR}/snowballtools-base
-docker build -t cerc/snowballtools-base-backend:local ${build_command_args} ${SCRIPT_DIR} 
@@ -1,19 +0,0 @@
-#!/bin/bash
-
-
-LACONIC_HOSTED_CONFIG_FILE=${LACONIC_HOSTED_CONFIG_FILE}
-if [ -z "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
-  if [ -f "/config/laconic-hosted-config.yml" ]; then
-    LACONIC_HOSTED_CONFIG_FILE="/config/laconic-hosted-config.yml"
-  elif [ -f "/config/config.yml" ]; then
-    LACONIC_HOSTED_CONFIG_FILE="/config/config.yml"
-  fi
-fi
-
-if [ -f "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
-  /scripts/apply-webapp-config.sh $LACONIC_HOSTED_CONFIG_FILE "`pwd`/dist"
-fi
-
-/scripts/apply-runtime-env.sh "`pwd`/dist"
-
-yarn start
@@ -6,5 +6,10 @@ WORKDIR /app

 COPY . .

+# Get the latest Git commit hash and set in package.json
+RUN COMMIT_HASH=$(git rev-parse HEAD) && \
+  jq --arg commitHash "$COMMIT_HASH" '.commitHash = $commitHash' package.json > tmp.json && \
+  mv tmp.json package.json
+
 RUN echo "Installing dependencies and building ajna-watcher-ts" && \
  yarn && yarn build
@@ -1,11 +1,20 @@
 FROM node:18.16.0-alpine3.16

-RUN apk --update --no-cache add git python3 alpine-sdk
+RUN apk --update --no-cache add git python3 alpine-sdk jq

 WORKDIR /app

 COPY . .

+# Get the latest Git commit hash and set it in package.json of all watchers
+RUN COMMIT_HASH=$(git rev-parse HEAD) && \
+    find . -name 'package.json' -exec sh -c ' \
+    for packageFile; do \
+        jq --arg commitHash "$0" ".commitHash = \$commitHash" "$packageFile" > "$packageFile.tmp" && \
+        mv "$packageFile.tmp" "$packageFile"; \
+    done \
+    ' "$COMMIT_HASH" {} \;
+
 RUN echo "Building azimuth-watcher-ts" && \
    yarn && yarn build

@@ -6,5 +6,10 @@ WORKDIR /app

 COPY . .

+# Get the latest Git commit hash and set in package.json
+RUN COMMIT_HASH=$(git rev-parse HEAD) && \
+  jq --arg commitHash "$COMMIT_HASH" '.commitHash = $commitHash' package.json > tmp.json && \
+  mv tmp.json package.json
+
 RUN echo "Installing dependencies and building merkl-sushiswap-v3-watcher-ts" && \
  yarn && yarn build
@@ -6,5 +6,10 @@ WORKDIR /app

 COPY . .

+# Get the latest Git commit hash and set in package.json
+RUN COMMIT_HASH=$(git rev-parse HEAD) && \
+  jq --arg commitHash "$COMMIT_HASH" '.commitHash = $commitHash' package.json > tmp.json && \
+  mv tmp.json package.json
+
 RUN echo "Installing dependencies and building sushiswap-v3-watcher-ts" && \
  yarn && yarn build
@@ -32,7 +32,7 @@ RUN \

 # [Optional] Uncomment this section to install additional OS packages.
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends jq gettext-base
+    && apt-get -y install --no-install-recommends jq gettext-base git

 # [Optional] Uncomment if you want to install an additional version of node using nvm
 # ARG EXTRA_NODE_VERSION=10
@@ -8,21 +8,27 @@ CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/data}"
 CERC_ENABLE_CORS="${CERC_ENABLE_CORS:-false}"
 CERC_SINGLE_PAGE_APP="${CERC_SINGLE_PAGE_APP}"

-if [ -z "${CERC_SINGLE_PAGE_APP}" ]; then
-  if [ 1 -eq $(find "${CERC_WEBAPP_FILES_DIR}" -name '*.html' | wc -l) ] && [ -d "${CERC_WEBAPP_FILES_DIR}/static" ]; then
+if [ -z "${CERC_SINGLE_PAGE_APP}" ]; then 
+  # If there is only one HTML file, assume an SPA.
+  if [ 1 -eq $(find "${CERC_WEBAPP_FILES_DIR}" -name '*.html' | wc -l) ]; then
    CERC_SINGLE_PAGE_APP=true
  else
    CERC_SINGLE_PAGE_APP=false
  fi
 fi

-if [ "true" == "$CERC_ENABLE_CORS" ]; then
+# ${var,,} is a lower-case comparison
+if [ "true" == "${CERC_ENABLE_CORS,,}" ]; then
  CERC_HTTP_EXTRA_ARGS="$CERC_HTTP_EXTRA_ARGS --cors"
 fi

-if [ "true" == "$CERC_SINGLE_PAGE_APP" ]; then
+# ${var,,} is a lower-case comparison
+if [ "true" == "${CERC_SINGLE_PAGE_APP,,}" ]; then
+  echo "Serving content as single-page app.  If this is wrong, set 'CERC_SINGLE_PAGE_APP=false'"
  # Create a catchall redirect back to /
  CERC_HTTP_EXTRA_ARGS="$CERC_HTTP_EXTRA_ARGS --proxy http://localhost:${CERC_LISTEN_PORT}?"
+else
+  echo "Serving content normally.  If this is a single-page app, set 'CERC_SINGLE_PAGE_APP=true'"
 fi

 LACONIC_HOSTED_CONFIG_FILE=${LACONIC_HOSTED_CONFIG_FILE}
@@ -39,4 +45,4 @@ if [ -f "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
 fi

 /scripts/apply-runtime-env.sh ${CERC_WEBAPP_FILES_DIR}
-http-server $CERC_HTTP_EXTRA_ARGS -p ${CERC_LISTEN_PORT} "${CERC_WEBAPP_FILES_DIR}"
+http-server $CERC_HTTP_EXTRA_ARGS -p ${CERC_LISTEN_PORT} "${CERC_WEBAPP_FILES_DIR}"
@@ -53,7 +53,7 @@ Inside deployment directory, open the `config.env` file  and set following env v

 ```bash
 # External Filecoin (ETH RPC) endpoint to point the watcher to
-CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
+CERC_ETH_RPC_ENDPOINTS=https://example-lotus-endpoint-1/rpc/v1,https://example-lotus-endpoint-2/rpc/v1
 ```

 ### Start the deployment
@@ -2,7 +2,7 @@ version: "1.0"
 name: ajna
 description: "Ajna watcher stack"
 repos:
-  - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.1
+  - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.13
 containers:
  - cerc/watcher-ajna
 pods:
@@ -4,7 +4,7 @@ Instructions to setup and deploy Azimuth Watcher stack

 ## Setup

-Prerequisite: `ipld-eth-server` RPC and GQL endpoints
+Prerequisite: External RPC endpoints

 Clone required repositories:

@@ -44,34 +44,42 @@ network:
     - 0.0.0.0:9000:9000
    azimuth-watcher-server:
     - 0.0.0.0:3001:3001
+     - 0.0.0.0:9001:9001
    censures-watcher-job-runner:
     - 0.0.0.0:9002:9002
    censures-watcher-server:
     - 0.0.0.0:3002:3002
+     - 0.0.0.0:9003:9003
    claims-watcher-job-runner:
     - 0.0.0.0:9004:9004
    claims-watcher-server:
     - 0.0.0.0:3003:3003
+     - 0.0.0.0:9005:9005
    conditional-star-release-watcher-job-runner:
     - 0.0.0.0:9006:9006
    conditional-star-release-watcher-server:
     - 0.0.0.0:3004:3004
+     - 0.0.0.0:9007:9007
    delegated-sending-watcher-job-runner:
     - 0.0.0.0:9008:9008
    delegated-sending-watcher-server:
     - 0.0.0.0:3005:3005
+     - 0.0.0.0:9009:9009
    ecliptic-watcher-job-runner:
     - 0.0.0.0:9010:9010
    ecliptic-watcher-server:
     - 0.0.0.0:3006:3006
+     - 0.0.0.0:9011:9011
    linear-star-release-watcher-job-runner:
     - 0.0.0.0:9012:9012
    linear-star-release-watcher-server:
     - 0.0.0.0:3007:3007
+     - 0.0.0.0:9013:9013
    polls-watcher-job-runner:
     - 0.0.0.0:9014:9014
    polls-watcher-server:
     - 0.0.0.0:3008:3008
+     - 0.0.0.0:9015:9015
    gateway-server:
     - 0.0.0.0:4000:4000
 ...
@@ -94,7 +102,7 @@ Inside the deployment directory, open the file `config.env` and add variable to

  ```bash
  # External RPC endpoints
-  CERC_IPLD_ETH_RPC=
+  CERC_ETH_RPC_ENDPOINTS=https://example-rpc-endpoint-1,https://example-rpc-endpoint-2
  ```

 * NOTE: If RPC endpoint is on the host machine, use `host.docker.internal` as the hostname to access the host port, or use the `ip a` command to find the IP address of the `docker0` interface (this will usually be something like `172.17.0.1` or `172.18.0.1`)
@@ -120,4 +128,7 @@ To stop all azimuth services and also delete data:

 ```bash
 laconic-so deployment --dir azimuth-deployment stop --delete-volumes
+
+# Remove deployment directory (deployment will have to be recreated for a re-run)
+rm -r azimuth-deployment
 ```
@@ -1,7 +1,7 @@
 version: "1.0"
 name: azimuth
 repos:
-  - github.com/cerc-io/azimuth-watcher-ts@v0.1.3
+  - github.com/cerc-io/azimuth-watcher-ts@0.1.6
 containers:
  - cerc/watcher-azimuth
 pods:
@@ -1,19 +0,0 @@
-# fixturenet-plugeth
-
-A variation of `fixturenet-eth` that uses `plugeth` instead of `go-ethereum`.
-
-See `stacks/fixturenet-eth/README.md` for more information.
-
-## Containers
-
-* cerc/lighthouse
-* cerc/fixturenet-eth-plugeth
-* cerc/fixturenet-eth-lighthouse
-
-## Deploy the stack
-
-```bash
-laconic-so --stack fixturenet-plugeth setup-repositories
-laconic-so --stack fixturenet-plugeth build-containers
-laconic-so --stack fixturenet-plugeth deploy up
-```
@@ -1,17 +0,0 @@
-version: "1.0"
-name: fixturenet-plugeth
-description: "Plugeth Ethereum Fixturenet"
-repos:
-  - git.vdb.to/cerc-io/plugeth@statediff
-  - git.vdb.to/cerc-io/plugeth-statediff
-  - git.vdb.to/cerc-io/lighthouse
-containers:
-  - cerc/plugeth-statediff
-  - cerc/plugeth
-  - cerc/fixturenet-eth-genesis
-  - cerc/fixturenet-plugeth-plugeth
-  - cerc/lighthouse
-  - cerc/lighthouse-cli
-  - cerc/fixturenet-eth-lighthouse
-pods:
-  - fixturenet-plugeth
@@ -43,16 +43,18 @@ customized by editing the "spec" file generated by `laconic-so deploy init`.
 ```
 $ cat graph-node-spec.yml
 stack: graph-node
-ports:
-  graph-node:
-    - '8000:8000'
-    - '8001'
-    - '8020:8020'
-    - '8030'
-  ipfs:
-    - '8080'
-    - '4001'
-    - '5001:5001'
+network:
+  ports:
+    graph-node:
+      - '8000:8000'
+      - '8001'
+      - '8020:8020'
+      - '8030'
+      - '8040'
+    ipfs:
+      - '8080'
+      - '4001'
+      - '5001:5001'
 ...
 ```

@@ -64,7 +66,7 @@ laconic-so --stack graph-node deploy create --spec-file graph-node-spec.yml --de

 ## Start the stack

-Create an env file with the following values to be set before starting the stack:
+Update `config.env` file inside deployment directory with the following values before starting the stack:

 ```bash
 # Set ETH RPC endpoint the graph node will use
@@ -88,10 +90,13 @@ export GRAPH_ETHEREUM_REQUEST_RETRIES=
 # Maximum number of blocks to scan for triggers in each request (default: 2000)
 export GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE=

+# Maximum number of concurrent requests made against Ethereum for requesting transaction receipts during block ingestion (default: 1000)
+export GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS=
+
 # Ref: https://git.vdb.to/cerc-io/graph-node/src/branch/master/docs/environment-variables.md
 ```

-Example env file:
+Example `config.env` file:

 ```bash
 export ETH_RPC_HOST=filecoin.chainup.net
@@ -104,12 +109,6 @@ export GRAPH_ETHEREUM_REQUEST_RETRIES=5
 export GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE=50
 ```

-Set the environment variables:
-
-```bash
-source <PATH_TO_ENV_FILE>
-```
-
 Deploy the stack:

 ```bash
@@ -3,7 +3,9 @@ name: graph-node
 description: "Stack for running graph-node"
 repos:
  - github.com/graphprotocol/graph-node
+  - github.com/cerc-io/watcher-ts@v0.2.92
 containers:
  - cerc/graph-node
+  - cerc/watcher-ts
 pods:
  - graph-node
@@ -14,7 +14,8 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.

 from stack_orchestrator.util import get_yaml
-from stack_orchestrator.deploy.deploy_types import DeployCommandContext, LaconicStackSetupCommand, DeploymentContext
+from stack_orchestrator.deploy.deploy_types import DeployCommandContext, LaconicStackSetupCommand
+from stack_orchestrator.deploy.deployment_context import DeploymentContext
 from stack_orchestrator.deploy.stack_state import State
 from stack_orchestrator.deploy.deploy_util import VolumeMapping, run_container_command
 from stack_orchestrator.command_types import CommandOptions
@@ -53,7 +53,7 @@ Inside deployment directory, open the `config.env` file  and set following env v

 ```bash
 # External Filecoin (ETH RPC) endpoint to point the watcher to
-CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
+CERC_ETH_RPC_ENDPOINTS=https://example-lotus-endpoint-1/rpc/v1,https://example-lotus-endpoint-2/rpc/v1
 ```

 ### Start the deployment
@@ -2,7 +2,7 @@ version: "1.0"
 name: merkl-sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.7
+  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.14
 containers:
  - cerc/watcher-merkl-sushiswap-v3
 pods:
@@ -134,6 +134,29 @@ Note: Use `host.docker.internal` as host to access ports on the host machine

 Place the dashboard json files in grafana dashboards config directory (`monitoring-deployment/config/monitoring/grafana/dashboards`) in the deployment folder

+#### Graph Node Config
+
+For graph-node dashboard postgres datasource needs to be setup in `monitoring-deployment/config/monitoring/grafana/provisioning/datasources/graph-node-postgres.yml` (in deployment folder)
+
+```yml
+# graph-node-postgres.yml
+...
+datasources:
+  - name: Graph Node Postgres
+    type: postgres
+    jsonData:
+      # Set name to remote graph-node database name
+      database: graph-node
+      ...
+    # Set user to remote graph-node database username
+    user: graph-node
+    # Add URL for remote graph-node database
+    url: graph-node-db:5432
+    # Set password for graph-node database
+    secureJsonData:
+      password: 'password'
+```
+
 ### Env

 Set the following env variables in the deployment env config file (`monitoring-deployment/config.env`):
@@ -156,6 +179,11 @@ Set the following env variables in the deployment env config file (`monitoring-d
  # Grafana server host URL (used in various links in alerts, etc.)
  # (Optional, default: http://localhost:3000)
  GF_SERVER_ROOT_URL=
+
+  
+  # RPC endpoint used by graph-node for upstream head metric
+  # (Optional, default: https://mainnet.infura.io/v3)
+  GRAPH_NODE_RPC_ENDPOINT=
  ```

 ## Start the stack
@@ -57,35 +57,35 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
    metrics_path: /metrics
    scheme: http
    static_configs:
-      - targets: ['AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_PORT']
+      - targets: ['AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_METRICS_PORT', 'AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'azimuth'
          chain: 'ethereum'
-      - targets: ['CENSURES_WATCHER_HOST:CENSURES_WATCHER_PORT']
+      - targets: ['CENSURES_WATCHER_HOST:CENSURES_WATCHER_METRICS_PORT', 'CENSURES_WATCHER_HOST:CENSURES_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'censures'
          chain: 'ethereum'
-      - targets: ['CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_PORT']
+      - targets: ['CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_METRICS_PORT', 'CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'claims'
          chain: 'ethereum'
-      - targets: ['CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_PORT']
+      - targets: ['CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_METRICS_PORT', 'CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'conditional_star_release'
          chain: 'ethereum'
-      - targets: ['DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_PORT']
+      - targets: ['DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_METRICS_PORT', 'DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'delegated_sending'
          chain: 'ethereum'
-      - targets: ['ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_PORT']
+      - targets: ['ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_METRICS_PORT', 'ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'ecliptic'
          chain: 'ethereum'
-      - targets: ['LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_PORT']
+      - targets: ['LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_METRICS_PORT', 'LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'linear_star_release'
          chain: 'ethereum'
-      - targets: ['POLLS_WATCHER_HOST:POLLS_WATCHER_PORT']
+      - targets: ['POLLS_WATCHER_HOST:POLLS_WATCHER_METRICS_PORT', 'POLLS_WATCHER_HOST:POLLS_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'polls'
          chain: 'ethereum'
@@ -95,11 +95,11 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
    metrics_path: /metrics
    scheme: http
    static_configs:
-      - targets: ['SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_PORT']
+      - targets: ['SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_METRICS_PORT', 'SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'sushiswap'
          chain: 'filecoin'
-      - targets: ['MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_PORT']
+      - targets: ['MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_METRICS_PORT', 'MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'merkl_sushiswap'
          chain: 'filecoin'
@@ -109,25 +109,35 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
    metrics_path: /metrics
    scheme: http
    static_configs:
-      - targets: ['AJNA_WATCHER_HOST:AJNA_WATCHER_PORT']
+      - targets: ['AJNA_WATCHER_HOST:AJNA_WATCHER_METRICS_PORT', 'AJNA_WATCHER_HOST:AJNA_WATCHER_GQL_METRICS_PORT']
        labels:
          instance: 'ajna'
          chain: 'filecoin'
+
+  - job_name: graph-node
+    metrics_path: /metrics
+    scrape_interval: 30s
+    static_configs:
+      - targets: ['GRAPH_NODE_HOST:GRAPH_NODE_HOST_METRICS_PORT']
  ```

 Add scrape config as done above for any additional watcher to add it to the Watchers dashboard.

 ### Grafana alerts config

-Place the pre-configured watcher alerts rules in Grafana provisioning directory:
+Place the pre-configured alerts rules in Grafana provisioning directory:

  ```bash
+  # watcher alert rules
  cp monitoring-watchers-deployment/config/monitoring/watcher-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
+
+  # subgraph alert rules
+  cp monitoring-watchers-deployment/config/monitoring/subgraph-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
  ```

 Update the alerting contact points config (`monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/contactpoints.yml`) with desired contact points

-Add corresponding routes to the notification policies config (`monitoring-watchers-deployment/monitoring/grafana/provisioning/alerting/policies.yaml`) with appropriate object-matchers:
+Add corresponding routes to the notification policies config (`monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/policies.yml`) with appropriate object-matchers:

  ```yml
  ...
@@ -135,7 +145,7 @@ Add corresponding routes to the notification policies config (`monitoring-watche
      - receiver: SlackNotifier
      object_matchers:
        # Add matchers below
-        - ['grafana_folder', '=', 'WatcherAlerts']
+        - ['grafana_folder', '=~', 'WatcherAlerts|SubgraphAlerts']
  ```

 ### Env
@@ -149,6 +159,9 @@ Set the following env variables in the deployment env config file (`monitoring-w
  # Grafana server host URL to be used
  # (Optional, default: http://localhost:3000)
  GF_SERVER_ROOT_URL=
+
+  # List of subgraph ids to configure alerts for (separated by |)
+  CERC_GRAFANA_ALERTS_SUBGRAPH_IDS=
  ```

 ## Start the stack
@@ -1,7 +1,7 @@
 version: "0.1"
 name: monitoring
 repos:
-  - github.com/cerc-io/watcher-ts@v0.2.81
+  - github.com/cerc-io/watcher-ts@v0.2.92
 containers:
  - cerc/watcher-ts
 pods:
@@ -1,10 +0,0 @@
-version: "1.0"
-name: snowballtools-base-backend
-description: "snowballtools-base-backend"
-repos:
-  - github.com/snowball-tools/snowballtools-base
-containers:
-  - cerc/webapp-base
-  - cerc/snowballtools-base-backend
-pods:
-  - snowballtools-base-backend
@@ -53,7 +53,7 @@ Inside deployment directory, open the `config.env` file  and set following env v

 ```bash
 # External Filecoin (ETH RPC) endpoint to point the watcher to
-CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
+CERC_ETH_RPC_ENDPOINTS=https://example-lotus-endpoint-1/rpc/v1,https://example-lotus-endpoint-2/rpc/v1
 ```

 ### Start the deployment
@@ -2,7 +2,7 @@ version: "1.0"
 name: sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.7
+  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.14
 containers:
  - cerc/watcher-sushiswap-v3
 pods:
@@ -29,16 +29,29 @@ def _image_needs_pushed(image: str):
    return image.endswith(":local")


+def _remote_tag_for_image(image: str, remote_repo_url: str):
+    # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
+    major_parts = image.split("/", 2)
+    image_name_with_version = major_parts[1] if 2 == len(major_parts) else major_parts[0]
+    (image_name, image_version) = image_name_with_version.split(":")
+    if image_version == "local":
+        return f"{remote_repo_url}/{image_name}:deploy"
+    else:
+        return image
+
+
+# Note: do not add any calls this function
 def remote_image_exists(remote_repo_url: str, local_tag: str):
    docker = DockerClient()
    try:
-        remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
+        remote_tag = _remote_tag_for_image(local_tag, remote_repo_url)
        result = docker.manifest.inspect(remote_tag)
        return True if result else False
    except Exception:  # noqa: E722
        return False


+# Note: do not add any calls this function
 def add_tags_to_image(remote_repo_url: str, local_tag: str, *additional_tags):
    if not additional_tags:
        return
@@ -47,18 +60,20 @@ def add_tags_to_image(remote_repo_url: str, local_tag: str, *additional_tags):
        raise Exception(f"{local_tag} does not exist in {remote_repo_url}")

    docker = DockerClient()
-    remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
-    new_remote_tags = [remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags]
+    remote_tag = _remote_tag_for_image(local_tag, remote_repo_url)
+    new_remote_tags = [_remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags]
    docker.buildx.imagetools.create(sources=[remote_tag], tags=new_remote_tags)


-def remote_tag_for_image(image: str, remote_repo_url: str):
+def remote_tag_for_image_unique(image: str, remote_repo_url: str, deployment_id: str):
    # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
    major_parts = image.split("/", 2)
    image_name_with_version = major_parts[1] if 2 == len(major_parts) else major_parts[0]
    (image_name, image_version) = image_name_with_version.split(":")
    if image_version == "local":
-        return f"{remote_repo_url}/{image_name}:deploy"
+        # Salt the tag with part of the deployment id to make it unique to this deployment
+        deployment_tag = deployment_id[-8:]
+        return f"{remote_repo_url}/{image_name}:deploy-{deployment_tag}"
    else:
        return image

@@ -73,14 +88,14 @@ def push_images_operation(command_context: DeployCommandContext, deployment_cont
    docker = DockerClient()
    for image in images:
        if _image_needs_pushed(image):
-            remote_tag = remote_tag_for_image(image, remote_repo_url)
+            remote_tag = remote_tag_for_image_unique(image, remote_repo_url, deployment_context.id)
            if opts.o.verbose:
                print(f"Tagging {image} to {remote_tag}")
            docker.image.tag(image, remote_tag)
    # Run docker push commands to upload
    for image in images:
        if _image_needs_pushed(image):
-            remote_tag = remote_tag_for_image(image, remote_repo_url)
+            remote_tag = remote_tag_for_image_unique(image, remote_repo_url, deployment_context.id)
            if opts.o.verbose:
                print(f"Pushing image {remote_tag}")
            docker.image.push(remote_tag)
@@ -26,7 +26,7 @@ from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variable
 from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment
 from stack_orchestrator.deploy.deploy_types import DeployEnvVars
 from stack_orchestrator.deploy.spec import Spec, Resources, ResourceLimits
-from stack_orchestrator.deploy.images import remote_tag_for_image
+from stack_orchestrator.deploy.images import remote_tag_for_image_unique

 DEFAULT_VOLUME_RESOURCES = Resources({
    "reservations": {"storage": "2Gi"}
@@ -326,8 +326,11 @@ class ClusterInfo:
                if opts.o.debug:
                    print(f"Merged envs: {envs}")
                # Re-write the image tag for remote deployment
-                image_to_use = remote_tag_for_image(
-                    image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image
+                # Note self.app_name has the same value as deployment_id
+                image_to_use = remote_tag_for_image_unique(
+                    image,
+                    self.spec.get_image_registry(),
+                    self.app_name) if self.spec.get_image_registry() is not None else image
                volume_mounts = volume_mounts_for_service(self.parsed_pod_yaml_map, service_name)
                container = client.V1Container(
                    name=container_name,
@@ -22,16 +22,16 @@ echo "$(date +"%Y-%m-%d %T"): Stack started"
 # Verify that the fixturenet is up and running
 $TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd ps

+# Wait for the laconid endpoint to come up
+echo "Waiting for the RPC endpoint to come up"
+docker exec laconicd-laconicd-1 sh -c "curl --retry 20 --retry-delay 3 --retry-connrefused http://127.0.0.1:9473/api"
+
 # Get the fixturenet account address
 laconicd_account_address=$(docker exec laconicd-laconicd-1 laconicd keys list | awk '/- address:/ {print $3}')

 # Copy over config
 docker exec laconicd-cli-1 cp config.yml laconic-registry-cli/

-# Wait for the laconid endpoint to come up
-echo "Waiting for the RPC endpoint to come up"
-docker exec laconicd-laconicd-1 sh -c "curl --retry 20 --retry-delay 3 --retry-connrefused http://127.0.0.1:9473/api"
-
 # Run the tests
 echo "Running the tests"
 docker exec -e TEST_ACCOUNT=$laconicd_account_address laconicd-cli-1 sh -c 'cd laconic-registry-cli && yarn && yarn test'
Author	SHA1	Message	Date
telackey	3ba9436863	Add git to webapp base container.	2024-06-24 23:57:15 -05:00
dboreham	b4e82ebc19	Merge pull request 'Fix mainnet laconic deploy setup' (#850 ) from dboreham/fix-mainnet-laconic-init into main Reviewed-on: cerc-io/stack-orchestrator#850	2024-06-22 01:25:07 +00:00
dboreham	2364924a59	Fix mainnet laconic deploy setup	2024-06-21 19:24:33 -06:00
nabarun	a223797b4a	Update graph-node dashboard to show individual subgraph increase in query count (#846 ) Part of [Metrics and logging for GQL queries in watcher](https://www.notion.so/Metrics-and-logging-for-GQL-queries-in-watcher-928c692292b140a2a4f52cda9795df5e) Reviewed-on: cerc-io/stack-orchestrator#846 Co-authored-by: Nabarun <nabarun@deepstacksoft.com> Co-committed-by: Nabarun <nabarun@deepstacksoft.com>	2024-06-20 09:27:23 +00:00
nabarun	b8004e9870	Add Grafana panels for graph-node subgraph GQL queries (#845 ) Part of [Deploy v2 and updated v3 sushiswap subgraphs](https://www.notion.so/Deploy-v2-and-updated-v3-sushiswap-subgraphs-e331945fdeea487c890706fc22c6cc94) Reviewed-on: cerc-io/stack-orchestrator#845 Co-authored-by: Nabarun <nabarun@deepstacksoft.com> Co-committed-by: Nabarun <nabarun@deepstacksoft.com>	2024-06-19 10:40:54 +00:00
prathamesh	3fd99a1522	Handle race condition in laconic registry CLI tests (#843 ) Part of cerc-io/laconic-registry-cli#63 Reviewed-on: cerc-io/stack-orchestrator#843 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-06-19 06:41:58 +00:00
nabarun	842d999792	Add alert rules for secured secured-finance subgraph watcher (#842 ) Part of [Generate secured-finance subgraph watcher with codegen](https://www.notion.so/Generate-secured-finance-subgraph-watcher-with-codegen-2923413e0af54ea787c5435d6966f3bb) - Update watcher dashboard labels Reviewed-on: cerc-io/stack-orchestrator#842 Co-authored-by: Nabarun <nabarun@deepstacksoft.com> Co-committed-by: Nabarun <nabarun@deepstacksoft.com>	2024-06-18 12:28:02 +00:00
dboreham	d6a1fb3279	Merge pull request 'Fix image tag name' (#841 ) from dboreham/fix-remote-image-tags into main Reviewed-on: cerc-io/stack-orchestrator#841	2024-06-13 14:32:26 +00:00
dboreham	bf1eccd486	Fix image tag name	2024-06-13 08:31:45 -06:00
dboreham	3fb025b5c9	Make remote image tags unique to the deployment (#838 ) Reviewed-on: cerc-io/stack-orchestrator#838 Reviewed-by: Thomas E Lackey <telackey@noreply.git.vdb.to> Co-authored-by: David Boreham <david@bozemanpass.com> Co-committed-by: David Boreham <david@bozemanpass.com>	2024-06-13 03:26:58 +00:00
prathameshandnabarun	4acb06325b	Update watcher dashboard and config templates (#835 ) Part of [Metrics and logging for GQL queries in watcher](https://www.notion.so/Metrics-and-logging-for-GQL-queries-in-watcher-928c692292b140a2a4f52cda9795df5e) - Update watcher config templates after config refactoring - Mount watcher GQL query log files on volumes - Update watcher dashboard to - add a panel to show latest processed block number - use latest processed block from sync status for diff values Co-authored-by: Nabarun <nabarun@deepstacksoft.com> Reviewed-on: cerc-io/stack-orchestrator#835 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-06-12 11:52:51 +00:00
dboreham	b80b647fa4	Merge pull request 'Remove files migrated to external repo' (#836 ) from dboreham/remove-snowball-stack into main Reviewed-on: cerc-io/stack-orchestrator#836	2024-06-07 03:17:48 +00:00
dboreham	9a1d3bb0f1	Remove files migrated to external repo	2024-06-06 20:44:51 -06:00
prathamesh	abc0c2423f	Add panels for GQL metrics to watcher dashboard (#834 ) Part of [Metrics and logging for GQL queries in watcher](https://www.notion.so/Metrics-and-logging-for-GQL-queries-in-watcher-928c692292b140a2a4f52cda9795df5e) Reviewed-on: cerc-io/stack-orchestrator#834 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-06-06 11:47:18 +00:00
nabarun	a322d6eed4	Add dashboard for graph-node subgraphs (#832 ) Part of [Deploy v2 and updated v3 sushiswap subgraphs](https://www.notion.so/Deploy-v2-and-updated-v3-sushiswap-subgraphs-e331945fdeea487c890706fc22c6cc94) - Add param `GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS` in graph-node stack - <https://github.com/graphprotocol/graph-node/blob/v0.31.0/docs/environment-variables.md#json-rpc-configuration-for-evm-chains> - Add dashboard for subgraphs deployment in graph-node - Show subgraph names in dashboard - Add watcher dashboard panel for showing watcher release version and commit hash Reviewed-on: cerc-io/stack-orchestrator#832 Co-authored-by: Nabarun <nabarun@deepstacksoft.com> Co-committed-by: Nabarun <nabarun@deepstacksoft.com>	2024-06-04 07:21:27 +00:00
prathameshandnabarun	ed8914b8d3	Upgrade watchers and their config (#827 ) Part of [Investigate subgraph watchers lagging behind head](https://www.notion.so/Investigate-subgraph-watchers-lagging-behind-head-01b72294ca8e4f658e4c0e86b36d19e2) Co-authored-by: Nabarun <nabarun@deepstacksoft.com> Reviewed-on: cerc-io/stack-orchestrator#827 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-05-23 04:12:31 +00:00
telackey	fef7649683	Refactor SPA check. (#831 ) Reviewed-on: cerc-io/stack-orchestrator#831 Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com> Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>	2024-05-22 19:00:42 +00:00
telackey	579b402f2f	Make the SPA detection even simpler. (#830 ) Reviewed-on: cerc-io/stack-orchestrator#830 Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com> Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>	2024-05-22 18:31:57 +00:00
telackey	25d0bc8a98	Case insensitive comparison (#829 ) Reviewed-on: cerc-io/stack-orchestrator#829 Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com> Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>	2024-05-22 18:18:27 +00:00
telackey	855288368c	Add messages around the SPA auto-detect. (#828 ) Tweak the auto-detection logic slightly for single-page apps, and also print the results. Reviewed-on: cerc-io/stack-orchestrator#828 Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com> Co-committed-by: Thomas E Lackey <telackey@bozemanpass.com>	2024-05-22 17:42:28 +00:00
prathamesh	8f2da38183	Add alerts for graph-node subgraphs (#821 ) Part of [Deploy v2 and updated v3 sushiswap subgraphs](https://www.notion.so/Deploy-v2-and-updated-v3-sushiswap-subgraphs-e331945fdeea487c890706fc22c6cc94) Reviewed-on: cerc-io/stack-orchestrator#821 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-05-17 04:01:41 +00:00
dboreham	254f95e59f	Merge pull request 'Remove legacy commands for docker startup' (#826 ) from dboreham/remove-old-docker-ci into main Reviewed-on: cerc-io/stack-orchestrator#826	2024-05-15 16:09:09 +00:00
dboreham	0acb6ea6bc	Remove legacy commands for docker startup	2024-05-15 09:04:19 -06:00
prathamesh	b9369a13e6	Update watcher dashboard with panels for ETH RPC request failures and durations (#825 ) Part of [Regenerate ajna watcher with updated subgraph config ](https://www.notion.so/Regenerate-ajna-watcher-with-updated-subgraph-config-c9bbecb033024c13a7515c7f1efc3363) Requires [Add metrics to monitor errors and duration for ETH RPC requests](https://github.com/cerc-io/watcher-ts/pull/507) Reviewed-on: cerc-io/stack-orchestrator#825 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-05-15 13:52:18 +00:00
prathamesh	0b1eb8eb0f	Update ajna-watcher version in ajna stack (#824 ) Part of [Regenerate ajna watcher with updated subgraph config](https://www.notion.so/Regenerate-ajna-watcher-with-updated-subgraph-config-c9bbecb033024c13a7515c7f1efc3363) Reviewed-on: cerc-io/stack-orchestrator#824 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-05-14 08:27:59 +00:00
nabarun	78092f5793	Update subgraph watcher stacks to configure multiple RPC endpoints (#822 ) Part of [Ability to configure watchers with multiple RPC endpoints](https://www.notion.so/Ability-to-configure-watchers-with-multiple-RPC-endpoints-dc8d3ff4d647404ab718dfd5a4c9035c) Reviewed-on: cerc-io/stack-orchestrator#822 Co-authored-by: Nabarun <nabarun@deepstacksoft.com> Co-committed-by: Nabarun <nabarun@deepstacksoft.com>	2024-05-10 04:58:30 +00:00
prathameshandnabarun	247dbdd2f0	Update subgraph watcher versions for improved `eth_getLogs` calls (#820 ) Part of [Investigate subgraph watchers lagging behind head](https://www.notion.so/Investigate-subgraph-watchers-lagging-behind-head-01b72294ca8e4f658e4c0e86b36d19e2) Co-authored-by: Nabarun <nabarun@deepstacksoft.com> Reviewed-on: cerc-io/stack-orchestrator#820 Co-authored-by: Prathamesh Musale <prathamesh.musale0@gmail.com> Co-committed-by: Prathamesh Musale <prathamesh.musale0@gmail.com>	2024-05-07 04:18:45 +00:00