Configure prometheus alert manager with Slack notifications

2024-05-31 20:22:25 +05:30
45 changed files with 481 additions and 2993 deletions
@@ -4,4 +4,3 @@ Trigger
 Trigger
 Trigger
 Trigger
-Trigger
@@ -18,8 +18,6 @@ services:
      "/update-grafana-alerts-config.sh && /run.sh"
    ports:
      - "3000"
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
    healthcheck:
      test: ["CMD", "nc", "-vz", "localhost", "3000"]
      interval: 30s
@@ -22,7 +22,6 @@ services:
      GRAPH_ETHEREUM_JSON_RPC_TIMEOUT: ${GRAPH_ETHEREUM_JSON_RPC_TIMEOUT:-180}
      GRAPH_ETHEREUM_REQUEST_RETRIES: ${GRAPH_ETHEREUM_REQUEST_RETRIES:-10}
      GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE: ${GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE:-2000}
-      GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS: ${GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS:-1000}
    entrypoint: ["bash", "-c"]
    # Wait for ETH RPC endpoint to be up when running with fixturenet-lotus
    command: |
@@ -32,7 +31,6 @@ services:
      - "8001"
      - "8020"
      - "8030"
-      - "8040"
    healthcheck:
      test: ["CMD", "nc", "-vz", "localhost", "8020"]
      interval: 30s
@@ -18,6 +18,14 @@ services:
    extra_hosts:
      - "host.docker.internal:host-gateway"

+  alert-manager:
+    image: prom/alertmanager
+    volumes:
+      - ../config/monitoring/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml
+      - alertmanager_data:/data
+    ports:
+      - "9093"
+
  blackbox:
    image: prom/blackbox-exporter:latest
    restart: always
@@ -28,37 +36,15 @@ services:
    extra_hosts:
      - "host.docker.internal:host-gateway"

-  ethereum-chain-head-exporter:
+  chain-head-exporter:
    image: cerc/watcher-ts:local
    restart: always
    working_dir: /app/packages/cli
    environment:
-      ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT:-https://mainnet.infura.io/v3}
+      ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
+      FIL_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT}
      ETH_RPC_API_KEY: ${CERC_INFURA_KEY}
-    command: ["sh", "-c", "yarn export-metrics:chain-heads"]
-    ports:
-      - '5000'
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-
-  filecoin-chain-head-exporter:
-    image: cerc/watcher-ts:local
-    restart: always
-    working_dir: /app/packages/cli
-    environment:
-      ETH_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT:-https://api.node.glif.io/rpc/v1}
-    command: ["sh", "-c", "yarn export-metrics:chain-heads"]
-    ports:
-      - '5000'
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-
-  graph-node-upstream-head-exporter:
-    image: cerc/watcher-ts:local
-    restart: always
-    working_dir: /app/packages/cli
-    environment:
-      ETH_RPC_ENDPOINT: ${GRAPH_NODE_RPC_ENDPOINT}
+      PORT: ${CERC_METRICS_PORT}
    command: ["sh", "-c", "yarn export-metrics:chain-heads"]
    ports:
      - '5000'
@@ -77,3 +63,4 @@ services:

 volumes:
  prometheus_data:
+  alertmanager_data:
@@ -0,0 +1,13 @@
+services:
+  snowballtools-base-backend:
+    image: cerc/snowballtools-base-backend:local
+    restart: always
+    volumes:
+      - data:/data
+      - config:/config:ro
+    ports:
+      - 8000
+
+volumes:
+  data:
+  config:
@@ -60,7 +60,6 @@ services:
    volumes:
      - ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
      - ../config/watcher-ajna/start-server.sh:/app/start-server.sh
-      - ajna_watcher_gql_logs_data:/app/gql-logs
    ports:
      - "3008"
      - "9001"
@@ -75,4 +74,3 @@ services:

 volumes:
  ajna_watcher_db_data:
-  ajna_watcher_gql_logs_data:
@@ -74,10 +74,8 @@ services:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/azimuth-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/azimuth-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/azimuth-watcher/start-server.sh
-      - azimuth_watcher_gql_logs_data:/app/packages/azimuth-watcher/gql-logs
    ports:
      - "3001"
-      - "9001"
    healthcheck:
      test: ["CMD", "nc", "-vz", "127.0.0.1", "3001"]
      interval: 20s
@@ -137,10 +135,8 @@ services:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/censures-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/censures-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/censures-watcher/start-server.sh
-      - censures_watcher_gql_logs_data:/app/packages/censures-watcher/gql-logs
    ports:
      - "3002"
-      - "9003"
    healthcheck:
      test: ["CMD", "nc", "-vz", "127.0.0.1", "3002"]
      interval: 20s
@@ -200,10 +196,8 @@ services:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/claims-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/claims-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/claims-watcher/start-server.sh
-      - claims_watcher_gql_logs_data:/app/packages/claims-watcher/gql-logs
    ports:
      - "3003"
-      - "9005"
    healthcheck:
      test: ["CMD", "nc", "-vz", "127.0.0.1", "3003"]
      interval: 20s
@@ -263,10 +257,8 @@ services:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/conditional-star-release-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/conditional-star-release-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/conditional-star-release-watcher/start-server.sh
-      - conditional_star_release_watcher_gql_logs_data:/app/packages/conditional-star-release-watcher/gql-logs
    ports:
      - "3004"
-      - "9007"
    healthcheck:
      test: ["CMD", "nc", "-vz", "127.0.0.1", "3004"]
      interval: 20s
@@ -326,10 +318,8 @@ services:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/delegated-sending-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/delegated-sending-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/delegated-sending-watcher/start-server.sh
-      - delegated_sending_watcher_gql_logs_data:/app/packages/delegated-sending-watcher/gql-logs
    ports:
      - "3005"
-      - "9009"
    healthcheck:
      test: ["CMD", "nc", "-vz", "127.0.0.1", "3005"]
      interval: 20s
@@ -389,10 +379,8 @@ services:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/ecliptic-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/ecliptic-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/ecliptic-watcher/start-server.sh
-      - ecliptic_watcher_gql_logs_data:/app/packages/ecliptic-watcher/gql-logs
    ports:
      - "3006"
-      - "9011"
    healthcheck:
      test: ["CMD", "nc", "-vz", "127.0.0.1", "3006"]
      interval: 20s
@@ -452,10 +440,8 @@ services:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/linear-star-release-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/linear-star-release-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/linear-star-release-watcher/start-server.sh
-      - linear_star_release_watcher_gql_logs_data:/app/packages/linear-star-release-watcher/gql-logs
    ports:
      - "3007"
-      - "9013"
    healthcheck:
      test: ["CMD", "nc", "-vz", "127.0.0.1", "3007"]
      interval: 20s
@@ -515,10 +501,8 @@ services:
      - ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/polls-watcher/environments/watcher-config-template.toml
      - ../config/watcher-azimuth/merge-toml.js:/app/packages/polls-watcher/merge-toml.js
      - ../config/watcher-azimuth/start-server.sh:/app/packages/polls-watcher/start-server.sh
-      - polls_watcher_gql_logs_data:/app/packages/polls-watcher/gql-logs
    ports:
      - "3008"
-      - "9015"
    healthcheck:
      test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
      interval: 20s
@@ -568,11 +552,3 @@ services:

 volumes:
  watcher_db_data:
-  azimuth_watcher_gql_logs_data:
-  censures_watcher_gql_logs_data:
-  claims_watcher_gql_logs_data:
-  conditional_star_release_watcher_gql_logs_data:
-  delegated_sending_watcher_gql_logs_data:
-  ecliptic_watcher_gql_logs_data:
-  linear_star_release_watcher_gql_logs_data:
-  polls_watcher_gql_logs_data:
@@ -60,7 +60,6 @@ services:
    volumes:
      - ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
      - ../config/watcher-merkl-sushiswap-v3/start-server.sh:/app/start-server.sh
-      - merkl_sushiswap_v3_watcher_gql_logs_data:/app/gql-logs
    ports:
      - "127.0.0.1:3007:3008"
      - "9003:9001"
@@ -75,4 +74,3 @@ services:

 volumes:
  merkl_sushiswap_v3_watcher_db_data:
-  merkl_sushiswap_v3_watcher_gql_logs_data:
@@ -60,7 +60,6 @@ services:
    volumes:
      - ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
      - ../config/watcher-sushiswap-v3/start-server.sh:/app/start-server.sh
-      - sushiswap_v3_watcher_gql_logs_data:/app/gql-logs
    ports:
      - "127.0.0.1:3008:3008"
      - "9001:9001"
@@ -75,4 +74,3 @@ services:

 volumes:
  sushiswap_v3_watcher_db_data:
-  sushiswap_v3_watcher_gql_logs_data:
@@ -0,0 +1,10 @@
+route:
+  receiver: 'slack'
+  # group_by: [ alertname ]
+
+receivers:
+  - name: 'slack'
+    slack_configs:
+      - send_resolved: true
+        api_url: <YOUR_SLACK_WEBHOOK_URL>
+        channel: <YOUR_SLACK_CHANNEL>
@@ -1,20 +0,0 @@
-apiVersion: 1
-
-datasources:
-  - name: Graph Node Postgres
-    type: postgres
-    jsonData:
-      database: graph-node
-      sslmode: 'disable'
-      maxOpenConns: 100
-      maxIdleConns: 100
-      maxIdleConnsAuto: true
-      connMaxLifetime: 14400
-      postgresVersion: 1411 # 903=9.3, 1000=10, 1411=14.11
-      timescaledb: false
-    user: graph-node
-    # # Add URL for graph-node database
-    # url: graph-node-db:5432
-    # # Set password for graph-node database
-    # secureJsonData:
-    #   password: 'password'
@@ -3,8 +3,13 @@ global:
  evaluation_interval: 15s

 rule_files:
-  # - "first.rules"
-  # - "second.rules"
+  - rules.yml
+
+alerting:
+  alertmanagers:
+   - static_configs:
+     - targets:
+        - alert-manager:9093

 scrape_configs:
  - job_name: prometheus
@@ -45,18 +50,7 @@ scrape_configs:
    metrics_path: /metrics
    scheme: http
    static_configs:
-      - targets: ['ethereum-chain-head-exporter:5000']
-        labels:
-          instance: 'external'
-          chain: 'ethereum'
-      - targets: ['filecoin-chain-head-exporter:5000']
-        labels:
-          instance: 'external'
-          chain: 'filecoin'
-      - targets: ['graph-node-upstream-head-exporter:5000']
-        labels:
-          instance: 'graph-node'
-          chain: 'filecoin'
+      - targets: ['chain-head-exporter:5000']

  - job_name: 'postgres'
    scrape_interval: 30s
@@ -85,11 +79,3 @@ scrape_configs:
      # - targets: ['example-host:1317']
    params:
      format: ['prometheus']
-
-  - job_name: graph-node
-    metrics_path: /metrics
-    scrape_interval: 30s
-    scheme: http
-    static_configs:
-      # Add graph-node targets to be monitored below
-      # - targets: ['graph-node:8040']
@@ -0,0 +1,10 @@
+groups:
+  - name: WatcherAlerts
+    rules:
+      - alert: SushiswapWatcherHeadTracking
+        expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"} > 16
+        for: 5m
+        labels:
+          severity: 'critical'
+        annotations:
+          summary: 'Watcher {{ $labels.instance }} of job {{ $labels.job }} is falling behind external head'
@@ -4,6 +4,4 @@ echo Using CERC_GRAFANA_ALERTS_SUBGRAPH_IDS ${CERC_GRAFANA_ALERTS_SUBGRAPH_IDS}

 # Replace subgraph ids in subgraph alerting config
 # Note: Requires the grafana container to be run with user root
-if [ -n "$CERC_GRAFANA_ALERTS_SUBGRAPH_IDS" ]; then
-  sed -i "s/REPLACE_WITH_SUBGRAPH_IDS/$CERC_GRAFANA_ALERTS_SUBGRAPH_IDS/g" /etc/grafana/provisioning/alerting/subgraph-alert-rules.yml
-fi
+sed -i "s/REPLACE_WITH_SUBGRAPH_IDS/$CERC_GRAFANA_ALERTS_SUBGRAPH_IDS/g" /etc/grafana/provisioning/alerting/subgraph-alert-rules.yml
@@ -24,7 +24,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -100,7 +100,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -176,7 +176,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -252,7 +252,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -328,7 +328,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -404,7 +404,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -480,7 +480,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="linear_star_release", kind="latest_indexed"}
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="linear_star_release", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -556,7 +556,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -634,7 +634,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -710,7 +710,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -788,85 +788,7 @@ groups:
                uid: PBFA97CFB590B2093
              disableTextWrap: false
              editorMode: code
-              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="ajna", instance="ajna", kind="latest_indexed"}
-              fullMetaSearch: false
-              includeNullMetadata: true
-              instant: true
-              intervalMs: 1000
-              legendFormat: __auto
-              maxDataPoints: 43200
-              range: false
-              refId: diff
-              useBackend: false
-          - refId: latest_external
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              editorMode: code
-              expr: latest_block_number{chain="filecoin"}
-              hide: false
-              instant: true
-              legendFormat: __auto
-              range: false
-              refId: latest_external
-          - refId: condition
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: __expr__
-            model:
-              conditions:
-                - evaluator:
-                    params:
-                      - 0
-                      - 0
-                    type: gt
-                  operator:
-                    type: and
-                  query:
-                    params: []
-                  reducer:
-                    params: []
-                    type: avg
-                  type: query
-              datasource:
-                name: Expression
-                type: __expr__
-                uid: __expr__
-              expression: ${diff} >= 16
-              intervalMs: 1000
-              maxDataPoints: 43200
-              refId: condition
-              type: math
-        noDataState: Alerting
-        execErrState: Alerting
-        for: 15m
-        annotations:
-          summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
-        isPaused: false
-      
-      # Secured Finance
-      - uid: secured_finance_diff_external
-        title: secured_finance_watcher_head_tracking
-        condition: condition
-        data:
-          - refId: diff
-            relativeTimeRange:
-              from: 600
-              to: 0
-            datasourceUid: PBFA97CFB590B2093
-            model:
-              datasource:
-                type: prometheus
-                uid: PBFA97CFB590B2093
-              disableTextWrap: false
-              editorMode: code
-              expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="secured-finance", instance="secured-finance", kind="latest_indexed"}
+              expr: latest_block_number - on(chain) group_right sync_status_block_number{job="ajna", instance="ajna", kind="latest_indexed"}
              fullMetaSearch: false
              includeNullMetadata: true
              instant: true
@@ -2,6 +2,7 @@
  host = "0.0.0.0"
  port = 3008
  kind = "active"
+  gqlPath = "/"

  # Checkpointing state.
  checkpointing = true
@@ -21,30 +22,23 @@
  # Interval in number of blocks at which to clear entities cache.
  clearEntitiesCacheInterval = 1000

+  # Max block range for which to return events in eventsInRange GQL query.
+  # Use -1 for skipping check on block range.
+  maxEventsBlockRange = 1000
+
  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
  rpcSupportsBlockHashParam = false

-  # Server GQL config
-  [server.gql]
-    path = "/"
+  # GQL cache settings
+  [server.gqlCache]
+    enabled = true

-    # Max block range for which to return events in eventsInRange GQL query.
-    # Use -1 for skipping check on block range.
-    maxEventsBlockRange = 1000
+    # Max in-memory cache size (in bytes) (default 8 MB)
+    # maxCacheSize

-    # Log directory for GQL requests
-    logDir = "./gql-logs"
-
-    # GQL cache settings
-    [server.gql.cache]
-      enabled = true
-
-      # Max in-memory cache size (in bytes) (default 8 MB)
-      # maxCacheSize
-
-      # GQL cache-control max-age settings (in seconds)
-      maxAge = 15
-      timeTravelMaxAge = 86400 # 1 day
+    # GQL cache-control max-age settings (in seconds)
+    maxAge = 15
+    timeTravelMaxAge = 86400 # 1 day

 [metrics]
  host = "0.0.0.0"
@@ -91,9 +85,6 @@
  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
  blockDelayInMilliSecs = 30000

-  # Number of blocks by which block processing lags behind head
-  blockProcessingOffset = 0
-
  # Boolean to switch between modes of processing events when starting the server.
  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
@@ -1,7 +1,6 @@
 [server]
  host = "0.0.0.0"
-  [server.gql]
-    maxSimultaneousRequests = -1
+  maxSimultaneousRequests = -1

 [metrics]
  host = "0.0.0.0"
@@ -2,6 +2,7 @@
  host = "0.0.0.0"
  port = 3008
  kind = "active"
+  gqlPath = '/'

  # Checkpointing state.
  checkpointing = true
@@ -21,30 +22,23 @@
  # Interval in number of blocks at which to clear entities cache.
  clearEntitiesCacheInterval = 1000

+  # Max block range for which to return events in eventsInRange GQL query.
+  # Use -1 for skipping check on block range.
+  maxEventsBlockRange = 1000
+
  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
  rpcSupportsBlockHashParam = false

-  # Server GQL config
-  [server.gql]
-    path = "/"
+  # GQL cache settings
+  [server.gqlCache]
+    enabled = true

-    # Max block range for which to return events in eventsInRange GQL query.
-    # Use -1 for skipping check on block range.
-    maxEventsBlockRange = 1000
+    # Max in-memory cache size (in bytes) (default 8 MB)
+    # maxCacheSize

-    # Log directory for GQL requests
-    logDir = "./gql-logs"
-
-    # GQL cache settings
-    [server.gql.cache]
-      enabled = true
-
-      # Max in-memory cache size (in bytes) (default 8 MB)
-      # maxCacheSize
-
-      # GQL cache-control max-age settings (in seconds)
-      maxAge = 15
-      timeTravelMaxAge = 86400 # 1 day
+    # GQL cache-control max-age settings (in seconds)
+    maxAge = 15
+    timeTravelMaxAge = 86400 # 1 day

 [metrics]
  host = "0.0.0.0"
@@ -91,9 +85,6 @@
  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
  blockDelayInMilliSecs = 30000

-  # Number of blocks by which block processing lags behind head
-  blockProcessingOffset = 0
-
  # Boolean to switch between modes of processing events when starting the server.
  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
@@ -2,6 +2,7 @@
  host = "0.0.0.0"
  port = 3008
  kind = "active"
+  gqlPath = "/"

  # Checkpointing state.
  checkpointing = true
@@ -21,30 +22,23 @@
  # Interval in number of blocks at which to clear entities cache.
  clearEntitiesCacheInterval = 1000

+  # Max block range for which to return events in eventsInRange GQL query.
+  # Use -1 for skipping check on block range.
+  maxEventsBlockRange = 1000
+
  # Flag to specify whether RPC endpoint supports block hash as block tag parameter
  rpcSupportsBlockHashParam = false

-  # Server GQL config
-  [server.gql]
-    path = "/"
+  # GQL cache settings
+  [server.gqlCache]
+    enabled = true

-    # Max block range for which to return events in eventsInRange GQL query.
-    # Use -1 for skipping check on block range.
-    maxEventsBlockRange = 1000
+    # Max in-memory cache size (in bytes) (default 8 MB)
+    # maxCacheSize

-    # Log directory for GQL requests
-    logDir = "./gql-logs"
-
-    # GQL cache settings
-    [server.gql.cache]
-      enabled = true
-
-      # Max in-memory cache size (in bytes) (default 8 MB)
-      # maxCacheSize
-
-      # GQL cache-control max-age settings (in seconds)
-      maxAge = 15
-      timeTravelMaxAge = 86400 # 1 day
+    # GQL cache-control max-age settings (in seconds)
+    maxAge = 15
+    timeTravelMaxAge = 86400 # 1 day

 [metrics]
  host = "0.0.0.0"
@@ -91,9 +85,6 @@
  # Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
  blockDelayInMilliSecs = 30000

-  # Number of blocks by which block processing lags behind head
-  blockProcessingOffset = 0
-
  # Boolean to switch between modes of processing events when starting the server.
  # Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
  # Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
@@ -0,0 +1,6 @@
+FROM cerc/snowballtools-base-backend-base:local
+
+WORKDIR /app/packages/backend
+COPY run.sh .
+
+ENTRYPOINT ["./run.sh"]
@@ -0,0 +1,26 @@
+FROM ubuntu:22.04 as builder
+
+RUN apt update && \
+      apt install -y --no-install-recommends --no-install-suggests \
+      ca-certificates curl gnupg
+
+# Node
+ARG NODE_MAJOR=20
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
+      echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \
+      apt update && apt install -y nodejs
+
+# npm setup
+RUN npm config set @cerc-io:registry https://git.vdb.to/api/packages/cerc-io/npm/ && npm install -g yarn
+
+COPY . /app/
+WORKDIR /app/
+
+RUN find . -name 'node_modules' | xargs -n1 rm -rf
+RUN yarn && yarn build --ignore frontend
+
+FROM cerc/webapp-base:local
+
+COPY --from=builder /app /app
+
+WORKDIR /app/packages/backend
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# Build cerc/webapp-deployer-backend
+
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/snowballtools-base-backend-base:local ${build_command_args} -f ${SCRIPT_DIR}/Dockerfile-base ${CERC_REPO_BASE_DIR}/snowballtools-base
+docker build -t cerc/snowballtools-base-backend:local ${build_command_args} ${SCRIPT_DIR} 
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+
+LACONIC_HOSTED_CONFIG_FILE=${LACONIC_HOSTED_CONFIG_FILE}
+if [ -z "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
+  if [ -f "/config/laconic-hosted-config.yml" ]; then
+    LACONIC_HOSTED_CONFIG_FILE="/config/laconic-hosted-config.yml"
+  elif [ -f "/config/config.yml" ]; then
+    LACONIC_HOSTED_CONFIG_FILE="/config/config.yml"
+  fi
+fi
+
+if [ -f "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
+  /scripts/apply-webapp-config.sh $LACONIC_HOSTED_CONFIG_FILE "`pwd`/dist"
+fi
+
+/scripts/apply-runtime-env.sh "`pwd`/dist"
+
+yarn start
@@ -6,10 +6,5 @@ WORKDIR /app

 COPY . .

-# Get the latest Git commit hash and set in package.json
-RUN COMMIT_HASH=$(git rev-parse HEAD) && \
-  jq --arg commitHash "$COMMIT_HASH" '.commitHash = $commitHash' package.json > tmp.json && \
-  mv tmp.json package.json
-
 RUN echo "Installing dependencies and building ajna-watcher-ts" && \
  yarn && yarn build
@@ -1,20 +1,11 @@
 FROM node:18.16.0-alpine3.16

-RUN apk --update --no-cache add git python3 alpine-sdk jq
+RUN apk --update --no-cache add git python3 alpine-sdk

 WORKDIR /app

 COPY . .

-# Get the latest Git commit hash and set it in package.json of all watchers
-RUN COMMIT_HASH=$(git rev-parse HEAD) && \
-    find . -name 'package.json' -exec sh -c ' \
-    for packageFile; do \
-        jq --arg commitHash "$0" ".commitHash = \$commitHash" "$packageFile" > "$packageFile.tmp" && \
-        mv "$packageFile.tmp" "$packageFile"; \
-    done \
-    ' "$COMMIT_HASH" {} \;
-
 RUN echo "Building azimuth-watcher-ts" && \
    yarn && yarn build

@@ -6,10 +6,5 @@ WORKDIR /app

 COPY . .

-# Get the latest Git commit hash and set in package.json
-RUN COMMIT_HASH=$(git rev-parse HEAD) && \
-  jq --arg commitHash "$COMMIT_HASH" '.commitHash = $commitHash' package.json > tmp.json && \
-  mv tmp.json package.json
-
 RUN echo "Installing dependencies and building merkl-sushiswap-v3-watcher-ts" && \
  yarn && yarn build
@@ -6,10 +6,5 @@ WORKDIR /app

 COPY . .

-# Get the latest Git commit hash and set in package.json
-RUN COMMIT_HASH=$(git rev-parse HEAD) && \
-  jq --arg commitHash "$COMMIT_HASH" '.commitHash = $commitHash' package.json > tmp.json && \
-  mv tmp.json package.json
-
 RUN echo "Installing dependencies and building sushiswap-v3-watcher-ts" && \
  yarn && yarn build
@@ -32,7 +32,7 @@ RUN \

 # [Optional] Uncomment this section to install additional OS packages.
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends jq gettext-base git
+    && apt-get -y install --no-install-recommends jq gettext-base

 # [Optional] Uncomment if you want to install an additional version of node using nvm
 # ARG EXTRA_NODE_VERSION=10
@@ -2,7 +2,7 @@ version: "1.0"
 name: ajna
 description: "Ajna watcher stack"
 repos:
-  - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.13
+  - git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.9
 containers:
  - cerc/watcher-ajna
 pods:
@@ -44,42 +44,34 @@ network:
     - 0.0.0.0:9000:9000
    azimuth-watcher-server:
     - 0.0.0.0:3001:3001
-     - 0.0.0.0:9001:9001
    censures-watcher-job-runner:
     - 0.0.0.0:9002:9002
    censures-watcher-server:
     - 0.0.0.0:3002:3002
-     - 0.0.0.0:9003:9003
    claims-watcher-job-runner:
     - 0.0.0.0:9004:9004
    claims-watcher-server:
     - 0.0.0.0:3003:3003
-     - 0.0.0.0:9005:9005
    conditional-star-release-watcher-job-runner:
     - 0.0.0.0:9006:9006
    conditional-star-release-watcher-server:
     - 0.0.0.0:3004:3004
-     - 0.0.0.0:9007:9007
    delegated-sending-watcher-job-runner:
     - 0.0.0.0:9008:9008
    delegated-sending-watcher-server:
     - 0.0.0.0:3005:3005
-     - 0.0.0.0:9009:9009
    ecliptic-watcher-job-runner:
     - 0.0.0.0:9010:9010
    ecliptic-watcher-server:
     - 0.0.0.0:3006:3006
-     - 0.0.0.0:9011:9011
    linear-star-release-watcher-job-runner:
     - 0.0.0.0:9012:9012
    linear-star-release-watcher-server:
     - 0.0.0.0:3007:3007
-     - 0.0.0.0:9013:9013
    polls-watcher-job-runner:
     - 0.0.0.0:9014:9014
    polls-watcher-server:
     - 0.0.0.0:3008:3008
-     - 0.0.0.0:9015:9015
    gateway-server:
     - 0.0.0.0:4000:4000
 ...
@@ -1,7 +1,7 @@
 version: "1.0"
 name: azimuth
 repos:
-  - github.com/cerc-io/azimuth-watcher-ts@0.1.6
+  - github.com/cerc-io/azimuth-watcher-ts@v0.1.4
 containers:
  - cerc/watcher-azimuth
 pods:
@@ -43,18 +43,16 @@ customized by editing the "spec" file generated by `laconic-so deploy init`.
 ```
 $ cat graph-node-spec.yml
 stack: graph-node
-network:
-  ports:
-    graph-node:
-      - '8000:8000'
-      - '8001'
-      - '8020:8020'
-      - '8030'
-      - '8040'
-    ipfs:
-      - '8080'
-      - '4001'
-      - '5001:5001'
+ports:
+  graph-node:
+    - '8000:8000'
+    - '8001'
+    - '8020:8020'
+    - '8030'
+  ipfs:
+    - '8080'
+    - '4001'
+    - '5001:5001'
 ...
 ```

@@ -66,7 +64,7 @@ laconic-so --stack graph-node deploy create --spec-file graph-node-spec.yml --de

 ## Start the stack

-Update `config.env` file inside deployment directory with the following values before starting the stack:
+Create an env file with the following values to be set before starting the stack:

 ```bash
 # Set ETH RPC endpoint the graph node will use
@@ -90,13 +88,10 @@ export GRAPH_ETHEREUM_REQUEST_RETRIES=
 # Maximum number of blocks to scan for triggers in each request (default: 2000)
 export GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE=

-# Maximum number of concurrent requests made against Ethereum for requesting transaction receipts during block ingestion (default: 1000)
-export GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS=
-
 # Ref: https://git.vdb.to/cerc-io/graph-node/src/branch/master/docs/environment-variables.md
 ```

-Example `config.env` file:
+Example env file:

 ```bash
 export ETH_RPC_HOST=filecoin.chainup.net
@@ -109,6 +104,12 @@ export GRAPH_ETHEREUM_REQUEST_RETRIES=5
 export GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE=50
 ```

+Set the environment variables:
+
+```bash
+source <PATH_TO_ENV_FILE>
+```
+
 Deploy the stack:

 ```bash
@@ -3,9 +3,7 @@ name: graph-node
 description: "Stack for running graph-node"
 repos:
  - github.com/graphprotocol/graph-node
-  - github.com/cerc-io/watcher-ts@v0.2.92
 containers:
  - cerc/graph-node
-  - cerc/watcher-ts
 pods:
  - graph-node
@@ -14,8 +14,7 @@
 # along with this program.  If not, see <http:#www.gnu.org/licenses/>.

 from stack_orchestrator.util import get_yaml
-from stack_orchestrator.deploy.deploy_types import DeployCommandContext, LaconicStackSetupCommand
-from stack_orchestrator.deploy.deployment_context import DeploymentContext
+from stack_orchestrator.deploy.deploy_types import DeployCommandContext, LaconicStackSetupCommand, DeploymentContext
 from stack_orchestrator.deploy.stack_state import State
 from stack_orchestrator.deploy.deploy_util import VolumeMapping, run_container_command
 from stack_orchestrator.command_types import CommandOptions
@@ -2,7 +2,7 @@ version: "1.0"
 name: merkl-sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.14
+  - github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.11
 containers:
  - cerc/watcher-merkl-sushiswap-v3
 pods:
@@ -134,29 +134,6 @@ Note: Use `host.docker.internal` as host to access ports on the host machine

 Place the dashboard json files in grafana dashboards config directory (`monitoring-deployment/config/monitoring/grafana/dashboards`) in the deployment folder

-#### Graph Node Config
-
-For graph-node dashboard postgres datasource needs to be setup in `monitoring-deployment/config/monitoring/grafana/provisioning/datasources/graph-node-postgres.yml` (in deployment folder)
-
-```yml
-# graph-node-postgres.yml
-...
-datasources:
-  - name: Graph Node Postgres
-    type: postgres
-    jsonData:
-      # Set name to remote graph-node database name
-      database: graph-node
-      ...
-    # Set user to remote graph-node database username
-    user: graph-node
-    # Add URL for remote graph-node database
-    url: graph-node-db:5432
-    # Set password for graph-node database
-    secureJsonData:
-      password: 'password'
-```
-
 ### Env

 Set the following env variables in the deployment env config file (`monitoring-deployment/config.env`):
@@ -179,11 +156,6 @@ Set the following env variables in the deployment env config file (`monitoring-d
  # Grafana server host URL (used in various links in alerts, etc.)
  # (Optional, default: http://localhost:3000)
  GF_SERVER_ROOT_URL=
-
-  
-  # RPC endpoint used by graph-node for upstream head metric
-  # (Optional, default: https://mainnet.infura.io/v3)
-  GRAPH_NODE_RPC_ENDPOINT=
  ```

 ## Start the stack
@@ -57,35 +57,35 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
    metrics_path: /metrics
    scheme: http
    static_configs:
-      - targets: ['AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_METRICS_PORT', 'AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_GQL_METRICS_PORT']
+      - targets: ['AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_PORT']
        labels:
          instance: 'azimuth'
          chain: 'ethereum'
-      - targets: ['CENSURES_WATCHER_HOST:CENSURES_WATCHER_METRICS_PORT', 'CENSURES_WATCHER_HOST:CENSURES_WATCHER_GQL_METRICS_PORT']
+      - targets: ['CENSURES_WATCHER_HOST:CENSURES_WATCHER_PORT']
        labels:
          instance: 'censures'
          chain: 'ethereum'
-      - targets: ['CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_METRICS_PORT', 'CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_GQL_METRICS_PORT']
+      - targets: ['CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_PORT']
        labels:
          instance: 'claims'
          chain: 'ethereum'
-      - targets: ['CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_METRICS_PORT', 'CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_GQL_METRICS_PORT']
+      - targets: ['CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_PORT']
        labels:
          instance: 'conditional_star_release'
          chain: 'ethereum'
-      - targets: ['DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_METRICS_PORT', 'DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_GQL_METRICS_PORT']
+      - targets: ['DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_PORT']
        labels:
          instance: 'delegated_sending'
          chain: 'ethereum'
-      - targets: ['ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_METRICS_PORT', 'ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_GQL_METRICS_PORT']
+      - targets: ['ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_PORT']
        labels:
          instance: 'ecliptic'
          chain: 'ethereum'
-      - targets: ['LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_METRICS_PORT', 'LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_GQL_METRICS_PORT']
+      - targets: ['LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_PORT']
        labels:
          instance: 'linear_star_release'
          chain: 'ethereum'
-      - targets: ['POLLS_WATCHER_HOST:POLLS_WATCHER_METRICS_PORT', 'POLLS_WATCHER_HOST:POLLS_WATCHER_GQL_METRICS_PORT']
+      - targets: ['POLLS_WATCHER_HOST:POLLS_WATCHER_PORT']
        labels:
          instance: 'polls'
          chain: 'ethereum'
@@ -95,11 +95,11 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
    metrics_path: /metrics
    scheme: http
    static_configs:
-      - targets: ['SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_METRICS_PORT', 'SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_GQL_METRICS_PORT']
+      - targets: ['SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_PORT']
        labels:
          instance: 'sushiswap'
          chain: 'filecoin'
-      - targets: ['MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_METRICS_PORT', 'MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_GQL_METRICS_PORT']
+      - targets: ['MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_PORT']
        labels:
          instance: 'merkl_sushiswap'
          chain: 'filecoin'
@@ -109,7 +109,7 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
    metrics_path: /metrics
    scheme: http
    static_configs:
-      - targets: ['AJNA_WATCHER_HOST:AJNA_WATCHER_METRICS_PORT', 'AJNA_WATCHER_HOST:AJNA_WATCHER_GQL_METRICS_PORT']
+      - targets: ['AJNA_WATCHER_HOST:AJNA_WATCHER_PORT']
        labels:
          instance: 'ajna'
          chain: 'filecoin'
@@ -1,7 +1,7 @@
 version: "0.1"
 name: monitoring
 repos:
-  - github.com/cerc-io/watcher-ts@v0.2.92
+  - github.com/cerc-io/watcher-ts@v0.2.81
 containers:
  - cerc/watcher-ts
 pods:
@@ -0,0 +1,10 @@
+version: "1.0"
+name: snowballtools-base-backend
+description: "snowballtools-base-backend"
+repos:
+  - github.com/snowball-tools/snowballtools-base
+containers:
+  - cerc/webapp-base
+  - cerc/snowballtools-base-backend
+pods:
+  - snowballtools-base-backend
@@ -2,7 +2,7 @@ version: "1.0"
 name: sushiswap-v3
 description: "SushiSwap v3 watcher stack"
 repos:
-  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.14
+  - github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.11
 containers:
  - cerc/watcher-sushiswap-v3
 pods:
@@ -29,29 +29,16 @@ def _image_needs_pushed(image: str):
    return image.endswith(":local")


-def _remote_tag_for_image(image: str, remote_repo_url: str):
-    # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
-    major_parts = image.split("/", 2)
-    image_name_with_version = major_parts[1] if 2 == len(major_parts) else major_parts[0]
-    (image_name, image_version) = image_name_with_version.split(":")
-    if image_version == "local":
-        return f"{remote_repo_url}/{image_name}:deploy"
-    else:
-        return image
-
-
-# Note: do not add any calls this function
 def remote_image_exists(remote_repo_url: str, local_tag: str):
    docker = DockerClient()
    try:
-        remote_tag = _remote_tag_for_image(local_tag, remote_repo_url)
+        remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
        result = docker.manifest.inspect(remote_tag)
        return True if result else False
    except Exception:  # noqa: E722
        return False


-# Note: do not add any calls this function
 def add_tags_to_image(remote_repo_url: str, local_tag: str, *additional_tags):
    if not additional_tags:
        return
@@ -60,20 +47,18 @@ def add_tags_to_image(remote_repo_url: str, local_tag: str, *additional_tags):
        raise Exception(f"{local_tag} does not exist in {remote_repo_url}")

    docker = DockerClient()
-    remote_tag = _remote_tag_for_image(local_tag, remote_repo_url)
-    new_remote_tags = [_remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags]
+    remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
+    new_remote_tags = [remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags]
    docker.buildx.imagetools.create(sources=[remote_tag], tags=new_remote_tags)


-def remote_tag_for_image_unique(image: str, remote_repo_url: str, deployment_id: str):
+def remote_tag_for_image(image: str, remote_repo_url: str):
    # Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
    major_parts = image.split("/", 2)
    image_name_with_version = major_parts[1] if 2 == len(major_parts) else major_parts[0]
    (image_name, image_version) = image_name_with_version.split(":")
    if image_version == "local":
-        # Salt the tag with part of the deployment id to make it unique to this deployment
-        deployment_tag = deployment_id[-8:]
-        return f"{remote_repo_url}/{image_name}:deploy-{deployment_tag}"
+        return f"{remote_repo_url}/{image_name}:deploy"
    else:
        return image

@@ -88,14 +73,14 @@ def push_images_operation(command_context: DeployCommandContext, deployment_cont
    docker = DockerClient()
    for image in images:
        if _image_needs_pushed(image):
-            remote_tag = remote_tag_for_image_unique(image, remote_repo_url, deployment_context.id)
+            remote_tag = remote_tag_for_image(image, remote_repo_url)
            if opts.o.verbose:
                print(f"Tagging {image} to {remote_tag}")
            docker.image.tag(image, remote_tag)
    # Run docker push commands to upload
    for image in images:
        if _image_needs_pushed(image):
-            remote_tag = remote_tag_for_image_unique(image, remote_repo_url, deployment_context.id)
+            remote_tag = remote_tag_for_image(image, remote_repo_url)
            if opts.o.verbose:
                print(f"Pushing image {remote_tag}")
            docker.image.push(remote_tag)
@@ -26,7 +26,7 @@ from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variable
 from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment
 from stack_orchestrator.deploy.deploy_types import DeployEnvVars
 from stack_orchestrator.deploy.spec import Spec, Resources, ResourceLimits
-from stack_orchestrator.deploy.images import remote_tag_for_image_unique
+from stack_orchestrator.deploy.images import remote_tag_for_image

 DEFAULT_VOLUME_RESOURCES = Resources({
    "reservations": {"storage": "2Gi"}
@@ -326,11 +326,8 @@ class ClusterInfo:
                if opts.o.debug:
                    print(f"Merged envs: {envs}")
                # Re-write the image tag for remote deployment
-                # Note self.app_name has the same value as deployment_id
-                image_to_use = remote_tag_for_image_unique(
-                    image,
-                    self.spec.get_image_registry(),
-                    self.app_name) if self.spec.get_image_registry() is not None else image
+                image_to_use = remote_tag_for_image(
+                    image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image
                volume_mounts = volume_mounts_for_service(self.parsed_pod_yaml_map, service_name)
                container = client.V1Container(
                    name=container_name,
@@ -22,16 +22,16 @@ echo "$(date +"%Y-%m-%d %T"): Stack started"
 # Verify that the fixturenet is up and running
 $TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd ps

-# Wait for the laconid endpoint to come up
-echo "Waiting for the RPC endpoint to come up"
-docker exec laconicd-laconicd-1 sh -c "curl --retry 20 --retry-delay 3 --retry-connrefused http://127.0.0.1:9473/api"
-
 # Get the fixturenet account address
 laconicd_account_address=$(docker exec laconicd-laconicd-1 laconicd keys list | awk '/- address:/ {print $3}')

 # Copy over config
 docker exec laconicd-cli-1 cp config.yml laconic-registry-cli/

+# Wait for the laconid endpoint to come up
+echo "Waiting for the RPC endpoint to come up"
+docker exec laconicd-laconicd-1 sh -c "curl --retry 20 --retry-delay 3 --retry-connrefused http://127.0.0.1:9473/api"
+
 # Run the tests
 echo "Running the tests"
 docker exec -e TEST_ACCOUNT=$laconicd_account_address laconicd-cli-1 sh -c 'cd laconic-registry-cli && yarn && yarn test'